Browse Source

Merge branch 'stable-5.5' into stable-5.6

* stable-5.5:
  Fix string format parameter for invalidRefAdvertisementLine
  WindowCache: add metric for cached bytes per repository
  pgm daemon: fallback to user and system config if no config specified
  WindowCache: add option to use strong refs to reference ByteWindows
  Replace usage of ArrayIndexOutOfBoundsException in treewalk
  Add config constants for WindowCache configuration options

Change-Id: I73d16b53df02bf735c2431588143efe225a4b5b4
stable-5.6
Matthias Sohn 5 years ago
parent
commit
3d59d1b80c
  1. 25
      org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Daemon.java
  2. 68
      org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/WindowCacheGetTest.java
  3. 44
      org.eclipse.jgit/.settings/.api_filters
  4. 2
      org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties
  5. 386
      org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCache.java
  6. 30
      org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java
  7. 64
      org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCacheConfig.java
  8. 9
      org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCacheStats.java
  9. 6
      org.eclipse.jgit/src/org/eclipse/jgit/treewalk/AbstractTreeIterator.java
  10. 9
      org.eclipse.jgit/src/org/eclipse/jgit/treewalk/CanonicalTreeParser.java

25
org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Daemon.java

@ -44,6 +44,7 @@
package org.eclipse.jgit.pgm; package org.eclipse.jgit.pgm;
import java.io.File; import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress; import java.net.InetSocketAddress;
import java.net.URISyntaxException; import java.net.URISyntaxException;
import java.text.MessageFormat; import java.text.MessageFormat;
@ -51,12 +52,14 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import org.eclipse.jgit.errors.ConfigInvalidException;
import org.eclipse.jgit.internal.ketch.KetchLeader; import org.eclipse.jgit.internal.ketch.KetchLeader;
import org.eclipse.jgit.internal.ketch.KetchLeaderCache; import org.eclipse.jgit.internal.ketch.KetchLeaderCache;
import org.eclipse.jgit.internal.ketch.KetchPreReceive; import org.eclipse.jgit.internal.ketch.KetchPreReceive;
import org.eclipse.jgit.internal.ketch.KetchSystem; import org.eclipse.jgit.internal.ketch.KetchSystem;
import org.eclipse.jgit.internal.ketch.KetchText; import org.eclipse.jgit.internal.ketch.KetchText;
import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.lib.StoredConfig;
import org.eclipse.jgit.pgm.internal.CLIText; import org.eclipse.jgit.pgm.internal.CLIText;
import org.eclipse.jgit.storage.file.FileBasedConfig; import org.eclipse.jgit.storage.file.FileBasedConfig;
import org.eclipse.jgit.storage.file.WindowCacheConfig; import org.eclipse.jgit.storage.file.WindowCacheConfig;
@ -68,6 +71,7 @@ import org.eclipse.jgit.transport.resolver.FileResolver;
import org.eclipse.jgit.transport.resolver.ReceivePackFactory; import org.eclipse.jgit.transport.resolver.ReceivePackFactory;
import org.eclipse.jgit.transport.resolver.ServiceNotEnabledException; import org.eclipse.jgit.transport.resolver.ServiceNotEnabledException;
import org.eclipse.jgit.util.FS; import org.eclipse.jgit.util.FS;
import org.eclipse.jgit.util.SystemReader;
import org.kohsuke.args4j.Argument; import org.kohsuke.args4j.Argument;
import org.kohsuke.args4j.Option; import org.kohsuke.args4j.Option;
@ -120,19 +124,20 @@ class Daemon extends TextBuiltin {
@Override @Override
protected void run() throws Exception { protected void run() throws Exception {
PackConfig packConfig = new PackConfig(); PackConfig packConfig = new PackConfig();
StoredConfig cfg;
if (configFile != null) { if (configFile == null) {
cfg = getUserConfig();
} else {
if (!configFile.exists()) { if (!configFile.exists()) {
throw die(MessageFormat.format( throw die(MessageFormat.format(
CLIText.get().configFileNotFound, // CLIText.get().configFileNotFound, //
configFile.getAbsolutePath())); configFile.getAbsolutePath()));
} }
cfg = new FileBasedConfig(configFile, FS.DETECTED);
FileBasedConfig cfg = new FileBasedConfig(configFile, FS.DETECTED); }
cfg.load(); cfg.load();
new WindowCacheConfig().fromConfig(cfg).install(); new WindowCacheConfig().fromConfig(cfg).install();
packConfig.fromConfig(cfg); packConfig.fromConfig(cfg);
}
int threads = packConfig.getThreads(); int threads = packConfig.getThreads();
if (threads <= 0) if (threads <= 0)
@ -172,6 +177,16 @@ class Daemon extends TextBuiltin {
outw.println(MessageFormat.format(CLIText.get().listeningOn, d.getAddress())); outw.println(MessageFormat.format(CLIText.get().listeningOn, d.getAddress()));
} }
private StoredConfig getUserConfig() throws IOException {
StoredConfig userConfig = null;
try {
userConfig = SystemReader.getInstance().getUserConfig();
} catch (ConfigInvalidException e) {
throw die(e.getMessage());
}
return userConfig;
}
private static DaemonService service( private static DaemonService service(
final org.eclipse.jgit.transport.Daemon d, final org.eclipse.jgit.transport.Daemon d,
final String n) { final String n) {

68
org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/WindowCacheGetTest.java

@ -53,6 +53,8 @@ import java.io.FileInputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStreamReader; import java.io.InputStreamReader;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List; import java.util.List;
import org.eclipse.jgit.errors.CorruptObjectException; import org.eclipse.jgit.errors.CorruptObjectException;
@ -66,9 +68,25 @@ import org.eclipse.jgit.test.resources.SampleDataRepositoryTestCase;
import org.eclipse.jgit.util.MutableInteger; import org.eclipse.jgit.util.MutableInteger;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
@RunWith(Parameterized.class)
public class WindowCacheGetTest extends SampleDataRepositoryTestCase { public class WindowCacheGetTest extends SampleDataRepositoryTestCase {
private List<TestObject> toLoad; private List<TestObject> toLoad;
private WindowCacheConfig cfg;
private boolean useStrongRefs;
@Parameters(name = "useStrongRefs={0}")
public static Collection<Object[]> data() {
return Arrays
.asList(new Object[][] { { Boolean.TRUE }, { Boolean.FALSE } });
}
public WindowCacheGetTest(Boolean useStrongRef) {
this.useStrongRefs = useStrongRef.booleanValue();
}
@Override @Override
@Before @Before
@ -93,11 +111,12 @@ public class WindowCacheGetTest extends SampleDataRepositoryTestCase {
} }
} }
assertEquals(96, toLoad.size()); assertEquals(96, toLoad.size());
cfg = new WindowCacheConfig();
cfg.setPackedGitUseStrongRefs(useStrongRefs);
} }
@Test @Test
public void testCache_Defaults() throws IOException { public void testCache_Defaults() throws IOException {
WindowCacheConfig cfg = new WindowCacheConfig();
cfg.install(); cfg.install();
doCacheTests(); doCacheTests();
checkLimits(cfg); checkLimits(cfg);
@ -122,7 +141,6 @@ public class WindowCacheGetTest extends SampleDataRepositoryTestCase {
@Test @Test
public void testCache_TooFewFiles() throws IOException { public void testCache_TooFewFiles() throws IOException {
final WindowCacheConfig cfg = new WindowCacheConfig();
cfg.setPackedGitOpenFiles(2); cfg.setPackedGitOpenFiles(2);
cfg.install(); cfg.install();
doCacheTests(); doCacheTests();
@ -131,7 +149,6 @@ public class WindowCacheGetTest extends SampleDataRepositoryTestCase {
@Test @Test
public void testCache_TooSmallLimit() throws IOException { public void testCache_TooSmallLimit() throws IOException {
final WindowCacheConfig cfg = new WindowCacheConfig();
cfg.setPackedGitWindowSize(4096); cfg.setPackedGitWindowSize(4096);
cfg.setPackedGitLimit(4096); cfg.setPackedGitLimit(4096);
cfg.install(); cfg.install();
@ -142,26 +159,31 @@ public class WindowCacheGetTest extends SampleDataRepositoryTestCase {
private static void checkLimits(WindowCacheConfig cfg) { private static void checkLimits(WindowCacheConfig cfg) {
final WindowCache cache = WindowCache.getInstance(); final WindowCache cache = WindowCache.getInstance();
WindowCacheStats s = cache.getStats(); WindowCacheStats s = cache.getStats();
assertTrue(0 < s.getAverageLoadTime()); assertTrue("average load time should be > 0",
assertTrue(0 < s.getOpenByteCount()); 0 < s.getAverageLoadTime());
assertTrue(0 < s.getOpenByteCount()); assertTrue("open byte count should be > 0", 0 < s.getOpenByteCount());
assertTrue(0.0 < s.getAverageLoadTime()); assertTrue("eviction count should be >= 0", 0 <= s.getEvictionCount());
assertTrue(0 <= s.getEvictionCount()); assertTrue("hit count should be > 0", 0 < s.getHitCount());
assertTrue(0 < s.getHitCount()); assertTrue("hit ratio should be > 0", 0 < s.getHitRatio());
assertTrue(0 < s.getHitRatio()); assertTrue("hit ratio should be < 1", 1 > s.getHitRatio());
assertTrue(1 > s.getHitRatio()); assertTrue("load count should be > 0", 0 < s.getLoadCount());
assertTrue(0 < s.getLoadCount()); assertTrue("load failure count should be >= 0",
assertTrue(0 <= s.getLoadFailureCount()); 0 <= s.getLoadFailureCount());
assertTrue(0.0 <= s.getLoadFailureRatio()); assertTrue("load failure ratio should be >= 0",
assertTrue(1 > s.getLoadFailureRatio()); 0.0 <= s.getLoadFailureRatio());
assertTrue(0 < s.getLoadSuccessCount()); assertTrue("load failure ratio should be < 1",
assertTrue(s.getOpenByteCount() <= cfg.getPackedGitLimit()); 1 > s.getLoadFailureRatio());
assertTrue(s.getOpenFileCount() <= cfg.getPackedGitOpenFiles()); assertTrue("load success count should be > 0",
assertTrue(0 <= s.getMissCount()); 0 < s.getLoadSuccessCount());
assertTrue(0 <= s.getMissRatio()); assertTrue("open byte count should be <= core.packedGitLimit",
assertTrue(1 > s.getMissRatio()); s.getOpenByteCount() <= cfg.getPackedGitLimit());
assertTrue(0 < s.getRequestCount()); assertTrue("open file count should be <= core.packedGitOpenFiles",
assertTrue(0 < s.getTotalLoadTime()); s.getOpenFileCount() <= cfg.getPackedGitOpenFiles());
assertTrue("miss success count should be >= 0", 0 <= s.getMissCount());
assertTrue("miss ratio should be > 0", 0 <= s.getMissRatio());
assertTrue("miss ratio should be < 1", 1 > s.getMissRatio());
assertTrue("request count should be > 0", 0 < s.getRequestCount());
assertTrue("total load time should be > 0", 0 < s.getTotalLoadTime());
} }
private void doCacheTests() throws IOException { private void doCacheTests() throws IOException {

44
org.eclipse.jgit/.settings/.api_filters

@ -23,6 +23,36 @@
<message_argument value="CONFIG_JMX_SECTION"/> <message_argument value="CONFIG_JMX_SECTION"/>
</message_arguments> </message_arguments>
</filter> </filter>
<filter id="1142947843">
<message_arguments>
<message_argument value="5.1.13"/>
<message_argument value="CONFIG_KEY_PACKED_GIT_LIMIT"/>
</message_arguments>
</filter>
<filter id="1142947843">
<message_arguments>
<message_argument value="5.1.13"/>
<message_argument value="CONFIG_KEY_PACKED_GIT_MMAP"/>
</message_arguments>
</filter>
<filter id="1142947843">
<message_arguments>
<message_argument value="5.1.13"/>
<message_argument value="CONFIG_KEY_PACKED_GIT_OPENFILES"/>
</message_arguments>
</filter>
<filter id="1142947843">
<message_arguments>
<message_argument value="5.1.13"/>
<message_argument value="CONFIG_KEY_PACKED_GIT_USE_STRONGREFS"/>
</message_arguments>
</filter>
<filter id="1142947843">
<message_arguments>
<message_argument value="5.1.13"/>
<message_argument value="CONFIG_KEY_PACKED_GIT_WINDOWSIZE"/>
</message_arguments>
</filter>
</resource> </resource>
<resource path="src/org/eclipse/jgit/lib/Constants.java" type="org.eclipse.jgit.lib.Constants"> <resource path="src/org/eclipse/jgit/lib/Constants.java" type="org.eclipse.jgit.lib.Constants">
<filter id="1142947843"> <filter id="1142947843">
@ -40,6 +70,20 @@
</message_arguments> </message_arguments>
</filter> </filter>
</resource> </resource>
<resource path="src/org/eclipse/jgit/storage/file/WindowCacheConfig.java" type="org.eclipse.jgit.storage.file.WindowCacheConfig">
<filter id="1142947843">
<message_arguments>
<message_argument value="5.1.13"/>
<message_argument value="isPackedGitUseStrongRefs()"/>
</message_arguments>
</filter>
<filter id="1142947843">
<message_arguments>
<message_argument value="5.1.13"/>
<message_argument value="setPackedGitUseStrongRefs(boolean)"/>
</message_arguments>
</filter>
</resource>
<resource path="src/org/eclipse/jgit/storage/file/WindowCacheStats.java" type="org.eclipse.jgit.storage.file.WindowCacheStats"> <resource path="src/org/eclipse/jgit/storage/file/WindowCacheStats.java" type="org.eclipse.jgit.storage.file.WindowCacheStats">
<filter id="337809484"> <filter id="337809484">
<message_arguments> <message_arguments>

2
org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties

@ -372,7 +372,7 @@ invalidPacketLineHeader=Invalid packet line header: {0}
invalidPath=Invalid path: {0} invalidPath=Invalid path: {0}
invalidPurgeFactor=Invalid purgeFactor {0}, values have to be in range between 0 and 1 invalidPurgeFactor=Invalid purgeFactor {0}, values have to be in range between 0 and 1
invalidRedirectLocation=Invalid redirect location {0} -> {1} invalidRedirectLocation=Invalid redirect location {0} -> {1}
invalidRefAdvertisementLine=Invalid ref advertisement line: ''{1}'' invalidRefAdvertisementLine=Invalid ref advertisement line: ''{0}''
invalidReflogRevision=Invalid reflog revision: {0} invalidReflogRevision=Invalid reflog revision: {0}
invalidRefName=Invalid ref name: {0} invalidRefName=Invalid ref name: {0}
invalidReftableBlock=Invalid reftable block invalidReftableBlock=Invalid reftable block

386
org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCache.java

@ -47,11 +47,16 @@ package org.eclipse.jgit.internal.storage.file;
import java.io.IOException; import java.io.IOException;
import java.lang.ref.ReferenceQueue; import java.lang.ref.ReferenceQueue;
import java.lang.ref.SoftReference; import java.lang.ref.SoftReference;
import java.util.Collections;
import java.util.Map;
import java.util.Random; import java.util.Random;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReferenceArray; import java.util.concurrent.atomic.AtomicReferenceArray;
import java.util.concurrent.atomic.LongAdder; import java.util.concurrent.atomic.LongAdder;
import java.util.concurrent.locks.ReentrantLock; import java.util.concurrent.locks.ReentrantLock;
import java.util.stream.Collectors;
import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.annotations.NonNull;
import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.JGitText;
@ -85,9 +90,16 @@ import org.eclipse.jgit.util.Monitoring;
* comprised of roughly 10% of the cache, and evicting the oldest accessed entry * comprised of roughly 10% of the cache, and evicting the oldest accessed entry
* within that window. * within that window.
* <p> * <p>
* Entities created by the cache are held under SoftReferences, permitting the * Entities created by the cache are held under SoftReferences if option
* {@code core.packedGitUseStrongRefs} is set to {@code false} in the git config
* (this is the default) or by calling
* {@link WindowCacheConfig#setPackedGitUseStrongRefs(boolean)}, permitting the
* Java runtime's garbage collector to evict entries when heap memory gets low. * Java runtime's garbage collector to evict entries when heap memory gets low.
* Most JREs implement a loose least recently used algorithm for this eviction. * Most JREs implement a loose least recently used algorithm for this eviction.
* When this option is set to {@code true} strong references are used which
* means that Java gc cannot evict the WindowCache to reclaim memory. On the
* other hand this provides more predictable performance since the cache isn't
* flushed when used heap comes close to the maximum heap size.
* <p> * <p>
* The internal hash table does not expand at runtime, instead it is fixed in * The internal hash table does not expand at runtime, instead it is fixed in
* size at cache creation time. The internal lock table used to gate load * size at cache creation time. The internal lock table used to gate load
@ -104,19 +116,19 @@ import org.eclipse.jgit.util.Monitoring;
* for a given <code>(PackFile,position)</code> tuple.</li> * for a given <code>(PackFile,position)</code> tuple.</li>
* <li>For every <code>load()</code> invocation there is exactly one * <li>For every <code>load()</code> invocation there is exactly one
* {@link #createRef(PackFile, long, ByteWindow)} invocation to wrap a * {@link #createRef(PackFile, long, ByteWindow)} invocation to wrap a
* SoftReference around the cached entity.</li> * SoftReference or a StrongReference around the cached entity.</li>
* <li>For every Reference created by <code>createRef()</code> there will be * <li>For every Reference created by <code>createRef()</code> there will be
* exactly one call to {@link #clear(Ref)} to cleanup any resources associated * exactly one call to {@link #clear(PageRef)} to cleanup any resources associated
* with the (now expired) cached entity.</li> * with the (now expired) cached entity.</li>
* </ul> * </ul>
* <p> * <p>
* Therefore, it is safe to perform resource accounting increments during the * Therefore, it is safe to perform resource accounting increments during the
* {@link #load(PackFile, long)} or * {@link #load(PackFile, long)} or
* {@link #createRef(PackFile, long, ByteWindow)} methods, and matching * {@link #createRef(PackFile, long, ByteWindow)} methods, and matching
* decrements during {@link #clear(Ref)}. Implementors may need to override * decrements during {@link #clear(PageRef)}. Implementors may need to override
* {@link #createRef(PackFile, long, ByteWindow)} in order to embed additional * {@link #createRef(PackFile, long, ByteWindow)} in order to embed additional
* accounting information into an implementation specific * accounting information into an implementation specific
* {@link org.eclipse.jgit.internal.storage.file.WindowCache.Ref} subclass, as * {@link org.eclipse.jgit.internal.storage.file.WindowCache.PageRef} subclass, as
* the cached entity may have already been evicted by the JRE's garbage * the cached entity may have already been evicted by the JRE's garbage
* collector. * collector.
* <p> * <p>
@ -176,18 +188,21 @@ public class WindowCache {
/** /**
* Record files opened by cache * Record files opened by cache
* *
* @param count * @param delta
* delta of number of files opened by cache * delta of number of files opened by cache
*/ */
void recordOpenFiles(int count); void recordOpenFiles(int delta);
/** /**
* Record cached bytes * Record cached bytes
* *
* @param count * @param pack
* pack file the bytes are read from
*
* @param delta
* delta of cached bytes * delta of cached bytes
*/ */
void recordOpenBytes(int count); void recordOpenBytes(PackFile pack, int delta);
/** /**
* Returns a snapshot of this recorder's stats. Note that this may be an * Returns a snapshot of this recorder's stats. Note that this may be an
@ -209,6 +224,7 @@ public class WindowCache {
private final LongAdder evictionCount; private final LongAdder evictionCount;
private final LongAdder openFileCount; private final LongAdder openFileCount;
private final LongAdder openByteCount; private final LongAdder openByteCount;
private final Map<String, LongAdder> openByteCountPerRepository;
/** /**
* Constructs an instance with all counts initialized to zero. * Constructs an instance with all counts initialized to zero.
@ -222,6 +238,7 @@ public class WindowCache {
evictionCount = new LongAdder(); evictionCount = new LongAdder();
openFileCount = new LongAdder(); openFileCount = new LongAdder();
openByteCount = new LongAdder(); openByteCount = new LongAdder();
openByteCountPerRepository = new ConcurrentHashMap<>();
} }
@Override @Override
@ -252,13 +269,28 @@ public class WindowCache {
} }
@Override @Override
public void recordOpenFiles(int count) { public void recordOpenFiles(int delta) {
openFileCount.add(count); openFileCount.add(delta);
} }
@Override @Override
public void recordOpenBytes(int count) { public void recordOpenBytes(PackFile pack, int delta) {
openByteCount.add(count); openByteCount.add(delta);
String repositoryId = repositoryId(pack);
LongAdder la = openByteCountPerRepository
.computeIfAbsent(repositoryId, k -> new LongAdder());
la.add(delta);
if (delta < 0) {
openByteCountPerRepository.computeIfPresent(repositoryId,
(k, v) -> v.longValue() == 0 ? null : v);
}
}
private static String repositoryId(PackFile pack) {
// use repository's gitdir since packfile doesn't know its
// repository
return pack.getPackFile().getParentFile().getParentFile()
.getParent();
} }
@Override @Override
@ -315,6 +347,15 @@ public class WindowCache {
totalLoadTime.reset(); totalLoadTime.reset();
evictionCount.reset(); evictionCount.reset();
} }
@Override
public Map<String, Long> getOpenByteCountPerRepository() {
return Collections.unmodifiableMap(
openByteCountPerRepository.entrySet().stream()
.collect(Collectors.toMap(Map.Entry::getKey,
e -> Long.valueOf(e.getValue().sum()),
(u, v) -> v)));
}
} }
private static final int bits(int newSize) { private static final int bits(int newSize) {
@ -390,8 +431,8 @@ public class WindowCache {
cache.removeAll(pack); cache.removeAll(pack);
} }
/** ReferenceQueue to cleanup released and garbage collected windows. */ /** cleanup released and/or garbage collected windows. */
private final ReferenceQueue<ByteWindow> queue; private final CleanupQueue queue;
/** Number of entries in {@link #table}. */ /** Number of entries in {@link #table}. */
private final int tableSize; private final int tableSize;
@ -425,6 +466,8 @@ public class WindowCache {
private final StatsRecorderImpl mbean; private final StatsRecorderImpl mbean;
private boolean useStrongRefs;
private WindowCache(WindowCacheConfig cfg) { private WindowCache(WindowCacheConfig cfg) {
tableSize = tableSize(cfg); tableSize = tableSize(cfg);
final int lockCount = lockCount(cfg); final int lockCount = lockCount(cfg);
@ -433,7 +476,6 @@ public class WindowCache {
if (lockCount < 1) if (lockCount < 1)
throw new IllegalArgumentException(JGitText.get().lockCountMustBeGreaterOrEqual1); throw new IllegalArgumentException(JGitText.get().lockCountMustBeGreaterOrEqual1);
queue = new ReferenceQueue<>();
clock = new AtomicLong(1); clock = new AtomicLong(1);
table = new AtomicReferenceArray<>(tableSize); table = new AtomicReferenceArray<>(tableSize);
locks = new Lock[lockCount]; locks = new Lock[lockCount];
@ -455,6 +497,9 @@ public class WindowCache {
mmap = cfg.isPackedGitMMAP(); mmap = cfg.isPackedGitMMAP();
windowSizeShift = bits(cfg.getPackedGitWindowSize()); windowSizeShift = bits(cfg.getPackedGitWindowSize());
windowSize = 1 << windowSizeShift; windowSize = 1 << windowSizeShift;
useStrongRefs = cfg.isPackedGitUseStrongRefs();
queue = useStrongRefs ? new StrongCleanupQueue(this)
: new SoftCleanupQueue(this);
mbean = new StatsRecorderImpl(); mbean = new StatsRecorderImpl();
statsRecorder = mbean; statsRecorder = mbean;
@ -503,16 +548,18 @@ public class WindowCache {
} }
} }
private Ref createRef(PackFile p, long o, ByteWindow v) { private PageRef<ByteWindow> createRef(PackFile p, long o, ByteWindow v) {
final Ref ref = new Ref(p, o, v, queue); final PageRef<ByteWindow> ref = useStrongRefs
statsRecorder.recordOpenBytes(ref.size); ? new StrongRef(p, o, v, queue)
: new SoftRef(p, o, v, (SoftCleanupQueue) queue);
statsRecorder.recordOpenBytes(ref.getPack(), ref.getSize());
return ref; return ref;
} }
private void clear(Ref ref) { private void clear(PageRef<ByteWindow> ref) {
statsRecorder.recordOpenBytes(-ref.size); statsRecorder.recordOpenBytes(ref.getPack(), -ref.getSize());
statsRecorder.recordEvictions(1); statsRecorder.recordEvictions(1);
close(ref.pack); close(ref.getPack());
} }
private void close(PackFile pack) { private void close(PackFile pack) {
@ -577,7 +624,7 @@ public class WindowCache {
} }
v = load(pack, position); v = load(pack, position);
final Ref ref = createRef(pack, position, v); final PageRef<ByteWindow> ref = createRef(pack, position, v);
hit(ref); hit(ref);
for (;;) { for (;;) {
final Entry n = new Entry(clean(e2), ref); final Entry n = new Entry(clean(e2), ref);
@ -601,8 +648,8 @@ public class WindowCache {
private ByteWindow scan(Entry n, PackFile pack, long position) { private ByteWindow scan(Entry n, PackFile pack, long position) {
for (; n != null; n = n.next) { for (; n != null; n = n.next) {
final Ref r = n.ref; final PageRef<ByteWindow> r = n.ref;
if (r.pack == pack && r.position == position) { if (r.getPack() == pack && r.getPosition() == position) {
final ByteWindow v = r.get(); final ByteWindow v = r.get();
if (v != null) { if (v != null) {
hit(r); hit(r);
@ -615,7 +662,7 @@ public class WindowCache {
return null; return null;
} }
private void hit(Ref r) { private void hit(PageRef r) {
// We don't need to be 100% accurate here. Its sufficient that at least // We don't need to be 100% accurate here. Its sufficient that at least
// one thread performs the increment. Any other concurrent access at // one thread performs the increment. Any other concurrent access at
// exactly the same time can simply use the same clock value. // exactly the same time can simply use the same clock value.
@ -625,7 +672,7 @@ public class WindowCache {
// //
final long c = clock.get(); final long c = clock.get();
clock.compareAndSet(c, c + 1); clock.compareAndSet(c, c + 1);
r.lastAccess = c; r.setLastAccess(c);
} }
private void evict() { private void evict() {
@ -639,7 +686,8 @@ public class WindowCache {
for (Entry e = table.get(ptr); e != null; e = e.next) { for (Entry e = table.get(ptr); e != null; e = e.next) {
if (e.dead) if (e.dead)
continue; continue;
if (old == null || e.ref.lastAccess < old.ref.lastAccess) { if (old == null || e.ref.getLastAccess() < old.ref
.getLastAccess()) {
old = e; old = e;
slot = ptr; slot = ptr;
} }
@ -659,7 +707,7 @@ public class WindowCache {
* <p> * <p>
* This is a last-ditch effort to clear out the cache, such as before it * This is a last-ditch effort to clear out the cache, such as before it
* gets replaced by another cache that is configured differently. This * gets replaced by another cache that is configured differently. This
* method tries to force every cached entry through {@link #clear(Ref)} to * method tries to force every cached entry through {@link #clear(PageRef)} to
* ensure that resources are correctly accounted for and cleaned up by the * ensure that resources are correctly accounted for and cleaned up by the
* subclass. A concurrent reader loading entries while this method is * subclass. A concurrent reader loading entries while this method is
* running may cause resource accounting failures. * running may cause resource accounting failures.
@ -692,7 +740,7 @@ public class WindowCache {
final Entry e1 = table.get(s); final Entry e1 = table.get(s);
boolean hasDead = false; boolean hasDead = false;
for (Entry e = e1; e != null; e = e.next) { for (Entry e = e1; e != null; e = e.next) {
if (e.ref.pack == pack) { if (e.ref.getPack() == pack) {
e.kill(); e.kill();
hasDead = true; hasDead = true;
} else if (e.dead) } else if (e.dead)
@ -705,20 +753,7 @@ public class WindowCache {
} }
private void gc() { private void gc() {
Ref r; queue.gc();
while ((r = (Ref) queue.poll()) != null) {
clear(r);
final int s = slot(r.pack, r.position);
final Entry e1 = table.get(s);
for (Entry n = e1; n != null; n = n.next) {
if (n.ref == r) {
n.dead = true;
table.compareAndSet(s, e1, clean(e1));
break;
}
}
}
} }
private int slot(PackFile pack, long position) { private int slot(PackFile pack, long position) {
@ -731,7 +766,7 @@ public class WindowCache {
private static Entry clean(Entry top) { private static Entry clean(Entry top) {
while (top != null && top.dead) { while (top != null && top.dead) {
top.ref.enqueue(); top.ref.kill();
top = top.next; top = top.next;
} }
if (top == null) if (top == null)
@ -745,7 +780,7 @@ public class WindowCache {
final Entry next; final Entry next;
/** The referenced object. */ /** The referenced object. */
final Ref ref; final PageRef<ByteWindow> ref;
/** /**
* Marked true when ref.get() returns null and the ref is dead. * Marked true when ref.get() returns null and the ref is dead.
@ -756,34 +791,275 @@ public class WindowCache {
*/ */
volatile boolean dead; volatile boolean dead;
Entry(Entry n, Ref r) { Entry(Entry n, PageRef<ByteWindow> r) {
next = n; next = n;
ref = r; ref = r;
} }
final void kill() { final void kill() {
dead = true; dead = true;
ref.enqueue(); ref.kill();
}
} }
private static interface PageRef<T> {
/**
* Returns this reference object's referent. If this reference object
* has been cleared, either by the program or by the garbage collector,
* then this method returns <code>null</code>.
*
* @return The object to which this reference refers, or
* <code>null</code> if this reference object has been cleared
*/
T get();
/**
* Kill this ref
*
* @return <code>true</code> if this reference object was successfully
* killed; <code>false</code> if it was already killed
*/
boolean kill();
/**
* Get the packfile the referenced cache page is allocated for
*
* @return the packfile the referenced cache page is allocated for
*/
PackFile getPack();
/**
* Get the position of the referenced cache page in the packfile
*
* @return the position of the referenced cache page in the packfile
*/
long getPosition();
/**
* Get size of cache page
*
* @return size of cache page
*/
int getSize();
/**
* Get pseudo time of last access to this cache page
*
* @return pseudo time of last access to this cache page
*/
long getLastAccess();
/**
* Set pseudo time of last access to this cache page
*
* @param time
* pseudo time of last access to this cache page
*/
void setLastAccess(long time);
/**
* Whether this is a strong reference.
* @return {@code true} if this is a strong reference
*/
boolean isStrongRef();
} }
/** A soft reference wrapped around a cached object. */ /** A soft reference wrapped around a cached object. */
private static class Ref extends SoftReference<ByteWindow> { private static class SoftRef extends SoftReference<ByteWindow>
final PackFile pack; implements PageRef<ByteWindow> {
private final PackFile pack;
final long position; private final long position;
final int size; private final int size;
long lastAccess; private long lastAccess;
protected Ref(final PackFile pack, final long position, protected SoftRef(final PackFile pack, final long position,
final ByteWindow v, final ReferenceQueue<ByteWindow> queue) { final ByteWindow v, final SoftCleanupQueue queue) {
super(v, queue); super(v, queue);
this.pack = pack; this.pack = pack;
this.position = position; this.position = position;
this.size = v.size(); this.size = v.size();
} }
@Override
public PackFile getPack() {
return pack;
}
@Override
public long getPosition() {
return position;
}
@Override
public int getSize() {
return size;
}
@Override
public long getLastAccess() {
return lastAccess;
}
@Override
public void setLastAccess(long time) {
this.lastAccess = time;
}
@Override
public boolean kill() {
return enqueue();
}
@Override
public boolean isStrongRef() {
return false;
}
}
/** A strong reference wrapped around a cached object. */
private static class StrongRef implements PageRef<ByteWindow> {
private ByteWindow referent;
private final PackFile pack;
private final long position;
private final int size;
private long lastAccess;
private CleanupQueue queue;
protected StrongRef(final PackFile pack, final long position,
final ByteWindow v, final CleanupQueue queue) {
this.pack = pack;
this.position = position;
this.referent = v;
this.size = v.size();
this.queue = queue;
}
@Override
public PackFile getPack() {
return pack;
}
@Override
public long getPosition() {
return position;
}
@Override
public int getSize() {
return size;
}
@Override
public long getLastAccess() {
return lastAccess;
}
@Override
public void setLastAccess(long time) {
this.lastAccess = time;
}
@Override
public ByteWindow get() {
return referent;
}
@Override
public boolean kill() {
if (referent == null) {
return false;
}
referent = null;
return queue.enqueue(this);
}
@Override
public boolean isStrongRef() {
return true;
}
}
private static interface CleanupQueue {
boolean enqueue(PageRef<ByteWindow> r);
void gc();
}
private static class SoftCleanupQueue extends ReferenceQueue<ByteWindow>
implements CleanupQueue {
private final WindowCache wc;
SoftCleanupQueue(WindowCache cache) {
this.wc = cache;
}
@Override
public boolean enqueue(PageRef<ByteWindow> r) {
// no need to explicitly add soft references which are enqueued by
// the JVM
return false;
}
@Override
public void gc() {
SoftRef r;
while ((r = (SoftRef) poll()) != null) {
wc.clear(r);
final int s = wc.slot(r.getPack(), r.getPosition());
final Entry e1 = wc.table.get(s);
for (Entry n = e1; n != null; n = n.next) {
if (n.ref == r) {
n.dead = true;
wc.table.compareAndSet(s, e1, clean(e1));
break;
}
}
}
}
}
private static class StrongCleanupQueue implements CleanupQueue {
private final WindowCache wc;
private final ConcurrentLinkedQueue<PageRef<ByteWindow>> queue = new ConcurrentLinkedQueue<>();
StrongCleanupQueue(WindowCache wc) {
this.wc = wc;
}
@Override
public boolean enqueue(PageRef<ByteWindow> r) {
if (queue.contains(r)) {
return false;
}
return queue.add(r);
}
@Override
public void gc() {
PageRef<ByteWindow> r;
while ((r = queue.poll()) != null) {
wc.clear(r);
final int s = wc.slot(r.getPack(), r.getPosition());
final Entry e1 = wc.table.get(s);
for (Entry n = e1; n != null; n = n.next) {
if (n.ref == r) {
n.dead = true;
wc.table.compareAndSet(s, e1, clean(e1));
break;
}
}
}
}
} }
private static final class Lock { private static final class Lock {

30
org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java

@ -242,6 +242,36 @@ public final class ConfigConstants {
/** The "streamFileThreshold" key */ /** The "streamFileThreshold" key */
public static final String CONFIG_KEY_STREAM_FILE_TRESHOLD = "streamFileThreshold"; public static final String CONFIG_KEY_STREAM_FILE_TRESHOLD = "streamFileThreshold";
/**
* The "packedGitMmap" key
* @since 5.1.13
*/
public static final String CONFIG_KEY_PACKED_GIT_MMAP = "packedgitmmap";
/**
* The "packedGitWindowSize" key
* @since 5.1.13
*/
public static final String CONFIG_KEY_PACKED_GIT_WINDOWSIZE = "packedgitwindowsize";
/**
* The "packedGitLimit" key
* @since 5.1.13
*/
public static final String CONFIG_KEY_PACKED_GIT_LIMIT = "packedgitlimit";
/**
* The "packedGitOpenFiles" key
* @since 5.1.13
*/
public static final String CONFIG_KEY_PACKED_GIT_OPENFILES = "packedgitopenfiles";
/**
* The "packedGitUseStrongRefs" key
* @since 5.1.13
*/
public static final String CONFIG_KEY_PACKED_GIT_USE_STRONGREFS = "packedgitusestrongrefs";
/** The "remote" key */ /** The "remote" key */
public static final String CONFIG_KEY_REMOTE = "remote"; public static final String CONFIG_KEY_REMOTE = "remote";

64
org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCacheConfig.java

@ -43,6 +43,15 @@
package org.eclipse.jgit.storage.file; package org.eclipse.jgit.storage.file;
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_CORE_SECTION;
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_DELTA_BASE_CACHE_LIMIT;
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_PACKED_GIT_LIMIT;
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_PACKED_GIT_MMAP;
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_PACKED_GIT_OPENFILES;
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_PACKED_GIT_WINDOWSIZE;
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_STREAM_FILE_TRESHOLD;
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_PACKED_GIT_USE_STRONGREFS;
import org.eclipse.jgit.internal.storage.file.WindowCache; import org.eclipse.jgit.internal.storage.file.WindowCache;
import org.eclipse.jgit.lib.Config; import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.storage.pack.PackConfig; import org.eclipse.jgit.storage.pack.PackConfig;
@ -61,6 +70,8 @@ public class WindowCacheConfig {
private long packedGitLimit; private long packedGitLimit;
private boolean useStrongRefs;
private int packedGitWindowSize; private int packedGitWindowSize;
private boolean packedGitMMAP; private boolean packedGitMMAP;
@ -75,6 +86,7 @@ public class WindowCacheConfig {
public WindowCacheConfig() { public WindowCacheConfig() {
packedGitOpenFiles = 128; packedGitOpenFiles = 128;
packedGitLimit = 10 * MB; packedGitLimit = 10 * MB;
useStrongRefs = false;
packedGitWindowSize = 8 * KB; packedGitWindowSize = 8 * KB;
packedGitMMAP = false; packedGitMMAP = false;
deltaBaseCacheLimit = 10 * MB; deltaBaseCacheLimit = 10 * MB;
@ -125,6 +137,31 @@ public class WindowCacheConfig {
packedGitLimit = newLimit; packedGitLimit = newLimit;
} }
/**
* Get whether the window cache should use strong references or
* SoftReferences
*
* @return {@code true} if the window cache should use strong references,
* otherwise it will use {@link java.lang.ref.SoftReference}s
* @since 5.1.13
*/
public boolean isPackedGitUseStrongRefs() {
return useStrongRefs;
}
/**
* Set if the cache should use strong refs or soft refs
*
* @param useStrongRefs
* if @{code true} the cache strongly references cache pages
* otherwise it uses {@link java.lang.ref.SoftReference}s which
* can be evicted by the Java gc if heap is almost full
* @since 5.1.13
*/
public void setPackedGitUseStrongRefs(boolean useStrongRefs) {
this.useStrongRefs = useStrongRefs;
}
/** /**
* Get size in bytes of a single window mapped or read in from the pack * Get size in bytes of a single window mapped or read in from the pack
* file. * file.
@ -227,20 +264,23 @@ public class WindowCacheConfig {
* @since 3.0 * @since 3.0
*/ */
public WindowCacheConfig fromConfig(Config rc) { public WindowCacheConfig fromConfig(Config rc) {
setPackedGitOpenFiles(rc.getInt( setPackedGitUseStrongRefs(rc.getBoolean(CONFIG_CORE_SECTION,
"core", null, "packedgitopenfiles", getPackedGitOpenFiles())); //$NON-NLS-1$ //$NON-NLS-2$ CONFIG_KEY_PACKED_GIT_USE_STRONGREFS,
setPackedGitLimit(rc.getLong( isPackedGitUseStrongRefs()));
"core", null, "packedgitlimit", getPackedGitLimit())); //$NON-NLS-1$ //$NON-NLS-2$ setPackedGitOpenFiles(rc.getInt(CONFIG_CORE_SECTION, null,
setPackedGitWindowSize(rc.getInt( CONFIG_KEY_PACKED_GIT_OPENFILES, getPackedGitOpenFiles()));
"core", null, "packedgitwindowsize", getPackedGitWindowSize())); //$NON-NLS-1$ //$NON-NLS-2$ setPackedGitLimit(rc.getLong(CONFIG_CORE_SECTION, null,
setPackedGitMMAP(rc.getBoolean( CONFIG_KEY_PACKED_GIT_LIMIT, getPackedGitLimit()));
"core", null, "packedgitmmap", isPackedGitMMAP())); //$NON-NLS-1$ //$NON-NLS-2$ setPackedGitWindowSize(rc.getInt(CONFIG_CORE_SECTION, null,
setDeltaBaseCacheLimit(rc.getInt( CONFIG_KEY_PACKED_GIT_WINDOWSIZE, getPackedGitWindowSize()));
"core", null, "deltabasecachelimit", getDeltaBaseCacheLimit())); //$NON-NLS-1$ //$NON-NLS-2$ setPackedGitMMAP(rc.getBoolean(CONFIG_CORE_SECTION, null,
CONFIG_KEY_PACKED_GIT_MMAP, isPackedGitMMAP()));
setDeltaBaseCacheLimit(rc.getInt(CONFIG_CORE_SECTION, null,
CONFIG_KEY_DELTA_BASE_CACHE_LIMIT, getDeltaBaseCacheLimit()));
long maxMem = Runtime.getRuntime().maxMemory(); long maxMem = Runtime.getRuntime().maxMemory();
long sft = rc.getLong( long sft = rc.getLong(CONFIG_CORE_SECTION, null,
"core", null, "streamfilethreshold", getStreamFileThreshold()); //$NON-NLS-1$ //$NON-NLS-2$ CONFIG_KEY_STREAM_FILE_TRESHOLD, getStreamFileThreshold());
sft = Math.min(sft, maxMem / 4); // don't use more than 1/4 of the heap sft = Math.min(sft, maxMem / 4); // don't use more than 1/4 of the heap
sft = Math.min(sft, Integer.MAX_VALUE); // cannot exceed array length sft = Math.min(sft, Integer.MAX_VALUE); // cannot exceed array length
setStreamFileThreshold((int) sft); setStreamFileThreshold((int) sft);

9
org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCacheStats.java

@ -42,6 +42,8 @@
*/ */
package org.eclipse.jgit.storage.file; package org.eclipse.jgit.storage.file;
import java.util.Map;
import javax.management.MXBean; import javax.management.MXBean;
import org.eclipse.jgit.internal.storage.file.WindowCache; import org.eclipse.jgit.internal.storage.file.WindowCache;
@ -225,6 +227,13 @@ public interface WindowCacheStats {
*/ */
long getOpenByteCount(); long getOpenByteCount();
/**
* Number of bytes cached per repository
*
* @return number of bytes cached per repository
*/
Map<String, Long> getOpenByteCountPerRepository();
/** /**
* Reset counters. Does not reset open bytes and open files counters. * Reset counters. Does not reset open bytes and open files counters.
*/ */

6
org.eclipse.jgit/src/org/eclipse/jgit/treewalk/AbstractTreeIterator.java

@ -239,12 +239,10 @@ public abstract class AbstractTreeIterator {
path = p.path; path = p.path;
pathOffset = p.pathLen + 1; pathOffset = p.pathLen + 1;
try { if (pathOffset > path.length) {
path[pathOffset - 1] = '/';
} catch (ArrayIndexOutOfBoundsException e) {
growPath(p.pathLen); growPath(p.pathLen);
path[pathOffset - 1] = '/';
} }
path[pathOffset - 1] = '/';
} }
/** /**

9
org.eclipse.jgit/src/org/eclipse/jgit/treewalk/CanonicalTreeParser.java

@ -387,14 +387,13 @@ public class CanonicalTreeParser extends AbstractTreeIterator {
tmp = pathOffset; tmp = pathOffset;
for (;; tmp++) { for (;; tmp++) {
c = raw[ptr++]; c = raw[ptr++];
if (c == 0) if (c == 0) {
break; break;
try { }
path[tmp] = c; if (tmp >= path.length) {
} catch (ArrayIndexOutOfBoundsException e) {
growPath(tmp); growPath(tmp);
path[tmp] = c;
} }
path[tmp] = c;
} }
pathLen = tmp; pathLen = tmp;
nextPtr = ptr + OBJECT_ID_LENGTH; nextPtr = ptr + OBJECT_ID_LENGTH;

Loading…
Cancel
Save