@ -47,11 +47,16 @@ package org.eclipse.jgit.internal.storage.file;
import java.io.IOException ; import java.io.IOException ;
import java.lang.ref.ReferenceQueue ; import java.lang.ref.ReferenceQueue ;
import java.lang.ref.SoftReference ; import java.lang.ref.SoftReference ;
import java.util.Collections ;
import java.util.Map ;
import java.util.Random ; import java.util.Random ;
import java.util.concurrent.ConcurrentHashMap ;
import java.util.concurrent.ConcurrentLinkedQueue ;
import java.util.concurrent.atomic.AtomicLong ; import java.util.concurrent.atomic.AtomicLong ;
import java.util.concurrent.atomic.AtomicReferenceArray ; import java.util.concurrent.atomic.AtomicReferenceArray ;
import java.util.concurrent.atomic.LongAdder ; import java.util.concurrent.atomic.LongAdder ;
import java.util.concurrent.locks.ReentrantLock ; import java.util.concurrent.locks.ReentrantLock ;
import java.util.stream.Collectors ;
import org.eclipse.jgit.annotations.NonNull ; import org.eclipse.jgit.annotations.NonNull ;
import org.eclipse.jgit.internal.JGitText ; import org.eclipse.jgit.internal.JGitText ;
@ -85,9 +90,16 @@ import org.eclipse.jgit.util.Monitoring;
* comprised of roughly 10 % of the cache , and evicting the oldest accessed entry * comprised of roughly 10 % of the cache , and evicting the oldest accessed entry
* within that window . * within that window .
* < p > * < p >
* Entities created by the cache are held under SoftReferences , permitting the * Entities created by the cache are held under SoftReferences if option
* { @code core . packedGitUseStrongRefs } is set to { @code false } in the git config
* ( this is the default ) or by calling
* { @link WindowCacheConfig # setPackedGitUseStrongRefs ( boolean ) } , permitting the
* Java runtime ' s garbage collector to evict entries when heap memory gets low . * Java runtime ' s garbage collector to evict entries when heap memory gets low .
* Most JREs implement a loose least recently used algorithm for this eviction . * Most JREs implement a loose least recently used algorithm for this eviction .
* When this option is set to { @code true } strong references are used which
* means that Java gc cannot evict the WindowCache to reclaim memory . On the
* other hand this provides more predictable performance since the cache isn ' t
* flushed when used heap comes close to the maximum heap size .
* < p > * < p >
* The internal hash table does not expand at runtime , instead it is fixed in * The internal hash table does not expand at runtime , instead it is fixed in
* size at cache creation time . The internal lock table used to gate load * size at cache creation time . The internal lock table used to gate load
@ -104,19 +116,19 @@ import org.eclipse.jgit.util.Monitoring;
* for a given < code > ( PackFile , position ) < / code > tuple . < / li > * for a given < code > ( PackFile , position ) < / code > tuple . < / li >
* < li > For every < code > load ( ) < / code > invocation there is exactly one * < li > For every < code > load ( ) < / code > invocation there is exactly one
* { @link # createRef ( PackFile , long , ByteWindow ) } invocation to wrap a * { @link # createRef ( PackFile , long , ByteWindow ) } invocation to wrap a
* SoftReference around the cached entity . < / li > * SoftReference or a StrongReference around the cached entity . < / li >
* < li > For every Reference created by < code > createRef ( ) < / code > there will be * < li > For every Reference created by < code > createRef ( ) < / code > there will be
* exactly one call to { @link # clear ( Ref ) } to cleanup any resources associated * exactly one call to { @link # clear ( Page Ref) } to cleanup any resources associated
* with the ( now expired ) cached entity . < / li > * with the ( now expired ) cached entity . < / li >
* < / ul > * < / ul >
* < p > * < p >
* Therefore , it is safe to perform resource accounting increments during the * Therefore , it is safe to perform resource accounting increments during the
* { @link # load ( PackFile , long ) } or * { @link # load ( PackFile , long ) } or
* { @link # createRef ( PackFile , long , ByteWindow ) } methods , and matching * { @link # createRef ( PackFile , long , ByteWindow ) } methods , and matching
* decrements during { @link # clear ( Ref ) } . Implementors may need to override * decrements during { @link # clear ( Page Ref) } . Implementors may need to override
* { @link # createRef ( PackFile , long , ByteWindow ) } in order to embed additional * { @link # createRef ( PackFile , long , ByteWindow ) } in order to embed additional
* accounting information into an implementation specific * accounting information into an implementation specific
* { @link org . eclipse . jgit . internal . storage . file . WindowCache . Ref } subclass , as * { @link org . eclipse . jgit . internal . storage . file . WindowCache . Page Ref} subclass , as
* the cached entity may have already been evicted by the JRE ' s garbage * the cached entity may have already been evicted by the JRE ' s garbage
* collector . * collector .
* < p > * < p >
@ -176,18 +188,21 @@ public class WindowCache {
/ * * / * *
* Record files opened by cache * Record files opened by cache
* *
* @param count * @param delta
* delta of number of files opened by cache * delta of number of files opened by cache
* / * /
void recordOpenFiles ( int count ) ; void recordOpenFiles ( int delta ) ;
/ * * / * *
* Record cached bytes * Record cached bytes
* *
* @param count * @param pack
* pack file the bytes are read from
*
* @param delta
* delta of cached bytes * delta of cached bytes
* / * /
void recordOpenBytes ( int count ) ; void recordOpenBytes ( PackFile pack , int delta ) ;
/ * * / * *
* Returns a snapshot of this recorder ' s stats . Note that this may be an * Returns a snapshot of this recorder ' s stats . Note that this may be an
@ -209,6 +224,7 @@ public class WindowCache {
private final LongAdder evictionCount ; private final LongAdder evictionCount ;
private final LongAdder openFileCount ; private final LongAdder openFileCount ;
private final LongAdder openByteCount ; private final LongAdder openByteCount ;
private final Map < String , LongAdder > openByteCountPerRepository ;
/ * * / * *
* Constructs an instance with all counts initialized to zero . * Constructs an instance with all counts initialized to zero .
@ -222,6 +238,7 @@ public class WindowCache {
evictionCount = new LongAdder ( ) ; evictionCount = new LongAdder ( ) ;
openFileCount = new LongAdder ( ) ; openFileCount = new LongAdder ( ) ;
openByteCount = new LongAdder ( ) ; openByteCount = new LongAdder ( ) ;
openByteCountPerRepository = new ConcurrentHashMap < > ( ) ;
} }
@Override @Override
@ -252,13 +269,28 @@ public class WindowCache {
} }
@Override @Override
public void recordOpenFiles ( int count ) { public void recordOpenFiles ( int delta ) {
openFileCount . add ( count ) ; openFileCount . add ( delta ) ;
} }
@Override @Override
public void recordOpenBytes ( int count ) { public void recordOpenBytes ( PackFile pack , int delta ) {
openByteCount . add ( count ) ; openByteCount . add ( delta ) ;
String repositoryId = repositoryId ( pack ) ;
LongAdder la = openByteCountPerRepository
. computeIfAbsent ( repositoryId , k - > new LongAdder ( ) ) ;
la . add ( delta ) ;
if ( delta < 0 ) {
openByteCountPerRepository . computeIfPresent ( repositoryId ,
( k , v ) - > v . longValue ( ) = = 0 ? null : v ) ;
}
}
private static String repositoryId ( PackFile pack ) {
// use repository's gitdir since packfile doesn't know its
// repository
return pack . getPackFile ( ) . getParentFile ( ) . getParentFile ( )
. getParent ( ) ;
} }
@Override @Override
@ -315,6 +347,15 @@ public class WindowCache {
totalLoadTime . reset ( ) ; totalLoadTime . reset ( ) ;
evictionCount . reset ( ) ; evictionCount . reset ( ) ;
} }
@Override
public Map < String , Long > getOpenByteCountPerRepository ( ) {
return Collections . unmodifiableMap (
openByteCountPerRepository . entrySet ( ) . stream ( )
. collect ( Collectors . toMap ( Map . Entry : : getKey ,
e - > Long . valueOf ( e . getValue ( ) . sum ( ) ) ,
( u , v ) - > v ) ) ) ;
}
} }
private static final int bits ( int newSize ) { private static final int bits ( int newSize ) {
@ -390,8 +431,8 @@ public class WindowCache {
cache . removeAll ( pack ) ; cache . removeAll ( pack ) ;
} }
/** ReferenceQueue to cleanup released and garbage collected windows. */ /** cleanup released and/or garbage collected windows. */
private final ReferenceQueue < ByteWindow > queue ; private final CleanupQueue queue ;
/** Number of entries in {@link #table}. */ /** Number of entries in {@link #table}. */
private final int tableSize ; private final int tableSize ;
@ -425,6 +466,8 @@ public class WindowCache {
private final StatsRecorderImpl mbean ; private final StatsRecorderImpl mbean ;
private boolean useStrongRefs ;
private WindowCache ( WindowCacheConfig cfg ) { private WindowCache ( WindowCacheConfig cfg ) {
tableSize = tableSize ( cfg ) ; tableSize = tableSize ( cfg ) ;
final int lockCount = lockCount ( cfg ) ; final int lockCount = lockCount ( cfg ) ;
@ -433,7 +476,6 @@ public class WindowCache {
if ( lockCount < 1 ) if ( lockCount < 1 )
throw new IllegalArgumentException ( JGitText . get ( ) . lockCountMustBeGreaterOrEqual1 ) ; throw new IllegalArgumentException ( JGitText . get ( ) . lockCountMustBeGreaterOrEqual1 ) ;
queue = new ReferenceQueue < > ( ) ;
clock = new AtomicLong ( 1 ) ; clock = new AtomicLong ( 1 ) ;
table = new AtomicReferenceArray < > ( tableSize ) ; table = new AtomicReferenceArray < > ( tableSize ) ;
locks = new Lock [ lockCount ] ; locks = new Lock [ lockCount ] ;
@ -455,6 +497,9 @@ public class WindowCache {
mmap = cfg . isPackedGitMMAP ( ) ; mmap = cfg . isPackedGitMMAP ( ) ;
windowSizeShift = bits ( cfg . getPackedGitWindowSize ( ) ) ; windowSizeShift = bits ( cfg . getPackedGitWindowSize ( ) ) ;
windowSize = 1 < < windowSizeShift ; windowSize = 1 < < windowSizeShift ;
useStrongRefs = cfg . isPackedGitUseStrongRefs ( ) ;
queue = useStrongRefs ? new StrongCleanupQueue ( this )
: new SoftCleanupQueue ( this ) ;
mbean = new StatsRecorderImpl ( ) ; mbean = new StatsRecorderImpl ( ) ;
statsRecorder = mbean ; statsRecorder = mbean ;
@ -503,16 +548,18 @@ public class WindowCache {
} }
} }
private Ref createRef ( PackFile p , long o , ByteWindow v ) { private PageRef < ByteWindow > createRef ( PackFile p , long o , ByteWindow v ) {
final Ref ref = new Ref ( p , o , v , queue ) ; final PageRef < ByteWindow > ref = useStrongRefs
statsRecorder . recordOpenBytes ( ref . size ) ; ? new StrongRef ( p , o , v , queue )
: new SoftRef ( p , o , v , ( SoftCleanupQueue ) queue ) ;
statsRecorder . recordOpenBytes ( ref . getPack ( ) , ref . getSize ( ) ) ;
return ref ; return ref ;
} }
private void clear ( Ref ref ) { private void clear ( Page Ref< ByteWindow > ref ) {
statsRecorder . recordOpenBytes ( - ref . size ) ; statsRecorder . recordOpenBytes ( ref . getPack ( ) , - ref . getSize ( ) ) ;
statsRecorder . recordEvictions ( 1 ) ; statsRecorder . recordEvictions ( 1 ) ;
close ( ref . pack ) ; close ( ref . getPack ( ) ) ;
} }
private void close ( PackFile pack ) { private void close ( PackFile pack ) {
@ -577,7 +624,7 @@ public class WindowCache {
} }
v = load ( pack , position ) ; v = load ( pack , position ) ;
final Ref ref = createRef ( pack , position , v ) ; final Page Ref< ByteWindow > ref = createRef ( pack , position , v ) ;
hit ( ref ) ; hit ( ref ) ;
for ( ; ; ) { for ( ; ; ) {
final Entry n = new Entry ( clean ( e2 ) , ref ) ; final Entry n = new Entry ( clean ( e2 ) , ref ) ;
@ -601,8 +648,8 @@ public class WindowCache {
private ByteWindow scan ( Entry n , PackFile pack , long position ) { private ByteWindow scan ( Entry n , PackFile pack , long position ) {
for ( ; n ! = null ; n = n . next ) { for ( ; n ! = null ; n = n . next ) {
final Ref r = n . ref ; final Page Ref< ByteWindow > r = n . ref ;
if ( r . pack = = pack & & r . position = = position ) { if ( r . getPack ( ) = = pack & & r . getPosition ( ) = = position ) {
final ByteWindow v = r . get ( ) ; final ByteWindow v = r . get ( ) ;
if ( v ! = null ) { if ( v ! = null ) {
hit ( r ) ; hit ( r ) ;
@ -615,7 +662,7 @@ public class WindowCache {
return null ; return null ;
} }
private void hit ( Ref r ) { private void hit ( Page Refr ) {
// We don't need to be 100% accurate here. Its sufficient that at least
// We don't need to be 100% accurate here. Its sufficient that at least
// one thread performs the increment. Any other concurrent access at
// one thread performs the increment. Any other concurrent access at
// exactly the same time can simply use the same clock value.
// exactly the same time can simply use the same clock value.
@ -625,7 +672,7 @@ public class WindowCache {
//
//
final long c = clock . get ( ) ; final long c = clock . get ( ) ;
clock . compareAndSet ( c , c + 1 ) ; clock . compareAndSet ( c , c + 1 ) ;
r . lastAccess = c ; r . setLastAccess ( c ) ;
} }
private void evict ( ) { private void evict ( ) {
@ -639,7 +686,8 @@ public class WindowCache {
for ( Entry e = table . get ( ptr ) ; e ! = null ; e = e . next ) { for ( Entry e = table . get ( ptr ) ; e ! = null ; e = e . next ) {
if ( e . dead ) if ( e . dead )
continue ; continue ;
if ( old = = null | | e . ref . lastAccess < old . ref . lastAccess ) { if ( old = = null | | e . ref . getLastAccess ( ) < old . ref
. getLastAccess ( ) ) {
old = e ; old = e ;
slot = ptr ; slot = ptr ;
} }
@ -659,7 +707,7 @@ public class WindowCache {
* < p > * < p >
* This is a last - ditch effort to clear out the cache , such as before it * This is a last - ditch effort to clear out the cache , such as before it
* gets replaced by another cache that is configured differently . This * gets replaced by another cache that is configured differently . This
* method tries to force every cached entry through { @link # clear ( Ref ) } to * method tries to force every cached entry through { @link # clear ( Page Ref) } to
* ensure that resources are correctly accounted for and cleaned up by the * ensure that resources are correctly accounted for and cleaned up by the
* subclass . A concurrent reader loading entries while this method is * subclass . A concurrent reader loading entries while this method is
* running may cause resource accounting failures . * running may cause resource accounting failures .
@ -692,7 +740,7 @@ public class WindowCache {
final Entry e1 = table . get ( s ) ; final Entry e1 = table . get ( s ) ;
boolean hasDead = false ; boolean hasDead = false ;
for ( Entry e = e1 ; e ! = null ; e = e . next ) { for ( Entry e = e1 ; e ! = null ; e = e . next ) {
if ( e . ref . pack = = pack ) { if ( e . ref . getPack ( ) = = pack ) {
e . kill ( ) ; e . kill ( ) ;
hasDead = true ; hasDead = true ;
} else if ( e . dead ) } else if ( e . dead )
@ -705,20 +753,7 @@ public class WindowCache {
} }
private void gc ( ) { private void gc ( ) {
Ref r ; queue . gc ( ) ;
while ( ( r = ( Ref ) queue . poll ( ) ) ! = null ) {
clear ( r ) ;
final int s = slot ( r . pack , r . position ) ;
final Entry e1 = table . get ( s ) ;
for ( Entry n = e1 ; n ! = null ; n = n . next ) {
if ( n . ref = = r ) {
n . dead = true ;
table . compareAndSet ( s , e1 , clean ( e1 ) ) ;
break ;
}
}
}
} }
private int slot ( PackFile pack , long position ) { private int slot ( PackFile pack , long position ) {
@ -731,7 +766,7 @@ public class WindowCache {
private static Entry clean ( Entry top ) { private static Entry clean ( Entry top ) {
while ( top ! = null & & top . dead ) { while ( top ! = null & & top . dead ) {
top . ref . enqueue ( ) ; top . ref . kill ( ) ;
top = top . next ; top = top . next ;
} }
if ( top = = null ) if ( top = = null )
@ -745,7 +780,7 @@ public class WindowCache {
final Entry next ; final Entry next ;
/** The referenced object. */ /** The referenced object. */
final Ref ref ; final Page Ref< ByteWindow > ref ;
/ * * / * *
* Marked true when ref . get ( ) returns null and the ref is dead . * Marked true when ref . get ( ) returns null and the ref is dead .
@ -756,34 +791,275 @@ public class WindowCache {
* / * /
volatile boolean dead ; volatile boolean dead ;
Entry ( Entry n , Ref r ) { Entry ( Entry n , Page Ref< ByteWindow > r ) {
next = n ; next = n ;
ref = r ; ref = r ;
} }
final void kill ( ) { final void kill ( ) {
dead = true ; dead = true ;
ref . enqueue ( ) ; ref . kill ( ) ;
}
} }
private static interface PageRef < T > {
/ * *
* Returns this reference object ' s referent . If this reference object
* has been cleared , either by the program or by the garbage collector ,
* then this method returns < code > null < / code > .
*
* @return The object to which this reference refers , or
* < code > null < / code > if this reference object has been cleared
* /
T get ( ) ;
/ * *
* Kill this ref
*
* @return < code > true < / code > if this reference object was successfully
* killed ; < code > false < / code > if it was already killed
* /
boolean kill ( ) ;
/ * *
* Get the packfile the referenced cache page is allocated for
*
* @return the packfile the referenced cache page is allocated for
* /
PackFile getPack ( ) ;
/ * *
* Get the position of the referenced cache page in the packfile
*
* @return the position of the referenced cache page in the packfile
* /
long getPosition ( ) ;
/ * *
* Get size of cache page
*
* @return size of cache page
* /
int getSize ( ) ;
/ * *
* Get pseudo time of last access to this cache page
*
* @return pseudo time of last access to this cache page
* /
long getLastAccess ( ) ;
/ * *
* Set pseudo time of last access to this cache page
*
* @param time
* pseudo time of last access to this cache page
* /
void setLastAccess ( long time ) ;
/ * *
* Whether this is a strong reference .
* @return { @code true } if this is a strong reference
* /
boolean isStrongRef ( ) ;
} }
/** A soft reference wrapped around a cached object. */ /** A soft reference wrapped around a cached object. */
private static class Ref extends SoftReference < ByteWindow > { private static class SoftRef extends SoftReference < ByteWindow >
final PackFile pack ; implements PageRef < ByteWindow > {
private final PackFile pack ;
final long position ; private final long position ;
final int size ; private final int size ;
long lastAccess ; private long lastAccess ;
protected Ref ( final PackFile pack , final long position , protected Soft Ref( final PackFile pack , final long position ,
final ByteWindow v , final ReferenceQueue < ByteWindow > queue ) { final ByteWindow v , final SoftCleanupQueue queue ) {
super ( v , queue ) ; super ( v , queue ) ;
this . pack = pack ; this . pack = pack ;
this . position = position ; this . position = position ;
this . size = v . size ( ) ; this . size = v . size ( ) ;
} }
@Override
public PackFile getPack ( ) {
return pack ;
}
@Override
public long getPosition ( ) {
return position ;
}
@Override
public int getSize ( ) {
return size ;
}
@Override
public long getLastAccess ( ) {
return lastAccess ;
}
@Override
public void setLastAccess ( long time ) {
this . lastAccess = time ;
}
@Override
public boolean kill ( ) {
return enqueue ( ) ;
}
@Override
public boolean isStrongRef ( ) {
return false ;
}
}
/** A strong reference wrapped around a cached object. */
private static class StrongRef implements PageRef < ByteWindow > {
private ByteWindow referent ;
private final PackFile pack ;
private final long position ;
private final int size ;
private long lastAccess ;
private CleanupQueue queue ;
protected StrongRef ( final PackFile pack , final long position ,
final ByteWindow v , final CleanupQueue queue ) {
this . pack = pack ;
this . position = position ;
this . referent = v ;
this . size = v . size ( ) ;
this . queue = queue ;
}
@Override
public PackFile getPack ( ) {
return pack ;
}
@Override
public long getPosition ( ) {
return position ;
}
@Override
public int getSize ( ) {
return size ;
}
@Override
public long getLastAccess ( ) {
return lastAccess ;
}
@Override
public void setLastAccess ( long time ) {
this . lastAccess = time ;
}
@Override
public ByteWindow get ( ) {
return referent ;
}
@Override
public boolean kill ( ) {
if ( referent = = null ) {
return false ;
}
referent = null ;
return queue . enqueue ( this ) ;
}
@Override
public boolean isStrongRef ( ) {
return true ;
}
}
private static interface CleanupQueue {
boolean enqueue ( PageRef < ByteWindow > r ) ;
void gc ( ) ;
}
private static class SoftCleanupQueue extends ReferenceQueue < ByteWindow >
implements CleanupQueue {
private final WindowCache wc ;
SoftCleanupQueue ( WindowCache cache ) {
this . wc = cache ;
}
@Override
public boolean enqueue ( PageRef < ByteWindow > r ) {
// no need to explicitly add soft references which are enqueued by
// the JVM
return false ;
}
@Override
public void gc ( ) {
SoftRef r ;
while ( ( r = ( SoftRef ) poll ( ) ) ! = null ) {
wc . clear ( r ) ;
final int s = wc . slot ( r . getPack ( ) , r . getPosition ( ) ) ;
final Entry e1 = wc . table . get ( s ) ;
for ( Entry n = e1 ; n ! = null ; n = n . next ) {
if ( n . ref = = r ) {
n . dead = true ;
wc . table . compareAndSet ( s , e1 , clean ( e1 ) ) ;
break ;
}
}
}
}
}
private static class StrongCleanupQueue implements CleanupQueue {
private final WindowCache wc ;
private final ConcurrentLinkedQueue < PageRef < ByteWindow > > queue = new ConcurrentLinkedQueue < > ( ) ;
StrongCleanupQueue ( WindowCache wc ) {
this . wc = wc ;
}
@Override
public boolean enqueue ( PageRef < ByteWindow > r ) {
if ( queue . contains ( r ) ) {
return false ;
}
return queue . add ( r ) ;
}
@Override
public void gc ( ) {
PageRef < ByteWindow > r ;
while ( ( r = queue . poll ( ) ) ! = null ) {
wc . clear ( r ) ;
final int s = wc . slot ( r . getPack ( ) , r . getPosition ( ) ) ;
final Entry e1 = wc . table . get ( s ) ;
for ( Entry n = e1 ; n ! = null ; n = n . next ) {
if ( n . ref = = r ) {
n . dead = true ;
wc . table . compareAndSet ( s , e1 , clean ( e1 ) ) ;
break ;
}
}
}
}
} }
private static final class Lock { private static final class Lock {
Matthias Sohn
5 years ago