Browse Source

Use read ahead during copyPackThroughCache

If a block is missing from the block cache, open the pack stream,
retain the ReadableChannel, and turn on read-ahead.  This should help
to load a medium sized pack into a cold cache more quickly from a
slower IO stream, as the pack is scanned sequentially and missing
blocks are more likely to be available through the read-ahead.

Change-Id: I3300d936b9299be6d9eb642992df7c04bb439cde
stable-4.9
Shawn Pearce 7 years ago
parent
commit
5fdbcc1081
  1. 11
      org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCache.java
  2. 54
      org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java

11
org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCache.java

@ -53,6 +53,7 @@ import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReferenceArray; import java.util.concurrent.atomic.AtomicReferenceArray;
import java.util.concurrent.locks.ReentrantLock; import java.util.concurrent.locks.ReentrantLock;
import org.eclipse.jgit.annotations.Nullable;
import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.JGitText;
/** /**
@ -308,12 +309,14 @@ public final class DfsBlockCache {
* offset within <code>pack</code> of the object. * offset within <code>pack</code> of the object.
* @param ctx * @param ctx
* current thread's reader. * current thread's reader.
* @param packChannel
* optional channel to read {@code pack}.
* @return the object reference. * @return the object reference.
* @throws IOException * @throws IOException
* the reference was not in the cache and could not be loaded. * the reference was not in the cache and could not be loaded.
*/ */
DfsBlock getOrLoad(DfsPackFile pack, long position, DfsReader ctx) DfsBlock getOrLoad(DfsPackFile pack, long position, DfsReader ctx,
throws IOException { @Nullable ReadableChannel packChannel) throws IOException {
final long requestedPosition = position; final long requestedPosition = position;
position = pack.alignToBlock(position); position = pack.alignToBlock(position);
@ -345,7 +348,7 @@ public final class DfsBlockCache {
statMiss.incrementAndGet(); statMiss.incrementAndGet();
boolean credit = true; boolean credit = true;
try { try {
v = pack.readOneBlock(position, ctx); v = pack.readOneBlock(position, ctx, packChannel);
credit = false; credit = false;
} finally { } finally {
if (credit) if (credit)
@ -376,7 +379,7 @@ public final class DfsBlockCache {
// that was loaded is the wrong block for the requested position. // that was loaded is the wrong block for the requested position.
if (v.contains(pack.key, requestedPosition)) if (v.contains(pack.key, requestedPosition))
return v; return v;
return getOrLoad(pack, requestedPosition, ctx); return getOrLoad(pack, requestedPosition, ctx, packChannel);
} }
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")

54
org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java

@ -62,6 +62,7 @@ import java.util.zip.CRC32;
import java.util.zip.DataFormatException; import java.util.zip.DataFormatException;
import java.util.zip.Inflater; import java.util.zip.Inflater;
import org.eclipse.jgit.annotations.Nullable;
import org.eclipse.jgit.errors.CorruptObjectException; import org.eclipse.jgit.errors.CorruptObjectException;
import org.eclipse.jgit.errors.LargeObjectException; import org.eclipse.jgit.errors.LargeObjectException;
import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.errors.MissingObjectException;
@ -489,15 +490,36 @@ public final class DfsPackFile {
private void copyPackThroughCache(PackOutputStream out, DfsReader ctx) private void copyPackThroughCache(PackOutputStream out, DfsReader ctx)
throws IOException { throws IOException {
long position = 12; ReadableChannel rc = null;
long remaining = length - (12 + 20); try {
while (0 < remaining) { long position = 12;
DfsBlock b = cache.getOrLoad(this, position, ctx); long remaining = length - (12 + 20);
int ptr = (int) (position - b.start); while (0 < remaining) {
int n = (int) Math.min(b.size() - ptr, remaining); DfsBlock b;
b.write(out, position, n); if (rc != null) {
position += n; b = cache.getOrLoad(this, position, ctx, rc);
remaining -= n; } else {
b = cache.get(key, alignToBlock(position));
if (b == null) {
rc = ctx.db.openFile(packDesc, PACK);
int sz = ctx.getOptions().getStreamPackBufferSize();
if (sz > 0) {
rc.setReadAheadBytes(sz);
}
b = cache.getOrLoad(this, position, ctx, rc);
}
}
int ptr = (int) (position - b.start);
int n = (int) Math.min(b.size() - ptr, remaining);
b.write(out, position, n);
position += n;
remaining -= n;
}
} finally {
if (rc != null) {
rc.close();
}
} }
} }
@ -780,17 +802,19 @@ public final class DfsPackFile {
} }
DfsBlock getOrLoadBlock(long pos, DfsReader ctx) throws IOException { DfsBlock getOrLoadBlock(long pos, DfsReader ctx) throws IOException {
return cache.getOrLoad(this, pos, ctx); return cache.getOrLoad(this, pos, ctx, null);
} }
DfsBlock readOneBlock(long pos, DfsReader ctx) DfsBlock readOneBlock(long pos, DfsReader ctx,
throws IOException { @Nullable ReadableChannel packChannel) throws IOException {
if (invalid) if (invalid)
throw new PackInvalidException(getPackName()); throw new PackInvalidException(getPackName());
ctx.stats.readBlock++; ctx.stats.readBlock++;
long start = System.nanoTime(); long start = System.nanoTime();
ReadableChannel rc = ctx.db.openFile(packDesc, PACK); ReadableChannel rc = packChannel != null
? packChannel
: ctx.db.openFile(packDesc, PACK);
try { try {
int size = blockSize(rc); int size = blockSize(rc);
pos = (pos / size) * size; pos = (pos / size) * size;
@ -840,7 +864,9 @@ public final class DfsPackFile {
return new DfsBlock(key, pos, buf); return new DfsBlock(key, pos, buf);
} finally { } finally {
rc.close(); if (rc != packChannel) {
rc.close();
}
ctx.stats.readBlockMicros += elapsedMicros(start); ctx.stats.readBlockMicros += elapsedMicros(start);
} }
} }

Loading…
Cancel
Save