dfs: Fix caching of index, bitmap index, reverse index

When 07f98a8b71 ("Derive DfsStreamKey from DfsPackDescription")
stopped caching DfsPackFile in the DfsBlockCache, the DfsPackFile began
to always load the idx, bitmap, or compute reverse index, as the cache
handles were no longer populated by prior requests.

Rework caching to lookup the objects from the DfsBlockCache if the
local DfsPackFile handle is invalid.  This allows the DfsPackFile to
be more of a flyweight instance across requests.

Change-Id: Ic7b42ce2d90692cccea36deb30c2c76ccc81638b

org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCache.java

@@ -390,6 +390,10 @@ public final class DfsBlockCache {
 		put(v.stream, v.start, v.size(), v);
 	}
 
+	<T> Ref<T> putRef(DfsStreamKey key, long size, T v) {
+		return put(key, 0, (int) Math.min(size, Integer.MAX_VALUE), v);
+	}
+
 	<T> Ref<T> put(DfsStreamKey key, long pos, int size, T v) {
 		int slot = slot(key, pos);
 		HashEntry e1 = table.get(slot);
@@ -444,6 +448,15 @@ public final class DfsBlockCache {
 		return r != null ? r.get() : null;
 	}
 
+	<T> Ref<T> getRef(DfsStreamKey key) {
+		Ref<T> r = scanRef(table.get(slot(key, 0)), key, 0);
+		if (r != null)
+			statHit.incrementAndGet();
+		else
+			statMiss.incrementAndGet();
+		return r;
+	}
+
 	@SuppressWarnings("unchecked")
 	private <T> Ref<T> scanRef(HashEntry n, DfsStreamKey key, long position) {
 		for (; n != null; n = n.next) {

org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java

@@ -144,8 +144,8 @@ public final class DfsPackFile extends BlockBasedFile {
 	void setPackIndex(PackIndex idx) {
 		long objCnt = idx.getObjectCount();
 		int recSize = Constants.OBJECT_ID_LENGTH + 8;
-		int sz = (int) Math.min(objCnt * recSize, Integer.MAX_VALUE);
+		long sz = objCnt * recSize;
-		index = cache.put(desc.getStreamKey(INDEX), 0, sz, idx);
+		index = cache.putRef(desc.getStreamKey(INDEX), sz, idx);
 	}
 
 	/**
@@ -184,6 +184,16 @@ public final class DfsPackFile extends BlockBasedFile {
 					return idx;
 			}
 
+			DfsStreamKey idxKey = desc.getStreamKey(INDEX);
+			idxref = cache.getRef(idxKey);
+			if (idxref != null) {
+				PackIndex idx = idxref.get();
+				if (idx != null) {
+					index = idxref;
+					return idx;
+				}
+			}
+
 			PackIndex idx;
 			try {
 				ctx.stats.readIdx++;
@@ -205,18 +215,14 @@ public final class DfsPackFile extends BlockBasedFile {
 				}
 			} catch (EOFException e) {
 				invalid = true;
-				IOException e2 = new IOException(MessageFormat.format(
+				throw new IOException(MessageFormat.format(
 						DfsText.get().shortReadOfIndex,
-						desc.getFileName(INDEX)));
+						desc.getFileName(INDEX)), e);
-				e2.initCause(e);
-				throw e2;
 			} catch (IOException e) {
 				invalid = true;
-				IOException e2 = new IOException(MessageFormat.format(
+				throw new IOException(MessageFormat.format(
 						DfsText.get().cannotReadIndex,
-						desc.getFileName(INDEX)));
+						desc.getFileName(INDEX)), e);
-				e2.initCause(e);
-				throw e2;
 			}
 
 			setPackIndex(idx);
@@ -229,8 +235,9 @@ public final class DfsPackFile extends BlockBasedFile {
 	}
 
 	PackBitmapIndex getBitmapIndex(DfsReader ctx) throws IOException {
-		if (invalid || isGarbage())
+		if (invalid || isGarbage() || !desc.hasFileExt(BITMAP_INDEX))
 			return null;
+
 		DfsBlockCache.Ref<PackBitmapIndex> idxref = bitmapIndex;
 		if (idxref != null) {
 			PackBitmapIndex idx = idxref.get();
@@ -238,9 +245,6 @@ public final class DfsPackFile extends BlockBasedFile {
 				return idx;
 		}
 
-		if (!desc.hasFileExt(BITMAP_INDEX))
-			return null;
-
 		synchronized (initLock) {
 			idxref = bitmapIndex;
 			if (idxref != null) {
@@ -249,6 +253,16 @@ public final class DfsPackFile extends BlockBasedFile {
 					return idx;
 			}
 
+			DfsStreamKey bitmapKey = desc.getStreamKey(BITMAP_INDEX);
+			idxref = cache.getRef(bitmapKey);
+			if (idxref != null) {
+				PackBitmapIndex idx = idxref.get();
+				if (idx != null) {
+					bitmapIndex = idxref;
+					return idx;
+				}
+			}
+
 			long size;
 			PackBitmapIndex idx;
 			try {
@@ -273,22 +287,16 @@ public final class DfsPackFile extends BlockBasedFile {
 					ctx.stats.readIdxMicros += elapsedMicros(start);
 				}
 			} catch (EOFException e) {
-				IOException e2 = new IOException(MessageFormat.format(
+				throw new IOException(MessageFormat.format(
 						DfsText.get().shortReadOfIndex,
-						desc.getFileName(BITMAP_INDEX)));
+						desc.getFileName(BITMAP_INDEX)), e);
-				e2.initCause(e);
-				throw e2;
 			} catch (IOException e) {
-				IOException e2 = new IOException(MessageFormat.format(
+				throw new IOException(MessageFormat.format(
 						DfsText.get().cannotReadIndex,
-						desc.getFileName(BITMAP_INDEX)));
+						desc.getFileName(BITMAP_INDEX)), e);
-				e2.initCause(e);
-				throw e2;
 			}
 
-			bitmapIndex = cache.put(
+			bitmapIndex = cache.putRef(bitmapKey, size, idx);
-					desc.getStreamKey(BITMAP_INDEX),
-					0, (int) Math.min(size, Integer.MAX_VALUE), idx);
 			return idx;
 		}
 	}
@@ -309,13 +317,21 @@ public final class DfsPackFile extends BlockBasedFile {
 					return revidx;
 			}
 
+			DfsStreamKey revKey =
+					new DfsStreamKey.ForReverseIndex(desc.getStreamKey(INDEX));
+			revref = cache.getRef(revKey);
+			if (revref != null) {
+				PackReverseIndex idx = revref.get();
+				if (idx != null) {
+					reverseIndex = revref;
+					return idx;
+				}
+			}
+
 			PackIndex idx = idx(ctx);
 			PackReverseIndex revidx = new PackReverseIndex(idx);
-			int sz = (int) Math.min(
+			long cnt = idx.getObjectCount();
-					idx.getObjectCount() * 8, Integer.MAX_VALUE);
+			reverseIndex = cache.putRef(revKey, cnt * 8, revidx);
-			reverseIndex = cache.put(
-					new DfsStreamKey.ForReverseIndex(desc.getStreamKey(INDEX)),
-					0, sz, revidx);
 			return revidx;
 		}
 	}