dfs: expose DfsReftable from DfsObjDatabase

Reftable storage in DFS is related to pack storage.  Reftables are
stored in the same namespace, but with PackExt.REFTABLE.  Include
the set of DfsReftable instances in the PackList and export some
helpers to access the tables.

Change-Id: I6a4f5f953ed6b0ff80a7780f4c6cbcc5eda0da3e

org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java

@@ -48,6 +48,7 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -61,7 +62,9 @@ import org.eclipse.jgit.lib.ObjectReader;
 
 /** Manages objects stored in {@link DfsPackFile} on a storage system. */
 public abstract class DfsObjDatabase extends ObjectDatabase {
-	private static final PackList NO_PACKS = new PackList(new DfsPackFile[0]) {
+	private static final PackList NO_PACKS = new PackList(
+			new DfsPackFile[0],
+			new DfsReftable[0]) {
 		@Override
 		boolean dirty() {
 			return true;
@@ -191,6 +194,18 @@ public abstract class DfsObjDatabase extends ObjectDatabase {
 		return getPackList().packs;
 	}
 
+	/**
+	 * Scan and list all available reftable files in the repository.
+	 *
+	 * @return list of available reftables. The returned array is shared with
+	 *         the implementation and must not be modified by the caller.
+	 * @throws IOException
+	 *             the pack list cannot be initialized.
+	 */
+	public DfsReftable[] getReftables() throws IOException {
+		return getPackList().reftables;
+	}
+
 	/**
 	 * Scan and list all available pack files in the repository.
 	 *
@@ -219,6 +234,16 @@ public abstract class DfsObjDatabase extends ObjectDatabase {
 		return getCurrentPackList().packs;
 	}
 
+	/**
+	 * List currently known reftable files in the repository, without scanning.
+	 *
+	 * @return list of available reftables. The returned array is shared with
+	 *         the implementation and must not be modified by the caller.
+	 */
+	public DfsReftable[] getCurrentReftables() {
+		return getCurrentPackList().reftables;
+	}
+
 	/**
 	 * List currently known pack files in the repository, without scanning.
 	 *
@@ -428,7 +453,7 @@ public abstract class DfsObjDatabase extends ObjectDatabase {
 			DfsPackFile[] packs = new DfsPackFile[1 + o.packs.length];
 			packs[0] = newPack;
 			System.arraycopy(o.packs, 0, packs, 1, o.packs.length);
-			n = new PackListImpl(packs);
+			n = new PackListImpl(packs, o.reftables);
 		} while (!packList.compareAndSet(o, n));
 	}
 
@@ -454,59 +479,93 @@ public abstract class DfsObjDatabase extends ObjectDatabase {
 
 	private PackList scanPacksImpl(PackList old) throws IOException {
 		DfsBlockCache cache = DfsBlockCache.getInstance();
-		Map<DfsPackDescription, DfsPackFile> forReuse = reuseMap(old);
+		Map<DfsPackDescription, DfsPackFile> packs = packMap(old);
+		Map<DfsPackDescription, DfsReftable> reftables = reftableMap(old);
+
 		List<DfsPackDescription> scanned = listPacks();
 		Collections.sort(scanned);
 
-		List<DfsPackFile> list = new ArrayList<>(scanned.size());
+		List<DfsPackFile> newPacks = new ArrayList<>(scanned.size());
+		List<DfsReftable> newReftables = new ArrayList<>(scanned.size());
 		boolean foundNew = false;
 		for (DfsPackDescription dsc : scanned) {
-			DfsPackFile oldPack = forReuse.remove(dsc);
+			DfsPackFile oldPack = packs.remove(dsc);
 			if (oldPack != null) {
-				list.add(oldPack);
+				newPacks.add(oldPack);
 			} else if (dsc.hasFileExt(PackExt.PACK)) {
-				list.add(new DfsPackFile(cache, dsc));
+				newPacks.add(new DfsPackFile(cache, dsc));
+				foundNew = true;
+			}
+
+			DfsReftable oldReftable = reftables.remove(dsc);
+			if (oldReftable != null) {
+				newReftables.add(oldReftable);
+			} else if (dsc.hasFileExt(PackExt.REFTABLE)) {
+				newReftables.add(new DfsReftable(cache, dsc));
 				foundNew = true;
 			}
 		}
 
-		for (DfsPackFile p : forReuse.values())
+		if (newPacks.isEmpty())
-			p.close();
+			return new PackListImpl(NO_PACKS.packs, NO_PACKS.reftables);
-		if (list.isEmpty())
-			return new PackListImpl(NO_PACKS.packs);
 		if (!foundNew) {
 			old.clearDirty();
 			return old;
 		}
-		return new PackListImpl(list.toArray(new DfsPackFile[list.size()]));
+		Collections.sort(newReftables, reftableComparator());
+		return new PackListImpl(
+				newPacks.toArray(new DfsPackFile[0]),
+				newReftables.toArray(new DfsReftable[0]));
 	}
 
-	private static Map<DfsPackDescription, DfsPackFile> reuseMap(PackList old) {
+	private static Map<DfsPackDescription, DfsPackFile> packMap(PackList old) {
 		Map<DfsPackDescription, DfsPackFile> forReuse = new HashMap<>();
 		for (DfsPackFile p : old.packs) {
-			if (p.invalid()) {
+			if (!p.invalid()) {
-				// The pack instance is corrupted, and cannot be safely used
+				forReuse.put(p.desc, p);
-				// again. Do not include it in our reuse map.
-				//
-				p.close();
-				continue;
 			}
+		}
+		return forReuse;
+	}
 
-			DfsPackFile prior = forReuse.put(p.getPackDescription(), p);
+	private static Map<DfsPackDescription, DfsReftable> reftableMap(PackList old) {
-			if (prior != null) {
+		Map<DfsPackDescription, DfsReftable> forReuse = new HashMap<>();
-				// This should never occur. It should be impossible for us
+		for (DfsReftable p : old.reftables) {
-				// to have two pack files with the same name, as all of them
+			if (!p.invalid()) {
-				// came out of the same directory. If it does, we promised to
+				forReuse.put(p.desc, p);
-				// close any PackFiles we did not reuse, so close the second,
-				// readers are likely to be actively using the first.
-				//
-				forReuse.put(prior.getPackDescription(), prior);
-				p.close();
 			}
 		}
 		return forReuse;
 	}
 
+	/** @return comparator to sort {@link DfsReftable} by priority. */
+	protected Comparator<DfsReftable> reftableComparator() {
+		return (fa, fb) -> {
+			DfsPackDescription a = fa.getPackDescription();
+			DfsPackDescription b = fb.getPackDescription();
+
+			// GC, COMPACT reftables first by higher category.
+			int c = category(b) - category(a);
+			if (c != 0) {
+				return c;
+			}
+
+			// Lower maxUpdateIndex first.
+			c = Long.signum(a.getMaxUpdateIndex() - b.getMaxUpdateIndex());
+			if (c != 0) {
+				return c;
+			}
+
+			// Older reftable first.
+			return Long.signum(a.getLastModified() - b.getLastModified());
+		};
+	}
+
+	static int category(DfsPackDescription d) {
+		PackSource s = d.getPackSource();
+		return s != null ? s.category : 0;
+	}
+
 	/** Clears the cached list of packs, forcing them to be scanned again. */
 	protected void clearCache() {
 		packList.set(NO_PACKS);
@@ -514,12 +573,7 @@ public abstract class DfsObjDatabase extends ObjectDatabase {
 
 	@Override
 	public void close() {
-		// PackList packs = packList.get();
 		packList.set(NO_PACKS);
-
-		// TODO Close packs if they aren't cached.
-		// for (DfsPackFile p : packs.packs)
-		// p.close();
 	}
 
 	/** Snapshot of packs scanned in a single pass. */
@@ -527,10 +581,14 @@ public abstract class DfsObjDatabase extends ObjectDatabase {
 		/** All known packs, sorted. */
 		public final DfsPackFile[] packs;
 
+		/** All known reftables, sorted. */
+		public final DfsReftable[] reftables;
+
 		private long lastModified = -1;
 
-		PackList(DfsPackFile[] packs) {
+		PackList(DfsPackFile[] packs, DfsReftable[] reftables) {
 			this.packs = packs;
+			this.reftables = reftables;
 		}
 
 		/** @return last modified time of all packs, in milliseconds. */
@@ -561,8 +619,8 @@ public abstract class DfsObjDatabase extends ObjectDatabase {
 	private static final class PackListImpl extends PackList {
 		private volatile boolean dirty;
 
-		PackListImpl(DfsPackFile[] packs) {
+		PackListImpl(DfsPackFile[] packs, DfsReftable[] reftables) {
-			super(packs);
+			super(packs, reftables);
 		}
 
 		@Override

org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackDescription.java

@@ -44,11 +44,13 @@
 package org.eclipse.jgit.internal.storage.dfs;
 
 import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;
+import static org.eclipse.jgit.internal.storage.pack.PackExt.REFTABLE;
 
 import java.util.Arrays;
 
 import org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource;
 import org.eclipse.jgit.internal.storage.pack.PackExt;
+import org.eclipse.jgit.internal.storage.reftable.ReftableWriter;
 import org.eclipse.jgit.storage.pack.PackStatistics;
 
 /**
@@ -68,7 +70,11 @@ public class DfsPackDescription implements Comparable<DfsPackDescription> {
 	private int[] blockSizeMap;
 	private long objectCount;
 	private long deltaCount;
-	private PackStatistics stats;
+	private long minUpdateIndex;
+	private long maxUpdateIndex;
+
+	private PackStatistics packStats;
+	private ReftableWriter.Stats refStats;
 	private int extensions;
 	private int indexVersion;
 	private long estimatedPackSize;
@@ -170,6 +176,36 @@ public class DfsPackDescription implements Comparable<DfsPackDescription> {
 		return this;
 	}
 
+	/** @return minUpdateIndex for the reftable, if present. */
+	public long getMinUpdateIndex() {
+		return minUpdateIndex;
+	}
+
+	/**
+	 * @param min
+	 *            minUpdateIndex for the reftable, or 0.
+	 * @return {@code this}
+	 */
+	public DfsPackDescription setMinUpdateIndex(long min) {
+		minUpdateIndex = min;
+		return this;
+	}
+
+	/** @return maxUpdateIndex for the reftable, if present. */
+	public long getMaxUpdateIndex() {
+		return maxUpdateIndex;
+	}
+
+	/**
+	 * @param max
+	 *            maxUpdateIndex for the reftable, or 0.
+	 * @return {@code this}
+	 */
+	public DfsPackDescription setMaxUpdateIndex(long max) {
+		maxUpdateIndex = max;
+		return this;
+	}
+
 	/**
 	 * @param ext
 	 *            the file extension.
@@ -281,24 +317,38 @@ public class DfsPackDescription implements Comparable<DfsPackDescription> {
 	 *         is being committed to the repository.
 	 */
 	public PackStatistics getPackStats() {
-		return stats;
+		return packStats;
 	}
 
 	DfsPackDescription setPackStats(PackStatistics stats) {
-		this.stats = stats;
+		this.packStats = stats;
 		setFileSize(PACK, stats.getTotalBytes());
 		setObjectCount(stats.getTotalObjects());
 		setDeltaCount(stats.getTotalDeltas());
 		return this;
 	}
 
+	/** @return stats from the sibling reftable, if created. */
+	public ReftableWriter.Stats getReftableStats() {
+		return refStats;
+	}
+
+	void setReftableStats(ReftableWriter.Stats stats) {
+		this.refStats = stats;
+		setMinUpdateIndex(stats.minUpdateIndex());
+		setMaxUpdateIndex(stats.maxUpdateIndex());
+		setFileSize(REFTABLE, stats.totalBytes());
+		setBlockSize(REFTABLE, stats.refBlockSize());
+	}
+
 	/**
 	 * Discard the pack statistics, if it was populated.
 	 *
 	 * @return {@code this}
 	 */
 	public DfsPackDescription clearPackStats() {
-		stats = null;
+		packStats = null;
+		refStats = null;
 		return this;
 	}
 

org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java

@@ -385,12 +385,6 @@ public final class DfsPackFile extends BlockBasedFile {
 		idx(ctx).resolve(matches, id, matchLimit);
 	}
 
-	/** Release all memory used by this DfsPackFile instance. */
-	public void close() {
-		index = null;
-		reverseIndex = null;
-	}
-
 	/**
 	 * Obtain the total number of objects available in this pack. This method
 	 * relies on pack index, giving number of effectively available objects.