Merge changes I159e9154,I06c722b2

* changes:
  DfsObjectDatabase: Expose PackList and move markDirty there
  Invalidate DfsObjDatabase pack list when refs are updated

org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java

@@ -61,7 +61,17 @@ import org.eclipse.jgit.lib.ObjectReader;
 
 /** Manages objects stored in {@link DfsPackFile} on a storage system. */
 public abstract class DfsObjDatabase extends ObjectDatabase {
-	private static final PackList NO_PACKS = new PackList(new DfsPackFile[0]);
+	private static final PackList NO_PACKS = new PackList(new DfsPackFile[0]) {
+		@Override
+		boolean dirty() {
+			return true;
+		}
+
+		@Override
+		public void markDirty() {
+			// Always dirty.
+		}
+	};
 
 	/** Sources for a pack file. */
 	public static enum PackSource {
@@ -173,7 +183,20 @@ public abstract class DfsObjDatabase extends ObjectDatabase {
 	 *             the pack list cannot be initialized.
 	 */
 	public DfsPackFile[] getPacks() throws IOException {
-		return scanPacks(NO_PACKS).packs;
+		return getPackList().packs;
+	}
+
+	/**
+	 * Scan and list all available pack files in the repository.
+	 *
+	 * @return list of available packs, with some additional metadata. The
+	 *         returned array is shared with the implementation and must not be
+	 *         modified by the caller.
+	 * @throws IOException
+	 *             the pack list cannot be initialized.
+	 */
+	public PackList getPackList() throws IOException {
+		return scanPacks(NO_PACKS);
 	}
 
 	/** @return repository owning this object database. */
@@ -188,7 +211,18 @@ public abstract class DfsObjDatabase extends ObjectDatabase {
 	 *         implementation and must not be modified by the caller.
 	 */
 	public DfsPackFile[] getCurrentPacks() {
-		return packList.get().packs;
+		return getCurrentPackList().packs;
+	}
+
+	/**
+	 * List currently known pack files in the repository, without scanning.
+	 *
+	 * @return list of available packs, with some additional metadata. The
+	 *         returned array is shared with the implementation and must not be
+	 *         modified by the caller.
+	 */
+	public PackList getCurrentPackList() {
+		return packList.get();
 	}
 
 	/**
@@ -363,11 +397,11 @@ public abstract class DfsObjDatabase extends ObjectDatabase {
 			DfsPackFile[] packs = new DfsPackFile[1 + o.packs.length];
 			packs[0] = newPack;
 			System.arraycopy(o.packs, 0, packs, 1, o.packs.length);
-			n = new PackList(packs);
+			n = new PackListImpl(packs);
 		} while (!packList.compareAndSet(o, n));
 	}
 
-	private PackList scanPacks(final PackList original) throws IOException {
+	PackList scanPacks(final PackList original) throws IOException {
 		PackList o, n;
 		synchronized (packList) {
 			do {
@@ -408,10 +442,10 @@ public abstract class DfsObjDatabase extends ObjectDatabase {
 		for (DfsPackFile p : forReuse.values())
 			p.close();
 		if (list.isEmpty())
-			return new PackList(NO_PACKS.packs);
+			return new PackListImpl(NO_PACKS.packs);
 		if (!foundNew)
 			return old;
-		return new PackList(list.toArray(new DfsPackFile[list.size()]));
+		return new PackListImpl(list.toArray(new DfsPackFile[list.size()]));
 	}
 
 	private static Map<DfsPackDescription, DfsPackFile> reuseMap(PackList old) {
@@ -456,12 +490,42 @@ public abstract class DfsObjDatabase extends ObjectDatabase {
 		// p.close();
 	}
 
-	private static final class PackList {
+	/** Snapshot of packs scanned in a single pass. */
+	public static abstract class PackList {
 		/** All known packs, sorted. */
-		final DfsPackFile[] packs;
+		public final DfsPackFile[] packs;
 
-		PackList(final DfsPackFile[] packs) {
+		PackList(DfsPackFile[] packs) {
 			this.packs = packs;
 		}
+
+		abstract boolean dirty();
+
+		/**
+		 * Mark pack list as dirty.
+		 * <p>
+		 * Used when the caller knows that new data might have been written to the
+		 * repository that could invalidate open readers depending on this pack list,
+		 * for example if refs are newly scanned.
+		 */
+		public abstract void markDirty();
+	}
+
+	private static final class PackListImpl extends PackList {
+		private volatile boolean dirty;
+
+		PackListImpl(DfsPackFile[] packs) {
+			super(packs);
+		}
+
+		@Override
+		boolean dirty() {
+			return dirty;
+		}
+
+		@Override
+		public void markDirty() {
+			dirty = true;
+		}
 	}
 }

org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java

@@ -53,6 +53,7 @@ import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Set;
 import java.util.zip.DataFormatException;
@@ -62,6 +63,7 @@ import org.eclipse.jgit.errors.IncorrectObjectTypeException;
 import org.eclipse.jgit.errors.MissingObjectException;
 import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException;
 import org.eclipse.jgit.internal.JGitText;
+import org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackList;
 import org.eclipse.jgit.internal.storage.file.BitmapIndexImpl;
 import org.eclipse.jgit.internal.storage.file.PackBitmapIndex;
 import org.eclipse.jgit.internal.storage.file.PackIndex;
@@ -91,6 +93,8 @@ import org.eclipse.jgit.util.BlockList;
  * reader is not thread safe.
  */
 public final class DfsReader extends ObjectReader implements ObjectReuseAsIs {
+	private static final int MAX_RESOLVE_MATCHES = 256;
+
 	/** Temporary buffer large enough for at least one raw object id. */
 	final byte[] tempId = new byte[OBJECT_ID_LENGTH];
 
@@ -163,14 +167,25 @@ public final class DfsReader extends ObjectReader implements ObjectReuseAsIs {
 			return Collections.singleton(id.toObjectId());
 		boolean noGarbage = avoidUnreachable;
 		HashSet<ObjectId> matches = new HashSet<ObjectId>(4);
-		for (DfsPackFile pack : db.getPacks()) {
+		PackList packList = db.getPackList();
-			if (noGarbage && pack.isGarbage())
+		resolveImpl(packList, id, noGarbage, matches);
+		if (matches.size() < MAX_RESOLVE_MATCHES && packList.dirty()) {
+			resolveImpl(db.scanPacks(packList), id, noGarbage, matches);
+		}
+		return matches;
+	}
+
+	private void resolveImpl(PackList packList, AbbreviatedObjectId id,
+			boolean noGarbage, HashSet<ObjectId> matches) throws IOException {
+		for (DfsPackFile pack : packList.packs) {
+			if (noGarbage && pack.isGarbage()) {
 				continue;
-			pack.resolve(this, matches, id, 256);
+			}
-			if (256 <= matches.size())
+			pack.resolve(this, matches, id, MAX_RESOLVE_MATCHES);
+			if (matches.size() >= MAX_RESOLVE_MATCHES) {
 				break;
+			}
 		}
-		return matches;
 	}
 
 	@Override
@@ -178,7 +193,18 @@ public final class DfsReader extends ObjectReader implements ObjectReuseAsIs {
 		if (last != null && last.hasObject(this, objectId))
 			return true;
 		boolean noGarbage = avoidUnreachable;
-		for (DfsPackFile pack : db.getPacks()) {
+		PackList packList = db.getPackList();
+		if (hasImpl(packList, objectId, noGarbage)) {
+			return true;
+		} else if (packList.dirty()) {
+			return hasImpl(db.scanPacks(packList), objectId, noGarbage);
+		}
+		return false;
+	}
+
+	private boolean hasImpl(PackList packList, AnyObjectId objectId,
+			boolean noGarbage) throws IOException {
+		for (DfsPackFile pack : packList.packs) {
 			if (pack == last || (noGarbage && pack.isGarbage()))
 				continue;
 			if (pack.hasObject(this, objectId)) {
@@ -193,19 +219,22 @@ public final class DfsReader extends ObjectReader implements ObjectReuseAsIs {
 	public ObjectLoader open(AnyObjectId objectId, int typeHint)
 			throws MissingObjectException, IncorrectObjectTypeException,
 			IOException {
+		ObjectLoader ldr;
 		if (last != null) {
-			ObjectLoader ldr = last.get(this, objectId);
+			ldr = last.get(this, objectId);
 			if (ldr != null)
 				return ldr;
 		}
 
+		PackList packList = db.getPackList();
 		boolean noGarbage = avoidUnreachable;
-		for (DfsPackFile pack : db.getPacks()) {
+		ldr = openImpl(packList, objectId, noGarbage);
-			if (pack == last || (noGarbage && pack.isGarbage()))
+		if (ldr != null) {
-				continue;
+			return ldr;
-			ObjectLoader ldr = pack.get(this, objectId);
+		}
+		if (packList.dirty()) {
+			ldr = openImpl(db.scanPacks(packList), objectId, noGarbage);
 			if (ldr != null) {
-				last = pack;
 				return ldr;
 			}
 		}
@@ -216,6 +245,21 @@ public final class DfsReader extends ObjectReader implements ObjectReuseAsIs {
 		throw new MissingObjectException(objectId.copy(), typeHint);
 	}
 
+	private ObjectLoader openImpl(PackList packList, AnyObjectId objectId,
+			boolean noGarbage) throws IOException {
+		for (DfsPackFile pack : packList.packs) {
+			if (pack == last || (noGarbage && pack.isGarbage())) {
+				continue;
+			}
+			ObjectLoader ldr = pack.get(this, objectId);
+			if (ldr != null) {
+				last = pack;
+				return ldr;
+			}
+		}
+		return null;
+	}
+
 	@Override
 	public Set<ObjectId> getShallowCommits() {
 		return Collections.emptySet();
@@ -253,39 +297,58 @@ public final class DfsReader extends ObjectReader implements ObjectReuseAsIs {
 
 	private <T extends ObjectId> Iterable<FoundObject<T>> findAll(
 			Iterable<T> objectIds) throws IOException {
-		ArrayList<FoundObject<T>> r = new ArrayList<FoundObject<T>>();
+		Collection<T> pending = new LinkedList<>();
-		DfsPackFile[] packList = db.getPacks();
+		for (T id : objectIds) {
-		if (packList.length == 0) {
+			pending.add(id);
-			for (T t : objectIds)
-				r.add(new FoundObject<T>(t));
-			return r;
 		}
 
+		PackList packList = db.getPackList();
+		List<FoundObject<T>> r = new ArrayList<>();
+		findAllImpl(packList, pending, r);
+		if (!pending.isEmpty() && packList.dirty()) {
+			findAllImpl(db.scanPacks(packList), pending, r);
+		}
+		for (T t : pending) {
+			r.add(new FoundObject<T>(t));
+		}
+		Collections.sort(r, FOUND_OBJECT_SORT);
+		return r;
+	}
+
+	private <T extends ObjectId> void findAllImpl(PackList packList,
+			Collection<T> pending, List<FoundObject<T>> r) {
+		DfsPackFile[] packs = packList.packs;
+		if (packs.length == 0) {
+			return;
+		}
 		int lastIdx = 0;
-		DfsPackFile lastPack = packList[lastIdx];
+		DfsPackFile lastPack = packs[lastIdx];
 		boolean noGarbage = avoidUnreachable;
 
-		OBJECT_SCAN: for (T t : objectIds) {
+		OBJECT_SCAN: for (Iterator<T> it = pending.iterator(); it.hasNext();) {
+			T t = it.next();
 			try {
 				long p = lastPack.findOffset(this, t);
 				if (0 < p) {
 					r.add(new FoundObject<T>(t, lastIdx, lastPack, p));
+					it.remove();
 					continue;
 				}
 			} catch (IOException e) {
 				// Fall though and try to examine other packs.
 			}
 
-			for (int i = 0; i < packList.length; i++) {
+			for (int i = 0; i < packs.length; i++) {
 				if (i == lastIdx)
 					continue;
-				DfsPackFile pack = packList[i];
+				DfsPackFile pack = packs[i];
 				if (noGarbage && pack.isGarbage())
 					continue;
 				try {
 					long p = pack.findOffset(this, t);
 					if (0 < p) {
 						r.add(new FoundObject<T>(t, i, pack, p));
+						it.remove();
 						lastIdx = i;
 						lastPack = pack;
 						continue OBJECT_SCAN;
@@ -294,13 +357,9 @@ public final class DfsReader extends ObjectReader implements ObjectReuseAsIs {
 					// Examine other packs.
 				}
 			}
-
-			r.add(new FoundObject<T>(t));
 		}
 
-		Collections.sort(r, FOUND_OBJECT_SORT);
 		last = lastPack;
-		return r;
 	}
 
 	@Override
@@ -418,26 +477,45 @@ public final class DfsReader extends ObjectReader implements ObjectReuseAsIs {
 			IOException {
 		if (last != null) {
 			long sz = last.getObjectSize(this, objectId);
-			if (0 <= sz)
+			if (0 <= sz) {
 				return sz;
+			}
 		}
 
-		for (DfsPackFile pack : db.getPacks()) {
+		PackList packList = db.getPackList();
-			if (pack == last)
+		long sz = getObjectSizeImpl(packList, objectId);
-				continue;
+		if (0 <= sz) {
-			long sz = pack.getObjectSize(this, objectId);
+			return sz;
+		}
+		if (packList.dirty()) {
+			sz = getObjectSizeImpl(packList, objectId);
 			if (0 <= sz) {
-				last = pack;
 				return sz;
 			}
 		}
 
-		if (typeHint == OBJ_ANY)
+		if (typeHint == OBJ_ANY) {
 			throw new MissingObjectException(objectId.copy(),
 					JGitText.get().unknownObjectType2);
+		}
 		throw new MissingObjectException(objectId.copy(), typeHint);
 	}
 
+	private long getObjectSizeImpl(PackList packList, AnyObjectId objectId)
+			throws IOException {
+		for (DfsPackFile pack : packList.packs) {
+			if (pack == last) {
+				continue;
+			}
+			long sz = pack.getObjectSize(this, objectId);
+			if (0 <= sz) {
+				last = pack;
+				return sz;
+			}
+		}
+		return -1;
+	}
+
 	public DfsObjectToPack newObjectToPack(AnyObjectId objectId, int type) {
 		return new DfsObjectToPack(objectId, type);
 	}
@@ -451,6 +529,8 @@ public final class DfsReader extends ObjectReader implements ObjectReuseAsIs {
 	public void selectObjectRepresentation(PackWriter packer,
 			ProgressMonitor monitor, Iterable<ObjectToPack> objects)
 			throws IOException, MissingObjectException {
+		// Don't check dirty bit on PackList; assume ObjectToPacks all came from the
+		// current list.
 		for (DfsPackFile pack : db.getPacks()) {
 			List<DfsObjectToPack> tmp = findAllFromPack(pack, objects);
 			if (tmp.isEmpty())

org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/InMemoryRepository.java

@@ -310,6 +310,7 @@ public class InMemoryRepository extends DfsRepository {
 			}
 			ids.sort();
 			sym.sort();
+			objdb.getCurrentPackList().markDirty();
 			return new RefCache(ids.toRefList(), sym.toRefList());
 		}