Merge "GC: Pack RefTrees in their own pack"

org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java

@@ -44,18 +44,18 @@
 package org.eclipse.jgit.internal.storage.dfs;
 
 import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC;
+import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC_TXN;
 import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE;
 import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX;
 import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
 import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;
-import static org.eclipse.jgit.lib.RefDatabase.ALL;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 
 import org.eclipse.jgit.internal.JGitText;
@@ -63,6 +63,7 @@ import org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource;
 import org.eclipse.jgit.internal.storage.file.PackIndex;
 import org.eclipse.jgit.internal.storage.pack.PackExt;
 import org.eclipse.jgit.internal.storage.pack.PackWriter;
+import org.eclipse.jgit.internal.storage.reftree.RefTreeNames;
 import org.eclipse.jgit.lib.AnyObjectId;
 import org.eclipse.jgit.lib.Constants;
 import org.eclipse.jgit.lib.NullProgressMonitor;
@@ -94,14 +95,11 @@ public class DfsGarbageCollector {
 
 	private long coalesceGarbageLimit = 50 << 20;
 
-	private Map<String, Ref> refsBefore;
-
 	private List<DfsPackFile> packsBefore;
 
 	private Set<ObjectId> allHeads;
-
 	private Set<ObjectId> nonHeads;
-
+	private Set<ObjectId> txnHeads;
 	private Set<ObjectId> tagTargets;
 
 	/**
@@ -197,19 +195,22 @@ public class DfsGarbageCollector {
 			refdb.refresh();
 			objdb.clearCache();
 
-			refsBefore = refdb.getRefs(ALL);
+			Collection<Ref> refsBefore = RefTreeNames.allRefs(refdb);
 			packsBefore = packsToRebuild();
 			if (packsBefore.isEmpty())
 				return true;
 
 			allHeads = new HashSet<ObjectId>();
 			nonHeads = new HashSet<ObjectId>();
+			txnHeads = new HashSet<ObjectId>();
 			tagTargets = new HashSet<ObjectId>();
-			for (Ref ref : refsBefore.values()) {
+			for (Ref ref : refsBefore) {
 				if (ref.isSymbolic() || ref.getObjectId() == null)
 					continue;
 				if (isHead(ref))
 					allHeads.add(ref.getObjectId());
+				else if (RefTreeNames.isRefTree(refdb, ref.getName()))
+					txnHeads.add(ref.getObjectId());
 				else
 					nonHeads.add(ref.getObjectId());
 				if (ref.getPeeledObjectId() != null)
@@ -221,6 +222,7 @@ public class DfsGarbageCollector {
 			try {
 				packHeads(pm);
 				packRest(pm);
+				packRefTreeGraph(pm);
 				packGarbage(pm);
 				objdb.commitPack(newPackDesc, toPrune());
 				rollback = false;
@@ -276,12 +278,11 @@ public class DfsGarbageCollector {
 
 		try (PackWriter pw = newPackWriter()) {
 			pw.setTagTargets(tagTargets);
-			pw.preparePack(pm, allHeads, Collections.<ObjectId> emptySet());
+			pw.preparePack(pm, allHeads, none());
 			if (0 < pw.getObjectCount())
 				writePack(GC, pw, pm);
 		}
 	}
-
 	private void packRest(ProgressMonitor pm) throws IOException {
 		if (nonHeads.isEmpty())
 			return;
@@ -295,6 +296,23 @@ public class DfsGarbageCollector {
 		}
 	}
 
+	private void packRefTreeGraph(ProgressMonitor pm) throws IOException {
+		if (txnHeads.isEmpty())
+			return;
+
+		try (PackWriter pw = newPackWriter()) {
+			for (ObjectIdSet packedObjs : newPackObj)
+				pw.excludeObjects(packedObjs);
+			pw.preparePack(pm, txnHeads, none());
+			if (0 < pw.getObjectCount())
+				writePack(GC_TXN, pw, pm);
+		}
+	}
+
+	private static Set<ObjectId> none() {
+		return Collections.<ObjectId> emptySet();
+	}
+
 	private void packGarbage(ProgressMonitor pm) throws IOException {
 		// TODO(sop) This is ugly. The garbage pack needs to be deleted.
 		PackConfig cfg = new PackConfig(packConfig);

org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java

@@ -90,6 +90,13 @@ public abstract class DfsObjDatabase extends ObjectDatabase {
 		 */
 		GC(1),
 
+		/**
+		 * RefTreeGraph pack was created by Git garbage collection.
+		 *
+		 * @see DfsGarbageCollector
+		 */
+		GC_TXN(1),
+
 		/**
 		 * The pack was created by compacting multiple packs together.
 		 * <p>

org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java

@@ -45,7 +45,6 @@ package org.eclipse.jgit.internal.storage.file;
 
 import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX;
 import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
-import static org.eclipse.jgit.lib.RefDatabase.ALL;
 
 import java.io.File;
 import java.io.FileOutputStream;
@@ -63,11 +62,9 @@ import java.util.Comparator;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
-import java.util.Map.Entry;
 import java.util.Objects;
 import java.util.Set;
 import java.util.TreeMap;
@@ -80,6 +77,7 @@ import org.eclipse.jgit.errors.NoWorkTreeException;
 import org.eclipse.jgit.internal.JGitText;
 import org.eclipse.jgit.internal.storage.pack.PackExt;
 import org.eclipse.jgit.internal.storage.pack.PackWriter;
+import org.eclipse.jgit.internal.storage.reftree.RefTreeNames;
 import org.eclipse.jgit.lib.ConfigConstants;
 import org.eclipse.jgit.lib.Constants;
 import org.eclipse.jgit.lib.FileMode;
@@ -128,7 +126,7 @@ public class GC {
 	 * difference between the current refs and the refs which existed during
 	 * last {@link #repack()}.
 	 */
-	private Map<String, Ref> lastPackedRefs;
+	private Collection<Ref> lastPackedRefs;
 
 	/**
 	 * Holds the starting time of the last repack() execution. This is needed in
@@ -362,17 +360,20 @@ public class GC {
 		// during last repack(). Only those refs will survive which have been
 		// added or modified since the last repack. Only these can save existing
 		// loose refs from being pruned.
-		Map<String, Ref> newRefs;
+		Collection<Ref> newRefs;
 		if (lastPackedRefs == null || lastPackedRefs.isEmpty())
 			newRefs = getAllRefs();
 		else {
-			newRefs = new HashMap<String, Ref>();
+			Map<String, Ref> last = new HashMap<>();
-			for (Iterator<Map.Entry<String, Ref>> i = getAllRefs().entrySet()
+			for (Ref r : lastPackedRefs) {
-					.iterator(); i.hasNext();) {
+				last.put(r.getName(), r);
-				Entry<String, Ref> newEntry = i.next();
+			}
-				Ref old = lastPackedRefs.get(newEntry.getKey());
+			newRefs = new ArrayList<>();
-				if (!equals(newEntry.getValue(), old))
+			for (Ref r : getAllRefs()) {
-					newRefs.put(newEntry.getKey(), newEntry.getValue());
+				Ref old = last.get(r.getName());
+				if (!equals(r, old)) {
+					newRefs.add(r);
+				}
 			}
 		}
 
@@ -384,10 +385,10 @@ public class GC {
 			// leave this method.
 			ObjectWalk w = new ObjectWalk(repo);
 			try {
-				for (Ref cr : newRefs.values())
+				for (Ref cr : newRefs)
 					w.markStart(w.parseAny(cr.getObjectId()));
 				if (lastPackedRefs != null)
-					for (Ref lpr : lastPackedRefs.values())
+					for (Ref lpr : lastPackedRefs)
 						w.markUninteresting(w.parseAny(lpr.getObjectId()));
 				removeReferenced(deletionCandidates, w);
 			} finally {
@@ -405,11 +406,11 @@ public class GC {
 		// additional reflog entries not handled during last repack()
 		ObjectWalk w = new ObjectWalk(repo);
 		try {
-			for (Ref ar : getAllRefs().values())
+			for (Ref ar : getAllRefs())
 				for (ObjectId id : listRefLogObjects(ar, lastRepackTime))
 					w.markStart(w.parseAny(id));
 			if (lastPackedRefs != null)
-				for (Ref lpr : lastPackedRefs.values())
+				for (Ref lpr : lastPackedRefs)
 					w.markUninteresting(w.parseAny(lpr.getObjectId()));
 			removeReferenced(deletionCandidates, w);
 		} finally {
@@ -530,19 +531,23 @@ public class GC {
 		Collection<PackFile> toBeDeleted = repo.getObjectDatabase().getPacks();
 
 		long time = System.currentTimeMillis();
-		Map<String, Ref> refsBefore = getAllRefs();
+		Collection<Ref> refsBefore = getAllRefs();
 
 		Set<ObjectId> allHeads = new HashSet<ObjectId>();
 		Set<ObjectId> nonHeads = new HashSet<ObjectId>();
+		Set<ObjectId> txnHeads = new HashSet<ObjectId>();
 		Set<ObjectId> tagTargets = new HashSet<ObjectId>();
 		Set<ObjectId> indexObjects = listNonHEADIndexObjects();
+		RefDatabase refdb = repo.getRefDatabase();
 
-		for (Ref ref : refsBefore.values()) {
+		for (Ref ref : refsBefore) {
 			nonHeads.addAll(listRefLogObjects(ref, 0));
 			if (ref.isSymbolic() || ref.getObjectId() == null)
 				continue;
 			if (ref.getName().startsWith(Constants.R_HEADS))
 				allHeads.add(ref.getObjectId());
+			else if (RefTreeNames.isRefTree(refdb, ref.getName()))
+				txnHeads.add(ref.getObjectId());
 			else
 				nonHeads.add(ref.getObjectId());
 			if (ref.getPeeledObjectId() != null)
@@ -572,6 +577,11 @@ public class GC {
 			if (rest != null)
 				ret.add(rest);
 		}
+		if (!txnHeads.isEmpty()) {
+			PackFile txn = writePack(txnHeads, null, null, excluded);
+			if (txn != null)
+				ret.add(txn);
+		}
 		try {
 			deleteOldPacks(toBeDeleted, ret);
 		} catch (ParseException e) {
@@ -624,11 +634,16 @@ public class GC {
 	 * @return a map where names of refs point to ref objects
 	 * @throws IOException
 	 */
-	private Map<String, Ref> getAllRefs() throws IOException {
+	private Collection<Ref> getAllRefs() throws IOException {
-		Map<String, Ref> ret = repo.getRefDatabase().getRefs(ALL);
+		Collection<Ref> refs = RefTreeNames.allRefs(repo.getRefDatabase());
-		for (Ref ref : repo.getRefDatabase().getAdditionalRefs())
+		List<Ref> addl = repo.getRefDatabase().getAdditionalRefs();
-			ret.put(ref.getName(), ref);
+		if (!addl.isEmpty()) {
-		return ret;
+			List<Ref> all = new ArrayList<>(refs.size() + addl.size());
+			all.addAll(refs);
+			all.addAll(addl);
+			return all;
+		}
+		return refs;
 	}
 
 	/**

org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeNames.java

@@ -0,0 +1,124 @@
+/*
+ * Copyright (C) 2016, Google Inc.
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.eclipse.jgit.internal.storage.reftree;
+
+import static org.eclipse.jgit.lib.RefDatabase.ALL;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.eclipse.jgit.lib.Ref;
+import org.eclipse.jgit.lib.RefDatabase;
+
+/** Magic reference name logic for RefTrees. */
+public class RefTreeNames {
+	/**
+	 * Suffix used on a {@link RefTreeDatabase#getTxnNamespace()} for user data.
+	 * <p>
+	 * A {@link RefTreeDatabase}'s namespace may include a subspace (e.g.
+	 * {@code "refs/txn/stage/"}) containing commit objects from the usual user
+	 * portion of the repository (e.g. {@code "refs/heads/"}). These should be
+	 * packed by the garbage collector alongside other user content rather than
+	 * with the RefTree.
+	 */
+	private static final String STAGE = "stage/"; //$NON-NLS-1$
+
+	/**
+	 * Determine if the reference is likely to be a RefTree.
+	 *
+	 * @param refdb
+	 *            database instance.
+	 * @param ref
+	 *            reference name.
+	 * @return {@code true} if the reference is a RefTree.
+	 */
+	public static boolean isRefTree(RefDatabase refdb, String ref) {
+		if (refdb instanceof RefTreeDatabase) {
+			RefTreeDatabase b = (RefTreeDatabase) refdb;
+			if (ref.equals(b.getTxnCommitted())) {
+				return true;
+			}
+
+			String namespace = b.getTxnNamespace();
+			if (namespace != null
+					&& ref.startsWith(namespace)
+					&& !ref.startsWith(namespace + STAGE)) {
+				return true;
+			}
+		}
+		return false;
+	}
+
+	/**
+	 * Snapshot all references from a RefTreeDatabase and its bootstrap.
+	 * <p>
+	 * There may be name conflicts with multiple {@link Ref} objects containing
+	 * the same name in the returned collection.
+	 *
+	 * @param refdb
+	 *            database instance.
+	 * @return all known references.
+	 * @throws IOException
+	 *             references cannot be enumerated.
+	 */
+	public static Collection<Ref> allRefs(RefDatabase refdb)
+			throws IOException {
+		Collection<Ref> refs = refdb.getRefs(ALL).values();
+		if (!(refdb instanceof RefTreeDatabase)) {
+			return refs;
+		}
+
+		RefDatabase bootstrap = ((RefTreeDatabase) refdb).getBootstrap();
+		Collection<Ref> br = bootstrap.getRefs(ALL).values();
+		List<Ref> all = new ArrayList<>(refs.size() + br.size());
+		all.addAll(refs);
+		all.addAll(br);
+		return all;
+	}
+
+	private RefTreeNames() {
+	}
+}