gc: Add options to preserve and prune old pack files

The new --preserve-oldpacks option moves old pack files into the
preserved subdirectory instead of deleting them after repacking.

The new --prune-preserved option prunes old pack files from the
preserved subdirectory after repacking, but before potentially
moving the latest old packfiles to this subdirectory.

These options are designed to prevent stale file handle exceptions
during git operations which can happen on users of NFS repos when
repacking is done on them. The strategy is to preserve old pack files
around until the next repack with the hopes that they will become
unreferenced by then and not cause any exceptions to running processes
when they are finally deleted (pruned).

Change-Id: If3f729f0d9ce920ee2c3e6acdde46f2068be61d2
Signed-off-by: James Melvin <jmelvin@codeaurora.org>

org.eclipse.jgit.pgm/resources/org/eclipse/jgit/pgm/internal/CLIText.properties

@@ -246,6 +246,8 @@ usage_LsTree=List the contents of a tree object
 usage_MakeCacheTree=Show the current cache tree structure
 usage_MergeBase=Find as good common ancestors as possible for a merge
 usage_MergesTwoDevelopmentHistories=Merges two development histories
+usage_PreserveOldPacks=Preserve old pack files by moving them into the preserved subdirectory instead of deleting them after repacking
+usage_PrunePreserved=Remove the preserved subdirectory containing previously preserved old pack files before repacking, and before preserving more old pack files
 usage_ReadDirCache= Read the DirCache 100 times
 usage_RebuildCommitGraph=Recreate a repository from another one's commit graph
 usage_RebuildRefTree=Copy references into a RefTree

org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Gc.java

@@ -52,10 +52,18 @@ class Gc extends TextBuiltin {
 	@Option(name = "--aggressive", usage = "usage_Aggressive")
 	private boolean aggressive;
 
+	@Option(name = "--preserve-oldpacks", usage = "usage_PreserveOldPacks")
+	private boolean preserveOldPacks;
+
+	@Option(name = "--prune-preserved", usage = "usage_PrunePreserved")
+	private boolean prunePreserved;
+
 	@Override
 	protected void run() throws Exception {
 		Git git = Git.wrap(db);
 		git.gc().setAggressive(aggressive)
+				.setPreserveOldPacks(preserveOldPacks)
+				.setPrunePreserved(prunePreserved)
 				.setProgressMonitor(new TextProgressMonitor(errw)).call();
 	}
 }

org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcBasicPackingTest.java

@@ -44,6 +44,8 @@
 package org.eclipse.jgit.internal.storage.file;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
 
 import java.io.File;
 import java.io.IOException;
@@ -233,7 +235,45 @@ public class GcBasicPackingTest extends GcTestCase {
 
 	}
 
-	private void configureGc(GC myGc, boolean aggressive) {
+	@Test
+	public void testPreserveAndPruneOldPacks() throws Exception {
+		testPreserveOldPacks();
+		configureGc(gc, false).setPrunePreserved(true);
+		gc.gc();
+
+		assertFalse(repo.getObjectDatabase().getPreservedDirectory().exists());
+	}
+
+	private void testPreserveOldPacks() throws Exception {
+		BranchBuilder bb = tr.branch("refs/heads/master");
+		bb.commit().message("P").add("P", "P").create();
+
+		// pack loose object into packfile
+		gc.setExpireAgeMillis(0);
+		gc.gc();
+		File oldPackfile = tr.getRepository().getObjectDatabase().getPacks()
+				.iterator().next().getPackFile();
+		assertTrue(oldPackfile.exists());
+
+		fsTick();
+		bb.commit().message("B").add("B", "Q").create();
+
+		// repack again but now without a grace period for packfiles. We should
+		// end up with a new packfile and the old one should be placed in the
+		// preserved directory
+		gc.setPackExpireAgeMillis(0);
+		configureGc(gc, false).setPreserveOldPacks(true);
+		gc.gc();
+
+		File oldPackDir = repo.getObjectDatabase().getPreservedDirectory();
+		String oldPackFileName = oldPackfile.getName();
+		String oldPackName = oldPackFileName.substring(0,
+				oldPackFileName.lastIndexOf('.')) + ".old-pack";  //$NON-NLS-1$
+		File preservePackFile = new File(oldPackDir, oldPackName);
+		assertTrue(preservePackFile.exists());
+	}
+
+	private PackConfig configureGc(GC myGc, boolean aggressive) {
 		PackConfig pconfig = new PackConfig(repo);
 		if (aggressive) {
 			pconfig.setDeltaSearchWindowSize(250);
@@ -242,5 +282,6 @@ public class GcBasicPackingTest extends GcTestCase {
 		} else
 			pconfig = new PackConfig(repo);
 		myGc.setPackConfig(pconfig);
+		return pconfig;
 	}
 }

org.eclipse.jgit/src/org/eclipse/jgit/api/GarbageCollectCommand.java

@@ -159,6 +159,38 @@ public class GarbageCollectCommand extends GitCommand<Properties> {
 		return this;
 	}
 
+	/**
+	 * Whether to preserve old pack files instead of deleting them.
+	 *
+	 * @since 4.7
+	 * @param preserveOldPacks
+	 *            whether to preserve old pack files
+	 * @return this instance
+	 */
+	public GarbageCollectCommand setPreserveOldPacks(boolean preserveOldPacks) {
+		if (pconfig == null)
+			pconfig = new PackConfig(repo);
+
+		pconfig.setPreserveOldPacks(preserveOldPacks);
+		return this;
+	}
+
+	/**
+	 * Whether to prune preserved pack files in the preserved directory.
+	 *
+	 * @since 4.7
+	 * @param prunePreserved
+	 *            whether to prune preserved pack files
+	 * @return this instance
+	 */
+	public GarbageCollectCommand setPrunePreserved(boolean prunePreserved) {
+		if (pconfig == null)
+			pconfig = new PackConfig(repo);
+
+		pconfig.setPrunePreserved(prunePreserved);
+		return this;
+	}
+
 	@Override
 	public Properties call() throws GitAPIException {
 		checkCallable();

org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java

@@ -211,9 +211,10 @@ public class GC {
 	/**
 	 * Delete old pack files. What is 'old' is defined by specifying a set of
 	 * old pack files and a set of new pack files. Each pack file contained in
-	 * old pack files but not contained in new pack files will be deleted. If an
-	 * expirationDate is set then pack files which are younger than the
-	 * expirationDate will not be deleted.
+	 * old pack files but not contained in new pack files will be deleted. If
+	 * preserveOldPacks is set, keep a copy of the pack file in the preserve
+	 * directory. If an expirationDate is set then pack files which are younger
+	 * than the expirationDate will not be deleted nor preserved.
 	 *
 	 * @param oldPacks
 	 * @param newPacks
@@ -222,6 +223,7 @@ public class GC {
 	 */
 	private void deleteOldPacks(Collection<PackFile> oldPacks,
 			Collection<PackFile> newPacks) throws ParseException, IOException {
+		prunePreserved();
 		long packExpireDate = getPackExpireDate();
 		oldPackLoop: for (PackFile oldPack : oldPacks) {
 			String oldName = oldPack.getPackName();
@@ -238,11 +240,49 @@ public class GC {
 				prunePack(oldName);
 			}
 		}
-		// close the complete object database. Thats my only chance to force
+		// close the complete object database. That's my only chance to force
 		// rescanning and to detect that certain pack files are now deleted.
 		repo.getObjectDatabase().close();
 	}
 
+	/**
+	 * Deletes old pack file, unless 'preserve-oldpacks' is set, in which case it
+	 * moves the pack file to the preserved directory
+	 *
+	 * @param packFile
+	 * @param packName
+	 * @param ext
+	 * @param deleteOptions
+	 * @throws IOException
+	 */
+	private void removeOldPack(File packFile, String packName, PackExt ext,
+			int deleteOptions) throws IOException {
+		if (pconfig != null && pconfig.isPreserveOldPacks()) {
+			File oldPackDir = repo.getObjectDatabase().getPreservedDirectory();
+			FileUtils.mkdir(oldPackDir, true);
+
+			String oldPackName = "pack-" + packName + ".old-" + ext.getExtension();  //$NON-NLS-1$ //$NON-NLS-2$
+			File oldPackFile = new File(oldPackDir, oldPackName);
+			FileUtils.rename(packFile, oldPackFile);
+		} else {
+			FileUtils.delete(packFile, deleteOptions);
+		}
+	}
+
+	/**
+	 * Delete the preserved directory including all pack files within
+	 */
+	private void prunePreserved() {
+		if (pconfig != null && pconfig.isPrunePreserved()) {
+			try {
+				FileUtils.delete(repo.getObjectDatabase().getPreservedDirectory(),
+						FileUtils.RECURSIVE | FileUtils.RETRY | FileUtils.SKIP_MISSING);
+			} catch (IOException e) {
+				// Deletion of the preserved pack files failed. Silently return.
+			}
+		}
+	}
+
 	/**
 	 * Delete files associated with a single pack file. First try to delete the
 	 * ".pack" file because on some platforms the ".pack" file may be locked and
@@ -262,7 +302,7 @@ public class GC {
 			for (PackExt ext : extensions)
 				if (PackExt.PACK.equals(ext)) {
 					File f = nameFor(packName, "." + ext.getExtension()); //$NON-NLS-1$
-					FileUtils.delete(f, deleteOptions);
+					removeOldPack(f, packName, ext, deleteOptions);
 					break;
 				}
 			// The .pack file has been deleted. Delete as many as the other
@@ -271,7 +311,7 @@ public class GC {
 			for (PackExt ext : extensions) {
 				if (!PackExt.PACK.equals(ext)) {
 					File f = nameFor(packName, "." + ext.getExtension()); //$NON-NLS-1$
-					FileUtils.delete(f, deleteOptions);
+					removeOldPack(f, packName, ext, deleteOptions);
 				}
 			}
 		} catch (IOException e) {

org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java

@@ -125,6 +125,8 @@ public class ObjectDirectory extends FileObjectDatabase {
 
 	private final File packDirectory;
 
+	private final File preservedDirectory;
+
 	private final File alternatesFile;
 
 	private final AtomicReference<PackList> packList;
@@ -165,6 +167,7 @@ public class ObjectDirectory extends FileObjectDatabase {
 		objects = dir;
 		infoDirectory = new File(objects, "info"); //$NON-NLS-1$
 		packDirectory = new File(objects, "pack"); //$NON-NLS-1$
+		preservedDirectory = new File(packDirectory, "preserved"); //$NON-NLS-1$
 		alternatesFile = new File(infoDirectory, "alternates"); //$NON-NLS-1$
 		packList = new AtomicReference<PackList>(NO_PACKS);
 		unpackedObjectCache = new UnpackedObjectCache();
@@ -189,6 +192,13 @@ public class ObjectDirectory extends FileObjectDatabase {
 		return objects;
 	}
 
+	/**
+	 * @return the location of the <code>preserved</code> directory.
+	 */
+	public final File getPreservedDirectory() {
+		return preservedDirectory;
+	}
+
 	@Override
 	public boolean exists() {
 		return fs.exists(objects);

org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java

@@ -74,6 +74,20 @@ public class PackConfig {
 	 */
 	public static final boolean DEFAULT_REUSE_OBJECTS = true;
 
+	/**
+	 * Default value of keep old packs option: {@value}
+	 *
+	 * @see #setPreserveOldPacks(boolean)
+	 */
+	public static final boolean DEFAULT_PRESERVE_OLD_PACKS = false;
+
+	/**
+	 * Default value of prune old packs option: {@value}
+	 *
+	 * @see #setPrunePreserved(boolean)
+	 */
+	public static final boolean DEFAULT_PRUNE_PRESERVED = false;
+
 	/**
 	 * Default value of delta compress option: {@value}
 	 *
@@ -204,6 +218,10 @@ public class PackConfig {
 
 	private boolean reuseObjects = DEFAULT_REUSE_OBJECTS;
 
+	private boolean preserveOldPacks = DEFAULT_PRESERVE_OLD_PACKS;
+
+	private boolean prunePreserved = DEFAULT_PRUNE_PRESERVED;
+
 	private boolean deltaBaseAsOffset = DEFAULT_DELTA_BASE_AS_OFFSET;
 
 	private boolean deltaCompress = DEFAULT_DELTA_COMPRESS;
@@ -281,6 +299,8 @@ public class PackConfig {
 		this.compressionLevel = cfg.compressionLevel;
 		this.reuseDeltas = cfg.reuseDeltas;
 		this.reuseObjects = cfg.reuseObjects;
+		this.preserveOldPacks = cfg.preserveOldPacks;
+		this.prunePreserved = cfg.prunePreserved;
 		this.deltaBaseAsOffset = cfg.deltaBaseAsOffset;
 		this.deltaCompress = cfg.deltaCompress;
 		this.maxDeltaDepth = cfg.maxDeltaDepth;
@@ -363,6 +383,61 @@ public class PackConfig {
 		this.reuseObjects = reuseObjects;
 	}
 
+	/**
+	 * Checks whether to preserve old packs in a preserved directory
+	 *
+	 * Default setting: {@value #DEFAULT_PRESERVE_OLD_PACKS}
+	 *
+	 * @return true if repacking will preserve old pack files.
+	 * @since 4.7
+	 */
+	public boolean isPreserveOldPacks() {
+		return preserveOldPacks;
+	}
+
+	/**
+	 * Set preserve old packs configuration option for repacking.
+	 *
+	 * If enabled, old pack files are moved into a preserved subdirectory instead
+	 * of being deleted
+	 *
+	 * Default setting: {@value #DEFAULT_PRESERVE_OLD_PACKS}
+	 *
+	 * @param preserveOldPacks
+	 *            boolean indicating whether or not preserve old pack files
+	 * @since 4.7
+	 */
+	public void setPreserveOldPacks(boolean preserveOldPacks) {
+		this.preserveOldPacks = preserveOldPacks;
+	}
+
+	/**
+	 * Checks whether to remove preserved pack files in a preserved directory
+	 *
+	 * Default setting: {@value #DEFAULT_PRUNE_PRESERVED}
+	 *
+	 * @return true if repacking will remove preserved pack files.
+	 * @since 4.7
+	 */
+	public boolean isPrunePreserved() {
+		return prunePreserved;
+	}
+
+	/**
+	 * Set prune preserved configuration option for repacking.
+	 *
+	 * If enabled, preserved pack files are removed from a preserved subdirectory
+	 *
+	 * Default setting: {@value #DEFAULT_PRESERVE_OLD_PACKS}
+	 *
+	 * @param prunePreserved
+	 *            boolean indicating whether or not preserve old pack files
+	 * @since 4.7
+	 */
+	public void setPrunePreserved(boolean prunePreserved) {
+		this.prunePreserved = prunePreserved;
+	}
+
 	/**
 	 * True if writer can use offsets to point to a delta base.
 	 *