From 227357f929562e8010d96fedbe52df561af82c1d Mon Sep 17 00:00:00 2001 From: Christian Halstrick Date: Wed, 17 Sep 2014 15:57:25 +0200 Subject: [PATCH] Add "aggressive" option to GC JGit should offer the possibility to do a garbage collection in "aggressive" mode. In this mode garbage collection more aggressively optimize the repository at the expense of taking much more time. Technically a aggressive mode garbage collection differs from a non-aggressive one by: - not reusing packed objects found in old packs. Recompress every object - the configuration pack.window is set to 250 (the default is 10) - the configuration pack.depths is set to 250 (the default is 50) The associated classes in org.eclipse.jgit.api and the command line command in org.eclipse.jgit.pgm expose this new option. The configuration parameters gc.aggressiveDepth and gc.aggressiveWindow have been introduced to configure this feature. Bug: 444332 Change-Id: I024101f2810acf6be13ce144c9893d98f5c4ae76 --- .../jgit/pgm/internal/CLIText.properties | 1 + .../src/org/eclipse/jgit/pgm/Gc.java | 13 +++-- .../storage/file/GcBasicPackingTest.java | 53 ++++++++++++++----- .../jgit/api/GarbageCollectCommand.java | 53 ++++++++++++++++++- .../jgit/internal/storage/file/GC.java | 18 ++++++- .../org/eclipse/jgit/lib/ConfigConstants.java | 6 +++ 6 files changed, 124 insertions(+), 20 deletions(-) diff --git a/org.eclipse.jgit.pgm/resources/org/eclipse/jgit/pgm/internal/CLIText.properties b/org.eclipse.jgit.pgm/resources/org/eclipse/jgit/pgm/internal/CLIText.properties index 574981d38..bdf05e901 100644 --- a/org.eclipse.jgit.pgm/resources/org/eclipse/jgit/pgm/internal/CLIText.properties +++ b/org.eclipse.jgit.pgm/resources/org/eclipse/jgit/pgm/internal/CLIText.properties @@ -184,6 +184,7 @@ unmergedPaths=Unmerged paths: unsupportedOperation=Unsupported operation: {0} untrackedFiles=Untracked files: updating=Updating {0}..{1} +usage_Aggressive=This option will cause gc to more aggressively optimize the repository at the expense of taking much more time usage_Blame=Show what revision and author last modified each line usage_CommandLineClientForamazonsS3Service=Command line client for Amazon's S3 service usage_CommitAll=commit all modified and deleted files diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Gc.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Gc.java index 58813bcb0..aa5c90590 100644 --- a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Gc.java +++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Gc.java @@ -43,16 +43,19 @@ package org.eclipse.jgit.pgm; -import org.eclipse.jgit.internal.storage.file.FileRepository; -import org.eclipse.jgit.internal.storage.file.GC; +import org.eclipse.jgit.api.Git; import org.eclipse.jgit.lib.TextProgressMonitor; +import org.kohsuke.args4j.Option; @Command(common = true, usage = "usage_Gc") class Gc extends TextBuiltin { + @Option(name = "--aggressive", usage = "usage_Aggressive") + private boolean aggressive; + @Override protected void run() throws Exception { - GC gc = new GC((FileRepository) db); - gc.setProgressMonitor(new TextProgressMonitor()); - gc.gc(); + Git git = Git.wrap(db); + git.gc().setAggressive(aggressive) + .setProgressMonitor(new TextProgressMonitor()).call(); } } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcBasicPackingTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcBasicPackingTest.java index 0f27099c0..0742504d2 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcBasicPackingTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcBasicPackingTest.java @@ -51,21 +51,32 @@ import java.util.Iterator; import org.eclipse.jgit.junit.TestRepository.BranchBuilder; import org.eclipse.jgit.revwalk.RevCommit; -import org.junit.Test; +import org.eclipse.jgit.storage.pack.PackConfig; +import org.junit.experimental.theories.DataPoints; +import org.junit.experimental.theories.Theories; +import org.junit.experimental.theories.Theory; +import org.junit.runner.RunWith; +@RunWith(Theories.class) public class GcBasicPackingTest extends GcTestCase { - @Test - public void repackEmptyRepo_noPackCreated() throws IOException { + @DataPoints + public static boolean[] aggressiveValues = { true, false }; + + @Theory + public void repackEmptyRepo_noPackCreated(boolean aggressive) + throws IOException { + configureGc(gc, aggressive); gc.repack(); assertEquals(0, repo.getObjectDatabase().getPacks().size()); } - @Test - public void testPackRepoWithNoRefs() throws Exception { + @Theory + public void testPackRepoWithNoRefs(boolean aggressive) throws Exception { tr.commit().add("A", "A").add("B", "B").create(); stats = gc.getStatistics(); assertEquals(4, stats.numberOfLooseObjects); assertEquals(0, stats.numberOfPackedObjects); + configureGc(gc, aggressive); gc.gc(); stats = gc.getStatistics(); assertEquals(4, stats.numberOfLooseObjects); @@ -73,8 +84,8 @@ public class GcBasicPackingTest extends GcTestCase { assertEquals(0, stats.numberOfPackFiles); } - @Test - public void testPack2Commits() throws Exception { + @Theory + public void testPack2Commits(boolean aggressive) throws Exception { BranchBuilder bb = tr.branch("refs/heads/master"); bb.commit().add("A", "A").add("B", "B").create(); bb.commit().add("A", "A2").add("B", "B2").create(); @@ -82,6 +93,7 @@ public class GcBasicPackingTest extends GcTestCase { stats = gc.getStatistics(); assertEquals(8, stats.numberOfLooseObjects); assertEquals(0, stats.numberOfPackedObjects); + configureGc(gc, aggressive); gc.gc(); stats = gc.getStatistics(); assertEquals(0, stats.numberOfLooseObjects); @@ -89,13 +101,15 @@ public class GcBasicPackingTest extends GcTestCase { assertEquals(1, stats.numberOfPackFiles); } - @Test - public void testPackAllObjectsInOnePack() throws Exception { + @Theory + public void testPackAllObjectsInOnePack(boolean aggressive) + throws Exception { tr.branch("refs/heads/master").commit().add("A", "A").add("B", "B") .create(); stats = gc.getStatistics(); assertEquals(4, stats.numberOfLooseObjects); assertEquals(0, stats.numberOfPackedObjects); + configureGc(gc, aggressive); gc.gc(); stats = gc.getStatistics(); assertEquals(0, stats.numberOfLooseObjects); @@ -110,8 +124,8 @@ public class GcBasicPackingTest extends GcTestCase { assertEquals(1, stats.numberOfPackFiles); } - @Test - public void testPackCommitsAndLooseOne() throws Exception { + @Theory + public void testPackCommitsAndLooseOne(boolean aggressive) throws Exception { BranchBuilder bb = tr.branch("refs/heads/master"); RevCommit first = bb.commit().add("A", "A").add("B", "B").create(); bb.commit().add("A", "A2").add("B", "B2").create(); @@ -120,6 +134,7 @@ public class GcBasicPackingTest extends GcTestCase { stats = gc.getStatistics(); assertEquals(8, stats.numberOfLooseObjects); assertEquals(0, stats.numberOfPackedObjects); + configureGc(gc, aggressive); gc.gc(); stats = gc.getStatistics(); assertEquals(0, stats.numberOfLooseObjects); @@ -127,8 +142,8 @@ public class GcBasicPackingTest extends GcTestCase { assertEquals(2, stats.numberOfPackFiles); } - @Test - public void testNotPackTwice() throws Exception { + @Theory + public void testNotPackTwice(boolean aggressive) throws Exception { BranchBuilder bb = tr.branch("refs/heads/master"); RevCommit first = bb.commit().message("M").add("M", "M").create(); bb.commit().message("B").add("B", "Q").create(); @@ -146,6 +161,7 @@ public class GcBasicPackingTest extends GcTestCase { gc.setExpireAgeMillis(0); fsTick(); + configureGc(gc, aggressive); gc.gc(); stats = gc.getStatistics(); assertEquals(0, stats.numberOfLooseObjects); @@ -159,4 +175,15 @@ public class GcBasicPackingTest extends GcTestCase { assertEquals(9, pIt.next().getObjectCount()); } } + + private void configureGc(GC myGc, boolean aggressive) { + PackConfig pconfig = new PackConfig(repo); + if (aggressive) { + pconfig.setDeltaSearchWindowSize(250); + pconfig.setMaxDeltaDepth(250); + pconfig.setReuseObjects(false); + } else + pconfig = new PackConfig(repo); + myGc.setPackConfig(pconfig); + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/GarbageCollectCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/GarbageCollectCommand.java index 77b84d3a3..2f7a3edbb 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/GarbageCollectCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/GarbageCollectCommand.java @@ -54,8 +54,11 @@ import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.file.FileRepository; import org.eclipse.jgit.internal.storage.file.GC; import org.eclipse.jgit.internal.storage.file.GC.RepoStatistics; +import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.ProgressMonitor; import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.lib.StoredConfig; +import org.eclipse.jgit.storage.pack.PackConfig; import org.eclipse.jgit.util.GitDateParser; /** @@ -63,17 +66,34 @@ import org.eclipse.jgit.util.GitDateParser; * supported options and arguments of this command and a {@link #call()} method * to finally execute the command. Each instance of this class should only be * used for one invocation of the command (means: one call to {@link #call()}) - * + * * @since 2.2 * @see Git documentation about gc */ public class GarbageCollectCommand extends GitCommand { + /** + * Default value of maximum delta chain depth during aggressive garbage + * collection: {@value} + * + * @since 3.6 + */ + public static final int DEFAULT_GC_AGGRESSIVE_DEPTH = 250; + + /** + * Default window size during packing during aggressive garbage collection: + * * {@value} + * + * @since 3.6 + */ + public static final int DEFAULT_GC_AGGRESSIVE_WINDOW = 250; private ProgressMonitor monitor; private Date expire; + private PackConfig pconfig; + /** * @param repo */ @@ -82,6 +102,7 @@ public class GarbageCollectCommand extends GitCommand { if (!(repo instanceof FileRepository)) throw new UnsupportedOperationException(MessageFormat.format( JGitText.get().unsupportedGC, repo.getClass().toString())); + pconfig = new PackConfig(repo); } /** @@ -110,11 +131,41 @@ public class GarbageCollectCommand extends GitCommand { return this; } + /** + * Whether to use aggressive mode or not. If set to true JGit behaves more + * similar to native git's "git gc --aggressive". If set to + * true compressed objects found in old packs are not reused + * but every object is compressed again. Configuration variables + * pack.window and pack.depth are set to 250 for this GC. + * + * @since 3.6 + * @param aggressive + * whether to turn on or off aggressive mode + * @return this instance + */ + public GarbageCollectCommand setAggressive(boolean aggressive) { + if (aggressive) { + StoredConfig repoConfig = repo.getConfig(); + pconfig.setDeltaSearchWindowSize(repoConfig.getInt( + ConfigConstants.CONFIG_GC_SECTION, + ConfigConstants.CONFIG_KEY_AGGRESSIVE_WINDOW, + DEFAULT_GC_AGGRESSIVE_WINDOW)); + pconfig.setMaxDeltaDepth(repoConfig.getInt( + ConfigConstants.CONFIG_GC_SECTION, + ConfigConstants.CONFIG_KEY_AGGRESSIVE_DEPTH, + DEFAULT_GC_AGGRESSIVE_DEPTH)); + pconfig.setReuseObjects(false); + } else + pconfig = new PackConfig(repo); + return this; + } + @Override public Properties call() throws GitAPIException { checkCallable(); GC gc = new GC((FileRepository) repo); + gc.setPackConfig(pconfig); gc.setProgressMonitor(monitor); if (this.expire != null) gc.setExpire(expire); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java index 3cc4e7b97..48335e48c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java @@ -93,6 +93,7 @@ import org.eclipse.jgit.lib.ReflogEntry; import org.eclipse.jgit.revwalk.ObjectWalk; import org.eclipse.jgit.revwalk.RevObject; import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.storage.pack.PackConfig; import org.eclipse.jgit.treewalk.TreeWalk; import org.eclipse.jgit.treewalk.filter.TreeFilter; import org.eclipse.jgit.util.FileUtils; @@ -117,6 +118,8 @@ public class GC { private Date expire; + private PackConfig pconfig = null; + /** * the refs which existed during the last call to {@link #repack()}. This is * needed during {@link #prune(Set)} where we can optimize by looking at the @@ -686,7 +689,7 @@ public class GC { } }); - PackWriter pw = new PackWriter(repo); + PackWriter pw = new PackWriter((pconfig == null) ? new PackConfig(repo) : pconfig, repo.newObjectReader()); try { // prepare the PackWriter pw.setDeltaBaseAsOffset(true); @@ -947,6 +950,19 @@ public class GC { expire = null; } + /** + * Set the PackConfig used when (re-)writing packfiles. This allows to + * influence how packs are written and to implement something similar to + * "git gc --aggressive" + * + * @since 3.6 + * @param pconfig + * the {@link PackConfig} used when writing packs + */ + public void setPackConfig(PackConfig pconfig) { + this.pconfig = pconfig; + } + /** * During gc() or prune() each unreferenced, loose object which has been * created or modified after or at expire will not be pruned. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java index 378d91c58..b905c9593 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java @@ -226,6 +226,12 @@ public class ConfigConstants { /** The "pruneexpire" key */ public static final String CONFIG_KEY_PRUNEEXPIRE = "pruneexpire"; + /** The "aggressiveDepth" key */ + public static final String CONFIG_KEY_AGGRESSIVE_DEPTH = "aggressiveDepth"; + + /** The "aggressiveWindow" key */ + public static final String CONFIG_KEY_AGGRESSIVE_WINDOW = "aggressiveWindow"; + /** The "mergeoptions" key */ public static final String CONFIG_KEY_MERGEOPTIONS = "mergeoptions";