diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/pack/PackWriterBitmapPreparerTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/pack/PackWriterBitmapPreparerTest.java new file mode 100644 index 000000000..b0f92ffa0 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/pack/PackWriterBitmapPreparerTest.java @@ -0,0 +1,136 @@ +package org.eclipse.jgit.internal.storage.pack; + +import static org.eclipse.jgit.storage.pack.PackConfig.DEFAULT_BITMAP_DISTANT_COMMIT_SPAN; +import static org.eclipse.jgit.storage.pack.PackConfig.DEFAULT_BITMAP_RECENT_COMMIT_COUNT; +import static org.eclipse.jgit.storage.pack.PackConfig.DEFAULT_BITMAP_RECENT_COMMIT_SPAN; +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Set; + +import org.eclipse.jgit.errors.IncorrectObjectTypeException; +import org.eclipse.jgit.errors.MissingObjectException; +import org.eclipse.jgit.internal.storage.file.PackBitmapIndexBuilder; +import org.eclipse.jgit.lib.AbbreviatedObjectId; +import org.eclipse.jgit.lib.AnyObjectId; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectLoader; +import org.eclipse.jgit.lib.ObjectReader; +import org.eclipse.jgit.storage.pack.PackConfig; +import org.junit.Test; + +/** Tests for the {@link PackWriterBitmapPreparer}. */ +public class PackWriterBitmapPreparerTest { + private static class StubObjectReader extends ObjectReader { + @Override + public ObjectReader newReader() { + return null; + } + + @Override + public Collection resolve(AbbreviatedObjectId id) + throws IOException { + return null; + } + + @Override + public ObjectLoader open(AnyObjectId objectId, int typeHint) + throws MissingObjectException, IncorrectObjectTypeException, + IOException { + return null; + } + + @Override + public Set getShallowCommits() throws IOException { + return null; + } + + @Override + public void close() { + // stub + } + } + + @Test + public void testNextSelectionDistanceForActiveBranch() throws Exception { + PackWriterBitmapPreparer preparer = newPeparer( + DEFAULT_BITMAP_RECENT_COMMIT_COUNT, // 20000 + DEFAULT_BITMAP_RECENT_COMMIT_SPAN, // 100 + DEFAULT_BITMAP_DISTANT_COMMIT_SPAN); // 5000 + int[][] distancesAndSpans = { { 0, 100 }, { 100, 100 }, { 10000, 100 }, + { 20000, 100 }, { 20100, 100 }, { 20102, 102 }, { 20200, 200 }, + { 22200, 2200 }, { 24999, 4999 }, { 25000, 5000 }, + { 50000, 5000 }, { 1000000, 5000 }, }; + + for (int[] pair : distancesAndSpans) { + assertEquals(pair[1], preparer.nextSpan(pair[0])); + } + } + + @Test + public void testNextSelectionDistanceWithFewerRecentCommits() + throws Exception { + PackWriterBitmapPreparer preparer = newPeparer(1000, + DEFAULT_BITMAP_RECENT_COMMIT_SPAN, // 100 + DEFAULT_BITMAP_DISTANT_COMMIT_SPAN); // 5000 + int[][] distancesAndSpans = { { 0, 100 }, { 100, 100 }, { 1000, 100 }, + { 1100, 100 }, { 1111, 111 }, { 2000, 1000 }, { 5999, 4999 }, + { 6000, 5000 }, { 10000, 5000 }, { 50000, 5000 }, + { 1000000, 5000 } }; + + for (int[] pair : distancesAndSpans) { + assertEquals(pair[1], preparer.nextSpan(pair[0])); + } + } + + @Test + public void testNextSelectionDistanceWithSmallerRecentSpan() + throws Exception { + PackWriterBitmapPreparer preparer = newPeparer( + DEFAULT_BITMAP_RECENT_COMMIT_COUNT, // 20000 + 10, // recent span + DEFAULT_BITMAP_DISTANT_COMMIT_SPAN); // 5000 + int[][] distancesAndSpans = { { 0, 10 }, { 100, 10 }, { 10000, 10 }, + { 20000, 10 }, { 20010, 10 }, { 20012, 12 }, { 20050, 50 }, + { 20200, 200 }, { 22200, 2200 }, { 24999, 4999 }, + { 25000, 5000 }, { 50000, 5000 }, { 1000000, 5000 } }; + + for (int[] pair : distancesAndSpans) { + assertEquals(pair[1], preparer.nextSpan(pair[0])); + } + } + + @Test + public void testNextSelectionDistanceWithSmallerDistantSpan() + throws Exception { + PackWriterBitmapPreparer preparer = newPeparer( + DEFAULT_BITMAP_RECENT_COMMIT_COUNT, // 20000 + DEFAULT_BITMAP_RECENT_COMMIT_SPAN, // 100 + 1000); + int[][] distancesAndSpans = { { 0, 100 }, { 100, 100 }, { 10000, 100 }, + { 20000, 100 }, { 20100, 100 }, { 20102, 102 }, { 20200, 200 }, + { 20999, 999 }, { 21000, 1000 }, { 22000, 1000 }, + { 25000, 1000 }, { 50000, 1000 }, { 1000000, 1000 } }; + + for (int[] pair : distancesAndSpans) { + assertEquals(pair[1], preparer.nextSpan(pair[0])); + } + } + + private PackWriterBitmapPreparer newPeparer(int recentCount, int recentSpan, + int distantSpan) throws IOException { + List objects = Collections.emptyList(); + Set wants = Collections.emptySet(); + PackConfig config = new PackConfig(); + config.setBitmapRecentCommitCount(recentCount); + config.setBitmapRecentCommitSpan(recentSpan); + config.setBitmapDistantCommitSpan(distantSpan); + PackBitmapIndexBuilder indexBuilder = new PackBitmapIndexBuilder( + objects); + return new PackWriterBitmapPreparer(new StubObjectReader(), + indexBuilder, null, wants, config); + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java index 8a8c173c2..1bff6ba4c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java @@ -2015,7 +2015,7 @@ public class PackWriter implements AutoCloseable { byName = null; PackWriterBitmapPreparer bitmapPreparer = new PackWriterBitmapPreparer( - reader, writeBitmaps, pm, stats.interestingObjects); + reader, writeBitmaps, pm, stats.interestingObjects, config); Collection selectedCommits = bitmapPreparer.selectCommits(numCommits); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriterBitmapPreparer.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriterBitmapPreparer.java index 9bb8789f6..1834fef81 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriterBitmapPreparer.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriterBitmapPreparer.java @@ -72,6 +72,7 @@ import org.eclipse.jgit.revwalk.ObjectWalk; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.RevObject; import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.storage.pack.PackConfig; import org.eclipse.jgit.util.BlockList; import org.eclipse.jgit.util.SystemReader; @@ -81,6 +82,8 @@ import org.eclipse.jgit.util.SystemReader; */ class PackWriterBitmapPreparer { + private static final int DAY_IN_SECONDS = 24 * 60 * 60; + private static final Comparator ORDER_BY_DESCENDING_CARDINALITY = new Comparator() { public int compare(BitmapBuilderEntry a, BitmapBuilderEntry b) { return Integer.signum(b.getBuilder().cardinality() @@ -105,7 +108,8 @@ class PackWriterBitmapPreparer { PackWriterBitmapPreparer(ObjectReader reader, PackBitmapIndexBuilder writeBitmaps, ProgressMonitor pm, - Set want) throws IOException { + Set want, PackConfig config) + throws IOException { this.reader = reader; this.writeBitmaps = writeBitmaps; this.pm = pm; @@ -114,13 +118,14 @@ class PackWriterBitmapPreparer { this.bitmapRemapper = PackBitmapIndexRemapper.newPackBitmapIndex( reader.getBitmapIndex(), writeBitmaps); this.bitmapIndex = new BitmapIndexImpl(bitmapRemapper); - this.contiguousCommitCount = 100; - this.recentCommitCount = 20000; - this.recentCommitSpan = 100; - this.distantCommitSpan = 5000; - this.excessiveBranchCount = 100; + this.contiguousCommitCount = config.getBitmapContiguousCommitCount(); + this.recentCommitCount = config.getBitmapRecentCommitCount(); + this.recentCommitSpan = config.getBitmapRecentCommitSpan(); + this.distantCommitSpan = config.getBitmapDistantCommitSpan(); + this.excessiveBranchCount = config.getBitmapExcessiveBranchCount(); long now = SystemReader.getInstance().getCurrentTime(); - long ageInSeconds = 90 * 24 * 60 * 60; + long ageInSeconds = config.getBitmapInactiveBranchAgeInDays() + * DAY_IN_SECONDS; this.inactiveBranchTimestamp = (now / 1000) - ageInSeconds; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java index a8835b76c..d594e9767 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java @@ -138,6 +138,65 @@ public class PackConfig { */ public static final boolean DEFAULT_BUILD_BITMAPS = true; + /** + * Default count of most recent commits to select for bitmaps. Only applies + * when bitmaps are enabled: {@value} + * + * @see #setBitmapContiguousCommitCount(int) + * @since 4.2 + */ + public static final int DEFAULT_BITMAP_CONTIGUOUS_COMMIT_COUNT = 100; + + /** + * Count at which the span between selected commits changes from + * "bitmapRecentCommitSpan" to "bitmapDistantCommitSpan". Only applies when + * bitmaps are enabled: {@value} + * + * @see #setBitmapRecentCommitCount(int) + * @since 4.2 + */ + public static final int DEFAULT_BITMAP_RECENT_COMMIT_COUNT = 20000; + + /** + * Default spacing between commits in recent history when selecting commits + * for bitmaps. Only applies when bitmaps are enabled: {@value} + * + * @see #setBitmapRecentCommitSpan(int) + * @since 4.2 + */ + public static final int DEFAULT_BITMAP_RECENT_COMMIT_SPAN = 100; + + /** + * Default spacing between commits in distant history when selecting commits + * for bitmaps. Only applies when bitmaps are enabled: {@value} + * + * @see #setBitmapDistantCommitSpan(int) + * @since 4.2 + */ + public static final int DEFAULT_BITMAP_DISTANT_COMMIT_SPAN = 5000; + + /** + * Default count of branches required to activate inactive branch commit + * selection. If the number of branches is less than this then bitmaps for + * the entire commit history of all branches will be created, otherwise + * branches marked as "inactive" will have coverage for only partial + * history: {@value} + * + * @see #setBitmapExcessiveBranchCount(int) + * @since 4.2 + */ + public static final int DEFAULT_BITMAP_EXCESSIVE_BRANCH_COUNT = 100; + + /** + * Default age at which a branch is considered inactive. Age is taken as the + * number of days ago that the most recent commit was made to a branch. Only + * affects bitmap processing if bitmaps are enabled and the + * "excessive branch count" has been exceeded: {@value} + * + * @see #setBitmapInactiveBranchAgeInDays(int) + * @since 4.2 + */ + public static final int DEFAULT_BITMAP_INACTIVE_BRANCH_AGE_IN_DAYS = 90; private int compressionLevel = Deflater.DEFAULT_COMPRESSION; @@ -169,6 +228,18 @@ public class PackConfig { private boolean buildBitmaps = DEFAULT_BUILD_BITMAPS; + private int bitmapContiguousCommitCount = DEFAULT_BITMAP_CONTIGUOUS_COMMIT_COUNT; + + private int bitmapRecentCommitCount = DEFAULT_BITMAP_RECENT_COMMIT_COUNT; + + private int bitmapRecentCommitSpan = DEFAULT_BITMAP_RECENT_COMMIT_SPAN; + + private int bitmapDistantCommitSpan = DEFAULT_BITMAP_DISTANT_COMMIT_SPAN; + + private int bitmapExcessiveBranchCount = DEFAULT_BITMAP_EXCESSIVE_BRANCH_COUNT; + + private int bitmapInactiveBranchAgeInDays = DEFAULT_BITMAP_INACTIVE_BRANCH_AGE_IN_DAYS; + private boolean cutDeltaChains; /** Create a default configuration. */ @@ -222,6 +293,12 @@ public class PackConfig { this.executor = cfg.executor; this.indexVersion = cfg.indexVersion; this.buildBitmaps = cfg.buildBitmaps; + this.bitmapContiguousCommitCount = cfg.bitmapContiguousCommitCount; + this.bitmapRecentCommitCount = cfg.bitmapRecentCommitCount; + this.bitmapRecentCommitSpan = cfg.bitmapRecentCommitSpan; + this.bitmapDistantCommitSpan = cfg.bitmapDistantCommitSpan; + this.bitmapExcessiveBranchCount = cfg.bitmapExcessiveBranchCount; + this.bitmapInactiveBranchAgeInDays = cfg.bitmapInactiveBranchAgeInDays; this.cutDeltaChains = cfg.cutDeltaChains; } @@ -650,7 +727,7 @@ public class PackConfig { * oldest (most compatible) format available for the objects. * @see PackIndexWriter */ - public void setIndexVersion(final int version) { + public void setIndexVersion(int version) { indexVersion = version; } @@ -683,6 +760,162 @@ public class PackConfig { this.buildBitmaps = buildBitmaps; } + /** + * Get the count of most recent commits for which to build bitmaps. + * + * Default setting: {@value #DEFAULT_BITMAP_CONTIGUOUS_COMMIT_COUNT} + * + * @return the count of most recent commits for which to build bitmaps + * @since 4.2 + */ + public int getBitmapContiguousCommitCount() { + return bitmapContiguousCommitCount; + } + + /** + * Set the count of most recent commits for which to build bitmaps. + * + * Default setting: {@value #DEFAULT_BITMAP_CONTIGUOUS_COMMIT_COUNT} + * + * @param count + * the count of most recent commits for which to build bitmaps + * @since 4.2 + */ + public void setBitmapContiguousCommitCount(int count) { + bitmapContiguousCommitCount = count; + } + + /** + * Get the count at which to switch from "bitmapRecentCommitSpan" to + * "bitmapDistantCommitSpan". + * + * Default setting: {@value #DEFAULT_BITMAP_RECENT_COMMIT_COUNT} + * + * @return the count for switching between recent and distant spans + * @since 4.2 + */ + public int getBitmapRecentCommitCount() { + return bitmapRecentCommitCount; + } + + /** + * Set the count at which to switch from "bitmapRecentCommitSpan" to + * "bitmapDistantCommitSpan". + * + * Default setting: {@value #DEFAULT_BITMAP_RECENT_COMMIT_COUNT} + * + * @param count + * the count for switching between recent and distant spans + * @since 4.2 + */ + public void setBitmapRecentCommitCount(int count) { + bitmapRecentCommitCount = count; + } + + /** + * Get the span of commits when building bitmaps for recent history. + * + * Default setting: {@value #DEFAULT_BITMAP_RECENT_COMMIT_SPAN} + * + * @return the span of commits when building bitmaps for recent history + * @since 4.2 + */ + public int getBitmapRecentCommitSpan() { + return bitmapRecentCommitSpan; + } + + /** + * Set the span of commits when building bitmaps for recent history. + * + * Default setting: {@value #DEFAULT_BITMAP_RECENT_COMMIT_SPAN} + * + * @param span + * the span of commits when building bitmaps for recent history + * @since 4.2 + */ + public void setBitmapRecentCommitSpan(int span) { + bitmapRecentCommitSpan = span; + } + + /** + * Get the span of commits when building bitmaps for distant history. + * + * Default setting: {@value #DEFAULT_BITMAP_DISTANT_COMMIT_SPAN} + * + * @return the span of commits when building bitmaps for distant history + * @since 4.2 + */ + public int getBitmapDistantCommitSpan() { + return bitmapDistantCommitSpan; + } + + /** + * Set the span of commits when building bitmaps for distant history. + * + * Default setting: {@value #DEFAULT_BITMAP_DISTANT_COMMIT_SPAN} + * + * @param span + * the span of commits when building bitmaps for distant history + * @since 4.2 + */ + public void setBitmapDistantCommitSpan(int span) { + bitmapDistantCommitSpan = span; + } + + /** + * Get the count of branches deemed "excessive". If the count of branches in + * a repository exceeds this number and bitmaps are enabled, "inactive" + * branches will have fewer bitmaps than "active" branches. + * + * Default setting: {@value #DEFAULT_BITMAP_EXCESSIVE_BRANCH_COUNT} + * + * @return the count of branches deemed "excessive" + * @since 4.2 + */ + public int getBitmapExcessiveBranchCount() { + return bitmapExcessiveBranchCount; + } + + /** + * Set the count of branches deemed "excessive". If the count of branches in + * a repository exceeds this number and bitmaps are enabled, "inactive" + * branches will have fewer bitmaps than "active" branches. + * + * Default setting: {@value #DEFAULT_BITMAP_EXCESSIVE_BRANCH_COUNT} + * + * @param count + * the count of branches deemed "excessive" + * @since 4.2 + */ + public void setBitmapExcessiveBranchCount(int count) { + bitmapExcessiveBranchCount = count; + } + + /** + * Get the the age in days that marks a branch as "inactive". + * + * Default setting: {@value #DEFAULT_BITMAP_INACTIVE_BRANCH_AGE_IN_DAYS} + * + * @return the age in days that marks a branch as "inactive" + * @since 4.2 + */ + public int getBitmapInactiveBranchAgeInDays() { + return bitmapInactiveBranchAgeInDays; + } + + /** + * Set the the age in days that marks a branch as "inactive". + * + * Default setting: {@value #DEFAULT_BITMAP_INACTIVE_BRANCH_AGE_IN_DAYS} + * + * @param ageInDays + * the age in days that marks a branch as "inactive" + * @since 4.2 + */ + public void setBitmapInactiveBranchAgeInDays(int ageInDays) { + bitmapInactiveBranchAgeInDays = ageInDays; + } + /** * Update properties by setting fields from the configuration. * @@ -712,19 +945,36 @@ public class PackConfig { // These variables aren't standardized // setReuseDeltas(rc.getBoolean("pack", "reusedeltas", isReuseDeltas())); //$NON-NLS-1$ //$NON-NLS-2$ - setReuseObjects(rc.getBoolean("pack", "reuseobjects", isReuseObjects())); //$NON-NLS-1$ //$NON-NLS-2$ - setDeltaCompress(rc.getBoolean( - "pack", "deltacompression", isDeltaCompress())); //$NON-NLS-1$ //$NON-NLS-2$ - setCutDeltaChains(rc.getBoolean( - "pack", "cutdeltachains", getCutDeltaChains())); //$NON-NLS-1$ //$NON-NLS-2$ - setBuildBitmaps(rc.getBoolean("pack", "buildbitmaps", isBuildBitmaps())); //$NON-NLS-1$ //$NON-NLS-2$ + setReuseObjects( + rc.getBoolean("pack", "reuseobjects", isReuseObjects())); //$NON-NLS-1$ //$NON-NLS-2$ + setDeltaCompress( + rc.getBoolean("pack", "deltacompression", isDeltaCompress())); //$NON-NLS-1$ //$NON-NLS-2$ + setCutDeltaChains( + rc.getBoolean("pack", "cutdeltachains", getCutDeltaChains())); //$NON-NLS-1$ //$NON-NLS-2$ + setBuildBitmaps( + rc.getBoolean("pack", "buildbitmaps", isBuildBitmaps())); //$NON-NLS-1$ //$NON-NLS-2$ + setBitmapContiguousCommitCount( + rc.getInt("pack", "bitmapcontiguouscommitcount", //$NON-NLS-1$ //$NON-NLS-2$ + getBitmapContiguousCommitCount())); + setBitmapRecentCommitCount(rc.getInt("pack", "bitmaprecentcommitcount", //$NON-NLS-1$ //$NON-NLS-2$ + getBitmapRecentCommitCount())); + setBitmapRecentCommitSpan(rc.getInt("pack", "bitmaprecentcommitspan", //$NON-NLS-1$ //$NON-NLS-2$ + getBitmapRecentCommitSpan())); + setBitmapDistantCommitSpan(rc.getInt("pack", "bitmapdistantcommitspan", //$NON-NLS-1$ //$NON-NLS-2$ + getBitmapDistantCommitSpan())); + setBitmapExcessiveBranchCount(rc.getInt("pack", //$NON-NLS-1$ + "bitmapexcessivebranchcount", getBitmapExcessiveBranchCount())); //$NON-NLS-1$ + setBitmapInactiveBranchAgeInDays( + rc.getInt("pack", "bitmapinactivebranchageindays", //$NON-NLS-1$ //$NON-NLS-2$ + getBitmapInactiveBranchAgeInDays())); } public String toString() { final StringBuilder b = new StringBuilder(); b.append("maxDeltaDepth=").append(getMaxDeltaDepth()); //$NON-NLS-1$ b.append(", deltaSearchWindowSize=").append(getDeltaSearchWindowSize()); //$NON-NLS-1$ - b.append(", deltaSearchMemoryLimit=").append(getDeltaSearchMemoryLimit()); //$NON-NLS-1$ + b.append(", deltaSearchMemoryLimit=") //$NON-NLS-1$ + .append(getDeltaSearchMemoryLimit()); b.append(", deltaCacheSize=").append(getDeltaCacheSize()); //$NON-NLS-1$ b.append(", deltaCacheLimit=").append(getDeltaCacheLimit()); //$NON-NLS-1$ b.append(", compressionLevel=").append(getCompressionLevel()); //$NON-NLS-1$ @@ -735,6 +985,18 @@ public class PackConfig { b.append(", reuseObjects=").append(isReuseObjects()); //$NON-NLS-1$ b.append(", deltaCompress=").append(isDeltaCompress()); //$NON-NLS-1$ b.append(", buildBitmaps=").append(isBuildBitmaps()); //$NON-NLS-1$ + b.append(", bitmapContiguousCommitCount=") //$NON-NLS-1$ + .append(getBitmapContiguousCommitCount()); + b.append(", bitmapRecentCommitCount=") //$NON-NLS-1$ + .append(getBitmapRecentCommitCount()); + b.append(", bitmapRecentCommitSpan=") //$NON-NLS-1$ + .append(getBitmapRecentCommitSpan()); + b.append(", bitmapDistantCommitSpan=") //$NON-NLS-1$ + .append(getBitmapDistantCommitSpan()); + b.append(", bitmapExcessiveBranchCount=") //$NON-NLS-1$ + .append(getBitmapExcessiveBranchCount()); + b.append(", bitmapInactiveBranchAge=") //$NON-NLS-1$ + .append(getBitmapInactiveBranchAgeInDays()); return b.toString(); } }