github
/
jgit
mirror of https://gitee.com/github-mirror-repo/jgit.git
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

Fix a GC scalability issue when selecting commit bitmaps 

The previous algorithm selected commits by creating bitmaps at
each branch tip, doing a revwalk to populate each bitmap, and
looping in this way:
1) Select the remaining branch with the most commits (the branch
   whose bitmap has the highest cardinality)
2) Select well-spaced bitmaps in that branch
3) Remove commits in the selected branch from the remaining
   branch-tip bitmaps
4) Repeat at #1

This algorithm gave good commit selection on all branches but
a more uniform selection on "important" branches, where branch
length is the proxy for "important". However the algorithm
required N bitmaps of size M solely for the purpose of commit
selection, where N is the number of branch tips in the primary
GC pack, and M is the number of objects in the pack.

This new algorithm uses branch modification date as the proxy for
"important" branches, replacing the N*M memory allocation with a
single M-sized bitmap and N revwalks from new branch tips to
shared history (which will be short when there is a lot of shared
history).

GcCommitSelectionTest.testDistributionOnMultipleBranches verifies
that this algorithm still yields good coverage on all branches.

Change-Id: Ib6019b102b67eabb379e6b85623e4b5549590e6e
Signed-off-by: Terry Parker <tparker@google.com>
stable-5.1
tparker 6 years ago committed by Terry Parker
parent
34fb1d7d21
commit
2070d146cb
1 changed files with 197 additions and 227 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 258
      org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriterBitmapPreparer.java

258
org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriterBitmapPreparer.java
Unescape View File

@ -91,11 +91,10 @@ class PackWriterBitmapPreparer {
private static final int DAY_IN_SECONDS = 24 * 60 * 60; private static final int DAY_IN_SECONDS = 24 * 60 * 60;
private static final Comparator<BitmapBuilderEntry> ORDER_BY_CARDINALITY = new Comparator<BitmapBuilderEntry>() { private static final Comparator<RevCommit> ORDER_BY_REVERSE_TIMESTAMP = new Comparator<RevCommit>() {
@Override @Override
public int compare(BitmapBuilderEntry a, BitmapBuilderEntry b) { public int compare(RevCommit a, RevCommit b) {
return Integer.signum(a.getBuilder().cardinality() return Integer.signum(b.getCommitTime() - a.getCommitTime());
- b.getBuilder().cardinality());
} }
}; };
@ -164,50 +163,68 @@ class PackWriterBitmapPreparer {
* the cache hits for clients that are close to HEAD, which is the * the cache hits for clients that are close to HEAD, which is the
* majority of calculations performed. * majority of calculations performed.
*/ */
pm.beginTask(JGitText.get().selectingCommits, ProgressMonitor.UNKNOWN); try (RevWalk rw = new RevWalk(reader);
RevWalk rw = new RevWalk(reader); RevWalk rw2 = new RevWalk(reader)) {
pm.beginTask(JGitText.get().selectingCommits,
ProgressMonitor.UNKNOWN);
rw.setRetainBody(false); rw.setRetainBody(false);
CommitSelectionHelper selectionHelper = setupTipCommitBitmaps(rw, CommitSelectionHelper selectionHelper = captureOldAndNewCommits(rw,
expectedCommitCount, excludeFromBitmapSelection); expectedCommitCount, excludeFromBitmapSelection);
pm.endTask(); pm.endTask();
int totCommits = selectionHelper.getCommitCount(); // Add reused bitmaps from the previous GC pack's bitmap indices.
// Currently they are always fully reused, even if their spans don't
// match this run's PackConfig values.
int newCommits = selectionHelper.getCommitCount();
BlockList<BitmapCommit> selections = new BlockList<>( BlockList<BitmapCommit> selections = new BlockList<>(
totCommits / recentCommitSpan + 1); selectionHelper.reusedCommits.size()
+ newCommits / recentCommitSpan + 1);
for (BitmapCommit reuse : selectionHelper.reusedCommits) { for (BitmapCommit reuse : selectionHelper.reusedCommits) {
selections.add(reuse); selections.add(reuse);
} }
if (totCommits == 0) { if (newCommits == 0) {
for (AnyObjectId id : selectionHelper.peeledWants) { for (AnyObjectId id : selectionHelper.newWants) {
selections.add(new BitmapCommit(id, false, 0)); selections.add(new BitmapCommit(id, false, 0));
} }
return selections; return selections;
} }
pm.beginTask(JGitText.get().selectingCommits, totCommits); pm.beginTask(JGitText.get().selectingCommits, newCommits);
int totalWants = selectionHelper.peeledWants.size(); int totalWants = want.size();
BitmapBuilder seen = commitBitmapIndex.newBitmapBuilder();
for (BitmapBuilderEntry entry : selectionHelper.tipCommitBitmaps) { seen.or(selectionHelper.reusedCommitsBitmap);
BitmapBuilder bitmap = entry.getBuilder(); rw2.setRetainBody(false);
int cardinality = bitmap.cardinality(); rw2.setRevFilter(new NotInBitmapFilter(seen));
// Within this branch, keep ordered lists of commits representing // For each branch, do a revwalk to enumerate its commits. Exclude
// chains in its history, where each chain is a "sub-branch". // both reused commits and any commits seen in a previous branch.
// Ordering commits by these chains makes for fewer differences // Then iterate through all new commits from oldest to newest,
// between consecutive selected commits, which in turn provides // selecting well-spaced commits in this branch.
// better compression/on the run-length encoding of the XORs between for (RevCommit rc : selectionHelper.newWantsByNewest) {
// them. BitmapBuilder tipBitmap = commitBitmapIndex.newBitmapBuilder();
List<List<BitmapCommit>> chains = rw2.markStart((RevCommit) rw2.peel(rw2.parseAny(rc)));
new ArrayList<>(); RevCommit rc2;
while ((rc2 = rw2.next()) != null) {
tipBitmap.addObject(rc2, Constants.OBJ_COMMIT);
}
int cardinality = tipBitmap.cardinality();
seen.or(tipBitmap);
// Within this branch, keep ordered lists of commits
// representing chains in its history, where each chain is a
// "sub-branch". Ordering commits by these chains makes for
// fewer differences between consecutive selected commits, which
// in turn provides better compression/on the run-length
// encoding of the XORs between them.
List<List<BitmapCommit>> chains = new ArrayList<>();
// Mark the current branch as inactive if its tip commit isn't // Mark the current branch as inactive if its tip commit isn't
// recent and there are an excessive number of branches, to // recent and there are an excessive number of branches, to
// prevent memory bloat of computing too many bitmaps for stale // prevent memory bloat of computing too many bitmaps for stale
// branches. // branches.
boolean isActiveBranch = true; boolean isActiveBranch = true;
if (totalWants > excessiveBranchCount if (totalWants > excessiveBranchCount && !isRecentCommit(rc)) {
&& !isRecentCommit(entry.getCommit())) {
isActiveBranch = false; isActiveBranch = false;
} }
@ -217,10 +234,11 @@ class PackWriterBitmapPreparer {
int index = -1; int index = -1;
int nextIn = nextSpan(cardinality); int nextIn = nextSpan(cardinality);
int nextFlg = nextIn == distantCommitSpan int nextFlg = nextIn == distantCommitSpan
? PackBitmapIndex.FLAG_REUSE : 0; ? PackBitmapIndex.FLAG_REUSE
: 0;
// For the current branch, iterate through all commits from oldest // For the current branch, iterate through all commits from
// to newest. // oldest to newest.
for (RevCommit c : selectionHelper) { for (RevCommit c : selectionHelper) {
// Optimization: if we have found all the commits for this // Optimization: if we have found all the commits for this
// branch, stop searching // branch, stop searching
@ -230,7 +248,7 @@ class PackWriterBitmapPreparer {
} }
// Ignore commits that are not in this branch // Ignore commits that are not in this branch
if (!bitmap.contains(c)) { if (!tipBitmap.contains(c)) {
continue; continue;
} }
@ -239,7 +257,7 @@ class PackWriterBitmapPreparer {
pm.update(1); pm.update(1);
// Always pick the items in wants, prefer merge commits. // Always pick the items in wants, prefer merge commits.
if (selectionHelper.peeledWants.remove(c)) { if (selectionHelper.newWants.remove(c)) {
if (nextIn > 0) { if (nextIn > 0) {
nextFlg = 0; nextFlg = 0;
} }
@ -264,27 +282,30 @@ class PackWriterBitmapPreparer {
int flags = nextFlg; int flags = nextFlg;
nextIn = nextSpan(distanceFromTip); nextIn = nextSpan(distanceFromTip);
nextFlg = nextIn == distantCommitSpan nextFlg = nextIn == distantCommitSpan
? PackBitmapIndex.FLAG_REUSE : 0; ? PackBitmapIndex.FLAG_REUSE
: 0;
BitmapBuilder fullBitmap = commitBitmapIndex.newBitmapBuilder(); // Create the commit bitmap for the current commit
BitmapBuilder bitmap = commitBitmapIndex.newBitmapBuilder();
rw.reset(); rw.reset();
rw.markStart(c); rw.markStart(c);
rw.setRevFilter(new AddUnseenToBitmapFilter( rw.setRevFilter(new AddUnseenToBitmapFilter(
selectionHelper.reusedCommitsBitmap, fullBitmap)); selectionHelper.reusedCommitsBitmap, bitmap));
while (rw.next() != null) { while (rw.next() != null) {
// The RevFilter adds the reachable commits from this // The filter adds the reachable commits to bitmap.
// selected commit to fullBitmap.
} }
// Sort the commits by independent chains in this branch's // Sort the commits by independent chains in this branch's
// history, yielding better compression when building bitmaps. // history, yielding better compression when building
// bitmaps.
List<BitmapCommit> longestAncestorChain = null; List<BitmapCommit> longestAncestorChain = null;
for (List<BitmapCommit> chain : chains) { for (List<BitmapCommit> chain : chains) {
BitmapCommit mostRecentCommit = chain.get(chain.size() - 1); BitmapCommit mostRecentCommit = chain
if (fullBitmap.contains(mostRecentCommit)) { .get(chain.size() - 1);
if (bitmap.contains(mostRecentCommit)) {
if (longestAncestorChain == null if (longestAncestorChain == null
|| longestAncestorChain.size() < chain.size()) { || longestAncestorChain.size() < chain
.size()) {
longestAncestorChain = chain; longestAncestorChain = chain;
} }
} }
@ -294,9 +315,9 @@ class PackWriterBitmapPreparer {
longestAncestorChain = new ArrayList<>(); longestAncestorChain = new ArrayList<>();
chains.add(longestAncestorChain); chains.add(longestAncestorChain);
} }
longestAncestorChain.add(new BitmapCommit( longestAncestorChain.add(new BitmapCommit(c,
c, !longestAncestorChain.isEmpty(), flags)); !longestAncestorChain.isEmpty(), flags));
writeBitmaps.addBitmap(c, fullBitmap, 0); writeBitmaps.addBitmap(c, bitmap, 0);
} }
for (List<BitmapCommit> chain : chains) { for (List<BitmapCommit> chain : chains) {
@ -306,13 +327,14 @@ class PackWriterBitmapPreparer {
writeBitmaps.clearBitmaps(); // Remove the temporary commit bitmaps. writeBitmaps.clearBitmaps(); // Remove the temporary commit bitmaps.
// Add the remaining peeledWant // Add the remaining peeledWant
for (AnyObjectId remainingWant : selectionHelper.peeledWants) { for (AnyObjectId remainingWant : selectionHelper.newWants) {
selections.add(new BitmapCommit(remainingWant, false, 0)); selections.add(new BitmapCommit(remainingWant, false, 0));
} }
pm.endTask(); pm.endTask();
return selections; return selections;
} }
}
private boolean isRecentCommit(RevCommit revCommit) { private boolean isRecentCommit(RevCommit revCommit) {
return revCommit.getCommitTime() > inactiveBranchTimestamp; return revCommit.getCommitTime() > inactiveBranchTimestamp;
@ -358,9 +380,8 @@ class PackWriterBitmapPreparer {
} }
/** /**
* For each of the {@code want}s, which represent the tip commit of each * Records which of the {@code wants} can be found in the previous GC pack's
* branch, set up an initial {@link BitmapBuilder}. Reuse previously built * bitmap indices and which are new.
* bitmaps if possible.
* *
* @param rw * @param rw
* a {@link RevWalk} to find reachable objects in this repository * a {@link RevWalk} to find reachable objects in this repository
@ -369,8 +390,9 @@ class PackWriterBitmapPreparer {
* unreachable garbage. * unreachable garbage.
* @param excludeFromBitmapSelection * @param excludeFromBitmapSelection
* commits that should be excluded from bitmap selection * commits that should be excluded from bitmap selection
* @return a {@link CommitSelectionHelper} containing bitmaps for the tip * @return a {@link CommitSelectionHelper} capturing which commits are
* commits * covered by a previous pack's bitmaps and which new commits need
* bitmap coverage
* @throws IncorrectObjectTypeException * @throws IncorrectObjectTypeException
* if any of the processed objects is not a commit * if any of the processed objects is not a commit
* @throws IOException * @throws IOException
@ -378,11 +400,12 @@ class PackWriterBitmapPreparer {
* @throws MissingObjectException * @throws MissingObjectException
* if an expected object is missing * if an expected object is missing
*/ */
private CommitSelectionHelper setupTipCommitBitmaps(RevWalk rw, private CommitSelectionHelper captureOldAndNewCommits(RevWalk rw,
int expectedCommitCount, int expectedCommitCount,
Set<? extends ObjectId> excludeFromBitmapSelection) Set<? extends ObjectId> excludeFromBitmapSelection)
throws IncorrectObjectTypeException, IOException, throws IncorrectObjectTypeException, IOException,
MissingObjectException { MissingObjectException {
// Track bitmaps and commits from the previous GC pack bitmap indices.
BitmapBuilder reuse = commitBitmapIndex.newBitmapBuilder(); BitmapBuilder reuse = commitBitmapIndex.newBitmapBuilder();
List<BitmapCommit> reuseCommits = new ArrayList<>(); List<BitmapCommit> reuseCommits = new ArrayList<>();
for (PackBitmapIndexRemapper.Entry entry : bitmapRemapper) { for (PackBitmapIndexRemapper.Entry entry : bitmapRemapper) {
@ -404,11 +427,10 @@ class PackWriterBitmapPreparer {
} }
} }
// Add branch tips that are not represented in old bitmap indices. Set // Add branch tips that are not represented in a previous pack's bitmap
// up the RevWalk to walk the new commits not in the old packs. // indices. Set up a RevWalk to find new commits not in the old packs.
List<BitmapBuilderEntry> tipCommitBitmaps = new ArrayList<>( List<RevCommit> newWantsByNewest = new ArrayList<>(want.size());
want.size()); Set<RevCommit> newWants = new HashSet<>(want.size());
Set<RevCommit> peeledWant = new HashSet<>(want.size());
for (AnyObjectId objectId : want) { for (AnyObjectId objectId : want) {
RevObject ro = rw.peel(rw.parseAny(objectId)); RevObject ro = rw.peel(rw.parseAny(objectId));
if (!(ro instanceof RevCommit) || reuse.contains(ro) if (!(ro instanceof RevCommit) || reuse.contains(ro)
@ -417,59 +439,25 @@ class PackWriterBitmapPreparer {
} }
RevCommit rc = (RevCommit) ro; RevCommit rc = (RevCommit) ro;
peeledWant.add(rc);
rw.markStart(rc); rw.markStart(rc);
newWants.add(rc);
BitmapBuilder bitmap = commitBitmapIndex.newBitmapBuilder(); newWantsByNewest.add(rc);
bitmap.addObject(rc, Constants.OBJ_COMMIT);
tipCommitBitmaps.add(new BitmapBuilderEntry(rc, bitmap));
} }
// Create a list of commits in reverse order (older to newer). // Create a list of commits in reverse order (older to newer) that are
// For each branch that contains the commit, mark its parents as being // not in the previous bitmap indices and are reachable.
// in the bitmap.
rw.setRevFilter(new NotInBitmapFilter(reuse)); rw.setRevFilter(new NotInBitmapFilter(reuse));
RevCommit[] commits = new RevCommit[expectedCommitCount]; RevCommit[] commits = new RevCommit[expectedCommitCount];
int pos = commits.length; int pos = commits.length;
RevCommit rc; RevCommit rc;
while ((rc = rw.next()) != null && pos > 0) { while ((rc = rw.next()) != null && pos > 0) {
commits[--pos] = rc; commits[--pos] = rc;
for (BitmapBuilderEntry entry : tipCommitBitmaps) {
BitmapBuilder bitmap = entry.getBuilder();
if (!bitmap.contains(rc)) {
continue;
}
for (RevCommit c : rc.getParents()) {
if (reuse.contains(c)) {
continue;
}
bitmap.addObject(c, Constants.OBJ_COMMIT);
}
}
pm.update(1);
}
// Sort the tip commit bitmaps. Find the one containing the most
// commits, remove those commits from the remaining bitmaps, resort and
// repeat.
List<BitmapBuilderEntry> orderedTipCommitBitmaps = new ArrayList<>(
tipCommitBitmaps.size());
while (!tipCommitBitmaps.isEmpty()) {
BitmapBuilderEntry largest =
Collections.max(tipCommitBitmaps, ORDER_BY_CARDINALITY);
tipCommitBitmaps.remove(largest);
orderedTipCommitBitmaps.add(largest);
// Update the remaining paths, by removing the objects from
// the path that was just added.
for (int i = tipCommitBitmaps.size() - 1; i >= 0; i--) {
tipCommitBitmaps.get(i).getBuilder()
.andNot(largest.getBuilder());
}
} }
return new CommitSelectionHelper(peeledWant, commits, pos, // Sort the new wants by reverse commit time.
orderedTipCommitBitmaps, reuse, reuseCommits); Collections.sort(newWantsByNewest, ORDER_BY_REVERSE_TIMESTAMP);
return new CommitSelectionHelper(newWants, commits, pos,
newWantsByNewest, reuse, reuseCommits);
} }
/*- /*-
@ -536,55 +524,37 @@ class PackWriterBitmapPreparer {
} }
} }
/**
* A POJO representing a Pair<RevCommit, BitmapBuidler>.
*/
private static final class BitmapBuilderEntry {
private final RevCommit commit;
private final BitmapBuilder builder;
BitmapBuilderEntry(RevCommit commit, BitmapBuilder builder) {
this.commit = commit;
this.builder = builder;
}
RevCommit getCommit() {
return commit;
}
BitmapBuilder getBuilder() {
return builder;
}
}
/** /**
* Container for state used in the first phase of selecting commits, which * Container for state used in the first phase of selecting commits, which
* walks all of the reachable commits via the branch tips ( * walks all of the reachable commits via the branch tips that are not
* {@code peeledWants}), stores them in {@code commitsByOldest}, and sets up * covered by a previous pack's bitmaps ({@code newWants}) and stores them
* bitmaps for each branch tip ({@code tipCommitBitmaps}). * in {@code newCommitsByOldest}. {@code newCommitsByOldest} is initialized
* {@code commitsByOldest} is initialized with an expected size of all * with an expected size of all commits, but may be smaller if some commits
* commits, but may be smaller if some commits are unreachable, in which * are unreachable and/or some commits are covered by a previous pack's
* case {@code commitStartPos} will contain a positive offset to the root * bitmaps. {@code commitStartPos} will contain a positive offset to either
* commit. * the root commit or the oldest commit not covered by previous bitmaps.
*/ */
private static final class CommitSelectionHelper implements Iterable<RevCommit> { private static final class CommitSelectionHelper implements Iterable<RevCommit> {
final Set<? extends ObjectId> peeledWants; final Set<? extends ObjectId> newWants;
final List<BitmapBuilderEntry> tipCommitBitmaps;
final List<RevCommit> newWantsByNewest;
final BitmapBuilder reusedCommitsBitmap; final BitmapBuilder reusedCommitsBitmap;
final Iterable<BitmapCommit> reusedCommits;
final RevCommit[] commitsByOldest;
final int commitStartPos;
CommitSelectionHelper(Set<? extends ObjectId> peeledWant, final List<BitmapCommit> reusedCommits;
final RevCommit[] newCommitsByOldest;
final int newCommitStartPos;
CommitSelectionHelper(Set<? extends ObjectId> newWants,
RevCommit[] commitsByOldest, int commitStartPos, RevCommit[] commitsByOldest, int commitStartPos,
List<BitmapBuilderEntry> bitmapEntries, List<RevCommit> newWantsByNewest,
BitmapBuilder reusedCommitsBitmap, BitmapBuilder reusedCommitsBitmap,
Iterable<BitmapCommit> reuse) { List<BitmapCommit> reuse) {
this.peeledWants = peeledWant; this.newWants = newWants;
this.commitsByOldest = commitsByOldest; this.newCommitsByOldest = commitsByOldest;
this.commitStartPos = commitStartPos; this.newCommitStartPos = commitStartPos;
this.tipCommitBitmaps = bitmapEntries; this.newWantsByNewest = newWantsByNewest;
this.reusedCommitsBitmap = reusedCommitsBitmap; this.reusedCommitsBitmap = reusedCommitsBitmap;
this.reusedCommits = reuse; this.reusedCommits = reuse;
} }
@ -594,16 +564,16 @@ class PackWriterBitmapPreparer {
// Member variables referenced by this iterator will have synthetic // Member variables referenced by this iterator will have synthetic
// accessors generated for them if they are made private. // accessors generated for them if they are made private.
return new Iterator<RevCommit>() { return new Iterator<RevCommit>() {
int pos = commitStartPos; int pos = newCommitStartPos;
@Override @Override
public boolean hasNext() { public boolean hasNext() {
return pos < commitsByOldest.length; return pos < newCommitsByOldest.length;
} }
@Override @Override
public RevCommit next() { public RevCommit next() {
return commitsByOldest[pos++]; return newCommitsByOldest[pos++];
} }
@Override @Override
@ -614,7 +584,7 @@ class PackWriterBitmapPreparer {
} }
int getCommitCount() { int getCommitCount() {
return commitsByOldest.length - commitStartPos; return newCommitsByOldest.length - newCommitStartPos;
} }
} }
} }

Write Preview
Loading…
Cancel
Save