github
/
jgit
mirror of https://gitee.com/github-mirror-repo/jgit.git
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

Fix a GC scalability issue when selecting commit bitmaps 

The previous algorithm selected commits by creating bitmaps at
each branch tip, doing a revwalk to populate each bitmap, and
looping in this way:
1) Select the remaining branch with the most commits (the branch
   whose bitmap has the highest cardinality)
2) Select well-spaced bitmaps in that branch
3) Remove commits in the selected branch from the remaining
   branch-tip bitmaps
4) Repeat at #1

This algorithm gave good commit selection on all branches but
a more uniform selection on "important" branches, where branch
length is the proxy for "important". However the algorithm
required N bitmaps of size M solely for the purpose of commit
selection, where N is the number of branch tips in the primary
GC pack, and M is the number of objects in the pack.

This new algorithm uses branch modification date as the proxy for
"important" branches, replacing the N*M memory allocation with a
single M-sized bitmap and N revwalks from new branch tips to
shared history (which will be short when there is a lot of shared
history).

GcCommitSelectionTest.testDistributionOnMultipleBranches verifies
that this algorithm still yields good coverage on all branches.

Change-Id: Ib6019b102b67eabb379e6b85623e4b5549590e6e
Signed-off-by: Terry Parker <tparker@google.com>
stable-5.1
tparker 6 years ago committed by Terry Parker
parent
34fb1d7d21
commit
2070d146cb
1 changed files with 197 additions and 227 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 424
      org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriterBitmapPreparer.java

424
org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriterBitmapPreparer.java
Unescape View File

@ -91,11 +91,10 @@ class PackWriterBitmapPreparer {
private static final int DAY_IN_SECONDS = 24 * 60 * 60; private static final int DAY_IN_SECONDS = 24 * 60 * 60;
private static final Comparator<BitmapBuilderEntry> ORDER_BY_CARDINALITY = new Comparator<BitmapBuilderEntry>() { private static final Comparator<RevCommit> ORDER_BY_REVERSE_TIMESTAMP = new Comparator<RevCommit>() {
@Override @Override
public int compare(BitmapBuilderEntry a, BitmapBuilderEntry b) { public int compare(RevCommit a, RevCommit b) {
return Integer.signum(a.getBuilder().cardinality() return Integer.signum(b.getCommitTime() - a.getCommitTime());
- b.getBuilder().cardinality());
} }
}; };
@ -164,154 +163,177 @@ class PackWriterBitmapPreparer {
* the cache hits for clients that are close to HEAD, which is the * the cache hits for clients that are close to HEAD, which is the
* majority of calculations performed. * majority of calculations performed.
*/ */
pm.beginTask(JGitText.get().selectingCommits, ProgressMonitor.UNKNOWN); try (RevWalk rw = new RevWalk(reader);
RevWalk rw = new RevWalk(reader); RevWalk rw2 = new RevWalk(reader)) {
rw.setRetainBody(false); pm.beginTask(JGitText.get().selectingCommits,
CommitSelectionHelper selectionHelper = setupTipCommitBitmaps(rw, ProgressMonitor.UNKNOWN);
expectedCommitCount, excludeFromBitmapSelection); rw.setRetainBody(false);
pm.endTask(); CommitSelectionHelper selectionHelper = captureOldAndNewCommits(rw,
expectedCommitCount, excludeFromBitmapSelection);
int totCommits = selectionHelper.getCommitCount(); pm.endTask();
BlockList<BitmapCommit> selections = new BlockList<>(
totCommits / recentCommitSpan + 1); // Add reused bitmaps from the previous GC pack's bitmap indices.
for (BitmapCommit reuse : selectionHelper.reusedCommits) { // Currently they are always fully reused, even if their spans don't
selections.add(reuse); // match this run's PackConfig values.
} int newCommits = selectionHelper.getCommitCount();
BlockList<BitmapCommit> selections = new BlockList<>(
if (totCommits == 0) { selectionHelper.reusedCommits.size()
for (AnyObjectId id : selectionHelper.peeledWants) { + newCommits / recentCommitSpan + 1);
selections.add(new BitmapCommit(id, false, 0)); for (BitmapCommit reuse : selectionHelper.reusedCommits) {
selections.add(reuse);
} }
return selections;
}
pm.beginTask(JGitText.get().selectingCommits, totCommits); if (newCommits == 0) {
int totalWants = selectionHelper.peeledWants.size(); for (AnyObjectId id : selectionHelper.newWants) {
selections.add(new BitmapCommit(id, false, 0));
for (BitmapBuilderEntry entry : selectionHelper.tipCommitBitmaps) { }
BitmapBuilder bitmap = entry.getBuilder(); return selections;
int cardinality = bitmap.cardinality();
// Within this branch, keep ordered lists of commits representing
// chains in its history, where each chain is a "sub-branch".
// Ordering commits by these chains makes for fewer differences
// between consecutive selected commits, which in turn provides
// better compression/on the run-length encoding of the XORs between
// them.
List<List<BitmapCommit>> chains =
new ArrayList<>();
// Mark the current branch as inactive if its tip commit isn't
// recent and there are an excessive number of branches, to
// prevent memory bloat of computing too many bitmaps for stale
// branches.
boolean isActiveBranch = true;
if (totalWants > excessiveBranchCount
&& !isRecentCommit(entry.getCommit())) {
isActiveBranch = false;
} }
// Insert bitmaps at the offsets suggested by the pm.beginTask(JGitText.get().selectingCommits, newCommits);
// nextSelectionDistance() heuristic. Only reuse bitmaps created int totalWants = want.size();
// for more distant commits. BitmapBuilder seen = commitBitmapIndex.newBitmapBuilder();
int index = -1; seen.or(selectionHelper.reusedCommitsBitmap);
int nextIn = nextSpan(cardinality); rw2.setRetainBody(false);
int nextFlg = nextIn == distantCommitSpan rw2.setRevFilter(new NotInBitmapFilter(seen));
? PackBitmapIndex.FLAG_REUSE : 0;
// For each branch, do a revwalk to enumerate its commits. Exclude
// For the current branch, iterate through all commits from oldest // both reused commits and any commits seen in a previous branch.
// to newest. // Then iterate through all new commits from oldest to newest,
for (RevCommit c : selectionHelper) { // selecting well-spaced commits in this branch.
// Optimization: if we have found all the commits for this for (RevCommit rc : selectionHelper.newWantsByNewest) {
// branch, stop searching BitmapBuilder tipBitmap = commitBitmapIndex.newBitmapBuilder();
int distanceFromTip = cardinality - index - 1; rw2.markStart((RevCommit) rw2.peel(rw2.parseAny(rc)));
if (distanceFromTip == 0) { RevCommit rc2;
break; while ((rc2 = rw2.next()) != null) {
tipBitmap.addObject(rc2, Constants.OBJ_COMMIT);
} }
int cardinality = tipBitmap.cardinality();
// Ignore commits that are not in this branch seen.or(tipBitmap);
if (!bitmap.contains(c)) {
continue; // Within this branch, keep ordered lists of commits
// representing chains in its history, where each chain is a
// "sub-branch". Ordering commits by these chains makes for
// fewer differences between consecutive selected commits, which
// in turn provides better compression/on the run-length
// encoding of the XORs between them.
List<List<BitmapCommit>> chains = new ArrayList<>();
// Mark the current branch as inactive if its tip commit isn't
// recent and there are an excessive number of branches, to
// prevent memory bloat of computing too many bitmaps for stale
// branches.
boolean isActiveBranch = true;
if (totalWants > excessiveBranchCount && !isRecentCommit(rc)) {
isActiveBranch = false;
} }
index++; // Insert bitmaps at the offsets suggested by the
nextIn--; // nextSelectionDistance() heuristic. Only reuse bitmaps created
pm.update(1); // for more distant commits.
int index = -1;
// Always pick the items in wants, prefer merge commits. int nextIn = nextSpan(cardinality);
if (selectionHelper.peeledWants.remove(c)) { int nextFlg = nextIn == distantCommitSpan
if (nextIn > 0) { ? PackBitmapIndex.FLAG_REUSE
nextFlg = 0; : 0;
// For the current branch, iterate through all commits from
// oldest to newest.
for (RevCommit c : selectionHelper) {
// Optimization: if we have found all the commits for this
// branch, stop searching
int distanceFromTip = cardinality - index - 1;
if (distanceFromTip == 0) {
break;
} }
} else {
boolean stillInSpan = nextIn >= 0; // Ignore commits that are not in this branch
boolean isMergeCommit = c.getParentCount() > 1; if (!tipBitmap.contains(c)) {
// Force selection if:
// a) we have exhausted the window looking for merges
// b) we are in the top commits of an active branch
// c) we are at a branch tip
boolean mustPick = (nextIn <= -recentCommitSpan)
|| (isActiveBranch
&& (distanceFromTip <= contiguousCommitCount))
|| (distanceFromTip == 1); // most recent commit
if (!mustPick && (stillInSpan || !isMergeCommit)) {
continue; continue;
} }
}
// This commit is selected. index++;
// Calculate where to look for the next one. nextIn--;
int flags = nextFlg; pm.update(1);
nextIn = nextSpan(distanceFromTip);
nextFlg = nextIn == distantCommitSpan
? PackBitmapIndex.FLAG_REUSE : 0;
BitmapBuilder fullBitmap = commitBitmapIndex.newBitmapBuilder();
rw.reset();
rw.markStart(c);
rw.setRevFilter(new AddUnseenToBitmapFilter(
selectionHelper.reusedCommitsBitmap, fullBitmap));
while (rw.next() != null) {
// The RevFilter adds the reachable commits from this
// selected commit to fullBitmap.
}
// Sort the commits by independent chains in this branch's // Always pick the items in wants, prefer merge commits.
// history, yielding better compression when building bitmaps. if (selectionHelper.newWants.remove(c)) {
List<BitmapCommit> longestAncestorChain = null; if (nextIn > 0) {
for (List<BitmapCommit> chain : chains) { nextFlg = 0;
BitmapCommit mostRecentCommit = chain.get(chain.size() - 1); }
if (fullBitmap.contains(mostRecentCommit)) { } else {
if (longestAncestorChain == null boolean stillInSpan = nextIn >= 0;
|| longestAncestorChain.size() < chain.size()) { boolean isMergeCommit = c.getParentCount() > 1;
longestAncestorChain = chain; // Force selection if:
// a) we have exhausted the window looking for merges
// b) we are in the top commits of an active branch
// c) we are at a branch tip
boolean mustPick = (nextIn <= -recentCommitSpan)
|| (isActiveBranch
&& (distanceFromTip <= contiguousCommitCount))
|| (distanceFromTip == 1); // most recent commit
if (!mustPick && (stillInSpan || !isMergeCommit)) {
continue;
} }
} }
// This commit is selected.
// Calculate where to look for the next one.
int flags = nextFlg;
nextIn = nextSpan(distanceFromTip);
nextFlg = nextIn == distantCommitSpan
? PackBitmapIndex.FLAG_REUSE
: 0;
// Create the commit bitmap for the current commit
BitmapBuilder bitmap = commitBitmapIndex.newBitmapBuilder();
rw.reset();
rw.markStart(c);
rw.setRevFilter(new AddUnseenToBitmapFilter(
selectionHelper.reusedCommitsBitmap, bitmap));
while (rw.next() != null) {
// The filter adds the reachable commits to bitmap.
}
// Sort the commits by independent chains in this branch's
// history, yielding better compression when building
// bitmaps.
List<BitmapCommit> longestAncestorChain = null;
for (List<BitmapCommit> chain : chains) {
BitmapCommit mostRecentCommit = chain
.get(chain.size() - 1);
if (bitmap.contains(mostRecentCommit)) {
if (longestAncestorChain == null
|| longestAncestorChain.size() < chain
.size()) {
longestAncestorChain = chain;
}
}
}
if (longestAncestorChain == null) {
longestAncestorChain = new ArrayList<>();
chains.add(longestAncestorChain);
}
longestAncestorChain.add(new BitmapCommit(c,
!longestAncestorChain.isEmpty(), flags));
writeBitmaps.addBitmap(c, bitmap, 0);
} }
if (longestAncestorChain == null) { for (List<BitmapCommit> chain : chains) {
longestAncestorChain = new ArrayList<>(); selections.addAll(chain);
chains.add(longestAncestorChain);
} }
longestAncestorChain.add(new BitmapCommit(
c, !longestAncestorChain.isEmpty(), flags));
writeBitmaps.addBitmap(c, fullBitmap, 0);
} }
writeBitmaps.clearBitmaps(); // Remove the temporary commit bitmaps.
for (List<BitmapCommit> chain : chains) { // Add the remaining peeledWant
selections.addAll(chain); for (AnyObjectId remainingWant : selectionHelper.newWants) {
selections.add(new BitmapCommit(remainingWant, false, 0));
} }
}
writeBitmaps.clearBitmaps(); // Remove the temporary commit bitmaps.
// Add the remaining peeledWant pm.endTask();
for (AnyObjectId remainingWant : selectionHelper.peeledWants) { return selections;
selections.add(new BitmapCommit(remainingWant, false, 0));
} }
pm.endTask();
return selections;
} }
private boolean isRecentCommit(RevCommit revCommit) { private boolean isRecentCommit(RevCommit revCommit) {
@ -358,9 +380,8 @@ class PackWriterBitmapPreparer {
} }
/** /**
* For each of the {@code want}s, which represent the tip commit of each * Records which of the {@code wants} can be found in the previous GC pack's
* branch, set up an initial {@link BitmapBuilder}. Reuse previously built * bitmap indices and which are new.
* bitmaps if possible.
* *
* @param rw * @param rw
* a {@link RevWalk} to find reachable objects in this repository * a {@link RevWalk} to find reachable objects in this repository
@ -369,8 +390,9 @@ class PackWriterBitmapPreparer {
* unreachable garbage. * unreachable garbage.
* @param excludeFromBitmapSelection * @param excludeFromBitmapSelection
* commits that should be excluded from bitmap selection * commits that should be excluded from bitmap selection
* @return a {@link CommitSelectionHelper} containing bitmaps for the tip * @return a {@link CommitSelectionHelper} capturing which commits are
* commits * covered by a previous pack's bitmaps and which new commits need
* bitmap coverage
* @throws IncorrectObjectTypeException * @throws IncorrectObjectTypeException
* if any of the processed objects is not a commit * if any of the processed objects is not a commit
* @throws IOException * @throws IOException
@ -378,11 +400,12 @@ class PackWriterBitmapPreparer {
* @throws MissingObjectException * @throws MissingObjectException
* if an expected object is missing * if an expected object is missing
*/ */
private CommitSelectionHelper setupTipCommitBitmaps(RevWalk rw, private CommitSelectionHelper captureOldAndNewCommits(RevWalk rw,
int expectedCommitCount, int expectedCommitCount,
Set<? extends ObjectId> excludeFromBitmapSelection) Set<? extends ObjectId> excludeFromBitmapSelection)
throws IncorrectObjectTypeException, IOException, throws IncorrectObjectTypeException, IOException,
MissingObjectException { MissingObjectException {
// Track bitmaps and commits from the previous GC pack bitmap indices.
BitmapBuilder reuse = commitBitmapIndex.newBitmapBuilder(); BitmapBuilder reuse = commitBitmapIndex.newBitmapBuilder();
List<BitmapCommit> reuseCommits = new ArrayList<>(); List<BitmapCommit> reuseCommits = new ArrayList<>();
for (PackBitmapIndexRemapper.Entry entry : bitmapRemapper) { for (PackBitmapIndexRemapper.Entry entry : bitmapRemapper) {
@ -404,11 +427,10 @@ class PackWriterBitmapPreparer {
} }
} }
// Add branch tips that are not represented in old bitmap indices. Set // Add branch tips that are not represented in a previous pack's bitmap
// up the RevWalk to walk the new commits not in the old packs. // indices. Set up a RevWalk to find new commits not in the old packs.
List<BitmapBuilderEntry> tipCommitBitmaps = new ArrayList<>( List<RevCommit> newWantsByNewest = new ArrayList<>(want.size());
want.size()); Set<RevCommit> newWants = new HashSet<>(want.size());
Set<RevCommit> peeledWant = new HashSet<>(want.size());
for (AnyObjectId objectId : want) { for (AnyObjectId objectId : want) {
RevObject ro = rw.peel(rw.parseAny(objectId)); RevObject ro = rw.peel(rw.parseAny(objectId));
if (!(ro instanceof RevCommit) || reuse.contains(ro) if (!(ro instanceof RevCommit) || reuse.contains(ro)
@ -417,59 +439,25 @@ class PackWriterBitmapPreparer {
} }
RevCommit rc = (RevCommit) ro; RevCommit rc = (RevCommit) ro;
peeledWant.add(rc);
rw.markStart(rc); rw.markStart(rc);
newWants.add(rc);
BitmapBuilder bitmap = commitBitmapIndex.newBitmapBuilder(); newWantsByNewest.add(rc);
bitmap.addObject(rc, Constants.OBJ_COMMIT);
tipCommitBitmaps.add(new BitmapBuilderEntry(rc, bitmap));
} }
// Create a list of commits in reverse order (older to newer). // Create a list of commits in reverse order (older to newer) that are
// For each branch that contains the commit, mark its parents as being // not in the previous bitmap indices and are reachable.
// in the bitmap.
rw.setRevFilter(new NotInBitmapFilter(reuse)); rw.setRevFilter(new NotInBitmapFilter(reuse));
RevCommit[] commits = new RevCommit[expectedCommitCount]; RevCommit[] commits = new RevCommit[expectedCommitCount];
int pos = commits.length; int pos = commits.length;
RevCommit rc; RevCommit rc;
while ((rc = rw.next()) != null && pos > 0) { while ((rc = rw.next()) != null && pos > 0) {
commits[--pos] = rc; commits[--pos] = rc;
for (BitmapBuilderEntry entry : tipCommitBitmaps) {
BitmapBuilder bitmap = entry.getBuilder();
if (!bitmap.contains(rc)) {
continue;
}
for (RevCommit c : rc.getParents()) {
if (reuse.contains(c)) {
continue;
}
bitmap.addObject(c, Constants.OBJ_COMMIT);
}
}
pm.update(1);
} }
// Sort the tip commit bitmaps. Find the one containing the most // Sort the new wants by reverse commit time.
// commits, remove those commits from the remaining bitmaps, resort and Collections.sort(newWantsByNewest, ORDER_BY_REVERSE_TIMESTAMP);
// repeat. return new CommitSelectionHelper(newWants, commits, pos,
List<BitmapBuilderEntry> orderedTipCommitBitmaps = new ArrayList<>( newWantsByNewest, reuse, reuseCommits);
tipCommitBitmaps.size());
while (!tipCommitBitmaps.isEmpty()) {
BitmapBuilderEntry largest =
Collections.max(tipCommitBitmaps, ORDER_BY_CARDINALITY);
tipCommitBitmaps.remove(largest);
orderedTipCommitBitmaps.add(largest);
// Update the remaining paths, by removing the objects from
// the path that was just added.
for (int i = tipCommitBitmaps.size() - 1; i >= 0; i--) {
tipCommitBitmaps.get(i).getBuilder()
.andNot(largest.getBuilder());
}
}
return new CommitSelectionHelper(peeledWant, commits, pos,
orderedTipCommitBitmaps, reuse, reuseCommits);
} }
/*- /*-
@ -536,55 +524,37 @@ class PackWriterBitmapPreparer {
} }
} }
/**
* A POJO representing a Pair<RevCommit, BitmapBuidler>.
*/
private static final class BitmapBuilderEntry {
private final RevCommit commit;
private final BitmapBuilder builder;
BitmapBuilderEntry(RevCommit commit, BitmapBuilder builder) {
this.commit = commit;
this.builder = builder;
}
RevCommit getCommit() {
return commit;
}
BitmapBuilder getBuilder() {
return builder;
}
}
/** /**
* Container for state used in the first phase of selecting commits, which * Container for state used in the first phase of selecting commits, which
* walks all of the reachable commits via the branch tips ( * walks all of the reachable commits via the branch tips that are not
* {@code peeledWants}), stores them in {@code commitsByOldest}, and sets up * covered by a previous pack's bitmaps ({@code newWants}) and stores them
* bitmaps for each branch tip ({@code tipCommitBitmaps}). * in {@code newCommitsByOldest}. {@code newCommitsByOldest} is initialized
* {@code commitsByOldest} is initialized with an expected size of all * with an expected size of all commits, but may be smaller if some commits
* commits, but may be smaller if some commits are unreachable, in which * are unreachable and/or some commits are covered by a previous pack's
* case {@code commitStartPos} will contain a positive offset to the root * bitmaps. {@code commitStartPos} will contain a positive offset to either
* commit. * the root commit or the oldest commit not covered by previous bitmaps.
*/ */
private static final class CommitSelectionHelper implements Iterable<RevCommit> { private static final class CommitSelectionHelper implements Iterable<RevCommit> {
final Set<? extends ObjectId> peeledWants; final Set<? extends ObjectId> newWants;
final List<BitmapBuilderEntry> tipCommitBitmaps;
final List<RevCommit> newWantsByNewest;
final BitmapBuilder reusedCommitsBitmap; final BitmapBuilder reusedCommitsBitmap;
final Iterable<BitmapCommit> reusedCommits;
final RevCommit[] commitsByOldest;
final int commitStartPos;
CommitSelectionHelper(Set<? extends ObjectId> peeledWant, final List<BitmapCommit> reusedCommits;
final RevCommit[] newCommitsByOldest;
final int newCommitStartPos;
CommitSelectionHelper(Set<? extends ObjectId> newWants,
RevCommit[] commitsByOldest, int commitStartPos, RevCommit[] commitsByOldest, int commitStartPos,
List<BitmapBuilderEntry> bitmapEntries, List<RevCommit> newWantsByNewest,
BitmapBuilder reusedCommitsBitmap, BitmapBuilder reusedCommitsBitmap,
Iterable<BitmapCommit> reuse) { List<BitmapCommit> reuse) {
this.peeledWants = peeledWant; this.newWants = newWants;
this.commitsByOldest = commitsByOldest; this.newCommitsByOldest = commitsByOldest;
this.commitStartPos = commitStartPos; this.newCommitStartPos = commitStartPos;
this.tipCommitBitmaps = bitmapEntries; this.newWantsByNewest = newWantsByNewest;
this.reusedCommitsBitmap = reusedCommitsBitmap; this.reusedCommitsBitmap = reusedCommitsBitmap;
this.reusedCommits = reuse; this.reusedCommits = reuse;
} }
@ -594,16 +564,16 @@ class PackWriterBitmapPreparer {
// Member variables referenced by this iterator will have synthetic // Member variables referenced by this iterator will have synthetic
// accessors generated for them if they are made private. // accessors generated for them if they are made private.
return new Iterator<RevCommit>() { return new Iterator<RevCommit>() {
int pos = commitStartPos; int pos = newCommitStartPos;
@Override @Override
public boolean hasNext() { public boolean hasNext() {
return pos < commitsByOldest.length; return pos < newCommitsByOldest.length;
} }
@Override @Override
public RevCommit next() { public RevCommit next() {
return commitsByOldest[pos++]; return newCommitsByOldest[pos++];
} }
@Override @Override
@ -614,7 +584,7 @@ class PackWriterBitmapPreparer {
} }
int getCommitCount() { int getCommitCount() {
return commitsByOldest.length - commitStartPos; return newCommitsByOldest.length - newCommitStartPos;
} }
} }
} }

Write Preview
Loading…
Cancel
Save