github
/
jgit
mirror of https://gitee.com/github-mirror-repo/jgit.git
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

Use a bucket sort for PackReverseIndex. 

Previously it took 1200ms to create a reverse index (sorted by offset).
Using a simple bucket sort algorithm, that time is reduced to 450ms.
The bucket index into the offset array is kept, in order to decrease
the binary search window.

Don't keep a copy of the offsets. Instead, use nth position
to lookup the offset in the PackIndex.

Change-Id: If51ab76752622e04a4430d9a14db95ad02f5329d
stable-3.1
Colby Ranger 12 years ago
parent
903fb9c739
commit
6cc532a43c
1 changed files with 87 additions and 86 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 157
      org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackReverseIndex.java

157
org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackReverseIndex.java
Unescape View File

@ -44,7 +44,6 @@
package org.eclipse.jgit.internal.storage.file;
import java.text.MessageFormat;
import java.util.Arrays;
import org.eclipse.jgit.errors.CorruptObjectException;
import org.eclipse.jgit.internal.JGitText;
@ -65,21 +64,22 @@ public class PackReverseIndex {
/** Index we were created from, and that has our ObjectId data. */
private final PackIndex index;
/**
* (offset31, truly) Offsets accommodating in 31 bits.
*/
private final int offsets32[];
/** The number of bytes per entry in the offsetIndex. */
private final long bucketSize;
/**
* Offsets not accommodating in 31 bits.
* An index into the nth mapping, where the value is the position after the
* the last index that contains the values of the bucket. For example given
* offset o (and bucket = o / bucketSize), the offset will be contained in
* the range nth[offsetIndex[bucket - 1]] inclusive to
* nth[offsetIndex[bucket]] exclusive.
*
* See {@link #binarySearch}
*/
private final long offsets64[];
/** Position of the corresponding {@link #offsets32} in {@link #index}. */
private final int nth32[];
private final int[] offsetIndex;
/** Position of the corresponding {@link #offsets64} in {@link #index}. */
private final int nth64[];
/** Mapping from indices in offset order to indices in SHA-1 order. */
private final int[] nth;
/**
* Create reverse index from straight/forward pack index, by indexing all
@ -92,38 +92,58 @@ public class PackReverseIndex {
index = packIndex;
final long cnt = index.getObjectCount();
final long n64 = index.getOffset64Count();
final long n32 = cnt - n64;
if (n32 > Integer.MAX_VALUE || n64 > Integer.MAX_VALUE
|| cnt > 0xffffffffL)
if (cnt + 1 > Integer.MAX_VALUE)
throw new IllegalArgumentException(
JGitText.get().hugeIndexesAreNotSupportedByJgitYet);
offsets32 = new int[(int) n32];
offsets64 = new long[(int) n64];
nth32 = new int[offsets32.length];
nth64 = new int[offsets64.length];
if (cnt == 0) {
bucketSize = Long.MAX_VALUE;
offsetIndex = new int[1];
nth = new int[0];
return;
}
final long[] offsetsBySha1 = new long[(int) cnt];
int i32 = 0;
int i64 = 0;
long maxOffset = 0;
int ith = 0;
for (final MutableEntry me : index) {
final long o = me.getOffset();
if (o <= Integer.MAX_VALUE)
offsets32[i32++] = (int) o;
else
offsets64[i64++] = o;
offsetsBySha1[ith++] = o;
if (o > maxOffset)
maxOffset = o;
}
Arrays.sort(offsets32);
Arrays.sort(offsets64);
bucketSize = maxOffset / cnt + 1;
int[] bucketIndex = new int[(int) cnt];
int[] bucketValues = new int[(int) cnt + 1];
for (int oi = 0; oi < offsetsBySha1.length; oi++) {
final long o = offsetsBySha1[oi];
final int bucket = (int) (o / bucketSize);
final int bucketValuesPos = oi + 1;
final int current = bucketIndex[bucket];
bucketIndex[bucket] = bucketValuesPos;
bucketValues[bucketValuesPos] = current;
}
int nth = 0;
for (final MutableEntry me : index) {
final long o = me.getOffset();
if (o <= Integer.MAX_VALUE)
nth32[Arrays.binarySearch(offsets32, (int) o)] = nth++;
else
nth64[Arrays.binarySearch(offsets64, o)] = nth++;
int nthByOffset = 0;
nth = new int[offsetsBySha1.length];
offsetIndex = bucketIndex; // Reuse the allocation
for (int bi = 0; bi < bucketIndex.length; bi++) {
final int start = nthByOffset;
// Insertion sort of the values in the bucket.
for (int vi = bucketIndex[bi]; vi > 0; vi = bucketValues[vi]) {
final int nthBySha1 = vi - 1;
final long o = offsetsBySha1[nthBySha1];
int insertion = nthByOffset++;
for (; start < insertion; insertion--) {
if (o > offsetsBySha1[nth[insertion - 1]])
break;
nth[insertion] = nth[insertion - 1];
}
nth[insertion] = nthBySha1;
}
offsetIndex[bi] = nthByOffset;
}
}
@ -136,17 +156,10 @@ public class PackReverseIndex {
* @return object id for this offset, or null if no object was found.
*/
public ObjectId findObject(final long offset) {
if (offset <= Integer.MAX_VALUE) {
final int i32 = Arrays.binarySearch(offsets32, (int) offset);
if (i32 < 0)
return null;
return index.getObjectId(nth32[i32]);
} else {
final int i64 = Arrays.binarySearch(offsets64, offset);
if (i64 < 0)
final int ith = binarySearch(offset);
if (ith < 0)
return null;
return index.getObjectId(nth64[i64]);
}
return index.getObjectId(nth[ith]);
}
/**
@ -166,52 +179,40 @@ public class PackReverseIndex {
*/
public long findNextOffset(final long offset, final long maxOffset)
throws CorruptObjectException {
if (offset <= Integer.MAX_VALUE) {
final int i32 = Arrays.binarySearch(offsets32, (int) offset);
if (i32 < 0)
final int ith = binarySearch(offset);
if (ith < 0)
throw new CorruptObjectException(
MessageFormat.format(
JGitText.get().cantFindObjectInReversePackIndexForTheSpecifiedOffset,
Long.valueOf(offset)));
if (i32 + 1 == offsets32.length) {
if (offsets64.length > 0)
return offsets64[0];
if (ith + 1 == nth.length)
return maxOffset;
return index.getOffset(nth[ith + 1]);
}
return offsets32[i32 + 1];
} else {
final int i64 = Arrays.binarySearch(offsets64, offset);
if (i64 < 0)
throw new CorruptObjectException(
MessageFormat.format(
JGitText.get().cantFindObjectInReversePackIndexForTheSpecifiedOffset,
Long.valueOf(offset)));
if (i64 + 1 == offsets64.length)
return maxOffset;
return offsets64[i64 + 1];
}
int findPostion(long offset) {
return binarySearch(offset);
}
int findPostion(long offset) {
if (offset <= Integer.MAX_VALUE) {
final int i32 = Arrays.binarySearch(offsets32, (int) offset);
if (i32 < 0)
return -1;
return i32;
} else {
final int i64 = Arrays.binarySearch(offsets64, offset);
if (i64 < 0)
return -1;
return nth32.length + i64;
private int binarySearch(final long offset) {
int bucket = (int) (offset / bucketSize);
int low = bucket == 0 ? 0 : offsetIndex[bucket - 1];
int high = offsetIndex[bucket];
while (low < high) {
final int mid = (low + high) >>> 1;
final long o = index.getOffset(nth[mid]);
if (offset < o)
high = mid;
else if (offset == o)
return mid;
else
low = mid + 1;
}
return -1;
}
ObjectId findObjectByPosition(int nthPosition) {
if (nthPosition < nth32.length)
return index.getObjectId(nth32[nthPosition]);
final int i64 = nthPosition - nth32.length;
return index.getObjectId(nth64[i64]);
return index.getObjectId(nth[nthPosition]);
}
}

Write Preview
Loading…
Cancel
Save