Browse Source

DfsFsck: reduce memory usage during verifyIndex

Don't convert a lot of ObjectId to String stored in generic
java.util.HashSet.  This is a very expensive way to store objects.

Instead rely on "this" from the FsckPackParser to lookup information
about the objects in this pack file, which lets the verify code avoid
sorting the object list.

Use ObjectIdOwnerMap, which is the most efficient format JGit has
for storing lots of objects.

Change-Id: Ib68f93acb4d91b96d0a44c0612f704500d332ac1
stable-4.9
Shawn Pearce 7 years ago
parent
commit
e5db7c1f0e
  1. 22
      org.eclipse.jgit/src/org/eclipse/jgit/internal/fsck/FsckPackParser.java
  2. 2
      org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsFsck.java

22
org.eclipse.jgit/src/org/eclipse/jgit/internal/fsck/FsckPackParser.java

@ -49,7 +49,6 @@ import java.nio.channels.Channels;
import java.text.MessageFormat; import java.text.MessageFormat;
import java.util.Arrays; import java.util.Arrays;
import java.util.HashSet; import java.util.HashSet;
import java.util.List;
import java.util.Set; import java.util.Set;
import java.util.zip.CRC32; import java.util.zip.CRC32;
@ -65,6 +64,7 @@ import org.eclipse.jgit.internal.storage.file.PackIndex.MutableEntry;
import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.AnyObjectId;
import org.eclipse.jgit.lib.ObjectChecker; import org.eclipse.jgit.lib.ObjectChecker;
import org.eclipse.jgit.lib.ObjectDatabase; import org.eclipse.jgit.lib.ObjectDatabase;
import org.eclipse.jgit.lib.ObjectIdOwnerMap;
import org.eclipse.jgit.transport.PackParser; import org.eclipse.jgit.transport.PackParser;
import org.eclipse.jgit.transport.PackedObjectInfo; import org.eclipse.jgit.transport.PackedObjectInfo;
@ -265,18 +265,18 @@ public class FsckPackParser extends PackParser {
/** /**
* Verify the existing index file with all objects from the pack. * Verify the existing index file with all objects from the pack.
* *
* @param entries
* all the entries that are expected in the index file
* @param idx * @param idx
* index file associate with the pack * index file associate with the pack
* @throws CorruptPackIndexException * @throws CorruptPackIndexException
* when the index file is corrupt. * when the index file is corrupt.
*/ */
public void verifyIndex(List<PackedObjectInfo> entries, PackIndex idx) public void verifyIndex(PackIndex idx)
throws CorruptPackIndexException { throws CorruptPackIndexException {
Set<String> all = new HashSet<>(); ObjectIdOwnerMap<ObjFromPack> inPack = new ObjectIdOwnerMap<>();
for (PackedObjectInfo entry : entries) { for (int i = 0; i < getObjectCount(); i++) {
all.add(entry.getName()); PackedObjectInfo entry = getObject(i);
inPack.add(new ObjFromPack(entry));
long offset = idx.findOffset(entry); long offset = idx.findOffset(entry);
if (offset == -1) { if (offset == -1) {
throw new CorruptPackIndexException( throw new CorruptPackIndexException(
@ -305,7 +305,7 @@ public class FsckPackParser extends PackParser {
} }
for (MutableEntry entry : idx) { for (MutableEntry entry : idx) {
if (!all.contains(entry.name())) { if (!inPack.contains(entry.toObjectId())) {
throw new CorruptPackIndexException(MessageFormat.format( throw new CorruptPackIndexException(MessageFormat.format(
JGitText.get().unknownObjectInIndex, entry.name()), JGitText.get().unknownObjectInIndex, entry.name()),
ErrorType.UNKNOWN_OBJ); ErrorType.UNKNOWN_OBJ);
@ -323,4 +323,10 @@ public class FsckPackParser extends PackParser {
public void overwriteObjectCount(long expectedObjectCount) { public void overwriteObjectCount(long expectedObjectCount) {
this.expectedObjectCount = expectedObjectCount; this.expectedObjectCount = expectedObjectCount;
} }
static class ObjFromPack extends ObjectIdOwnerMap.Entry {
ObjFromPack(AnyObjectId id) {
super(id);
}
}
} }

2
org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsFsck.java

@ -128,7 +128,7 @@ public class DfsFsck {
fpp.parse(pm); fpp.parse(pm);
errors.getCorruptObjects().addAll(fpp.getCorruptObjects()); errors.getCorruptObjects().addAll(fpp.getCorruptObjects());
fpp.verifyIndex(fpp.getSortedObjectList(null), pack.getPackIndex(ctx)); fpp.verifyIndex(pack.getPackIndex(ctx));
} }
private void checkConnectivity(ProgressMonitor pm, FsckError errors) private void checkConnectivity(ProgressMonitor pm, FsckError errors)

Loading…
Cancel
Save