Browse Source

Enable writing pack indexes with bitmaps in the GC.

Update the dfs and file GC implementations to prepare and write
bitmaps on the packs that contain the full closure of the object
graph. Update the DfsPackDescription to include the index version.

Change-Id: I3f1421e9cd90fe93e7e2ef2b8179ae2f1ba819ed
stable-3.0
Colby Ranger 12 years ago
parent
commit
f82821728b
  1. 13
      org.eclipse.jgit/src/org/eclipse/jgit/storage/dfs/DfsGarbageCollector.java
  2. 1
      org.eclipse.jgit/src/org/eclipse/jgit/storage/dfs/DfsInserter.java
  3. 1
      org.eclipse.jgit/src/org/eclipse/jgit/storage/dfs/DfsPackCompactor.java
  4. 17
      org.eclipse.jgit/src/org/eclipse/jgit/storage/dfs/DfsPackDescription.java
  5. 96
      org.eclipse.jgit/src/org/eclipse/jgit/storage/file/GC.java
  6. 13
      org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java

13
org.eclipse.jgit/src/org/eclipse/jgit/storage/dfs/DfsGarbageCollector.java

@ -45,6 +45,7 @@ package org.eclipse.jgit.storage.dfs;
import static org.eclipse.jgit.storage.dfs.DfsObjDatabase.PackSource.GC; import static org.eclipse.jgit.storage.dfs.DfsObjDatabase.PackSource.GC;
import static org.eclipse.jgit.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE; import static org.eclipse.jgit.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE;
import static org.eclipse.jgit.storage.pack.PackExt.BITMAP_INDEX;
import static org.eclipse.jgit.storage.pack.PackExt.PACK; import static org.eclipse.jgit.storage.pack.PackExt.PACK;
import static org.eclipse.jgit.storage.pack.PackExt.INDEX; import static org.eclipse.jgit.storage.pack.PackExt.INDEX;
@ -329,10 +330,22 @@ public class DfsGarbageCollector {
CountingOutputStream cnt = new CountingOutputStream(out); CountingOutputStream cnt = new CountingOutputStream(out);
pw.writeIndex(cnt); pw.writeIndex(cnt);
pack.setFileSize(INDEX, cnt.getCount()); pack.setFileSize(INDEX, cnt.getCount());
pack.setIndexVersion(pw.getIndexVersion());
} finally { } finally {
out.close(); out.close();
} }
if (pw.prepareBitmapIndex(pm)) {
out = objdb.writeFile(pack, BITMAP_INDEX);
try {
CountingOutputStream cnt = new CountingOutputStream(out);
pw.writeIndex(cnt);
pack.setFileSize(BITMAP_INDEX, cnt.getCount());
} finally {
out.close();
}
}
final ObjectIdOwnerMap<ObjectIdOwnerMap.Entry> packedObjs = pw final ObjectIdOwnerMap<ObjectIdOwnerMap.Entry> packedObjs = pw
.getObjectSet(); .getObjectSet();
newPackObj.add(new PackWriter.ObjectIdSet() { newPackObj.add(new PackWriter.ObjectIdSet() {

1
org.eclipse.jgit/src/org/eclipse/jgit/storage/dfs/DfsInserter.java

@ -240,6 +240,7 @@ public class DfsInserter extends ObjectInserter {
PackIndex writePackIndex(DfsPackDescription pack, byte[] packHash, PackIndex writePackIndex(DfsPackDescription pack, byte[] packHash,
List<PackedObjectInfo> list) throws IOException { List<PackedObjectInfo> list) throws IOException {
pack.setIndexVersion(INDEX_VERSION);
pack.setObjectCount(list.size()); pack.setObjectCount(list.size());
// If there are less than 58,000 objects, the entire index fits in under // If there are less than 58,000 objects, the entire index fits in under

1
org.eclipse.jgit/src/org/eclipse/jgit/storage/dfs/DfsPackCompactor.java

@ -303,6 +303,7 @@ public class DfsPackCompactor {
CountingOutputStream cnt = new CountingOutputStream(out); CountingOutputStream cnt = new CountingOutputStream(out);
pw.writeIndex(cnt); pw.writeIndex(cnt);
pack.setFileSize(INDEX, cnt.getCount()); pack.setFileSize(INDEX, cnt.getCount());
pack.setIndexVersion(pw.getIndexVersion());
} finally { } finally {
out.close(); out.close();
} }

17
org.eclipse.jgit/src/org/eclipse/jgit/storage/dfs/DfsPackDescription.java

@ -81,6 +81,8 @@ public class DfsPackDescription implements Comparable<DfsPackDescription> {
private int extensions; private int extensions;
private int indexVersion;
/** /**
* Initialize a description by pack name and repository. * Initialize a description by pack name and repository.
* <p> * <p>
@ -259,6 +261,21 @@ public class DfsPackDescription implements Comparable<DfsPackDescription> {
return this; return this;
} }
/** @return the version of the index file written. */
public int getIndexVersion() {
return indexVersion;
}
/**
* @param version
* the version of the index file written.
* @return {@code this}
*/
public DfsPackDescription setIndexVersion(int version) {
indexVersion = version;
return this;
}
@Override @Override
public int hashCode() { public int hashCode() {
return packName.hashCode(); return packName.hashCode();

96
org.eclipse.jgit/src/org/eclipse/jgit/storage/file/GC.java

@ -43,6 +43,9 @@
*/ */
package org.eclipse.jgit.storage.file; package org.eclipse.jgit.storage.file;
import static org.eclipse.jgit.storage.pack.PackExt.BITMAP_INDEX;
import static org.eclipse.jgit.storage.pack.PackExt.INDEX;
import java.io.File; import java.io.File;
import java.io.FileOutputStream; import java.io.FileOutputStream;
import java.io.IOException; import java.io.IOException;
@ -54,6 +57,7 @@ import java.text.ParseException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.Collections; import java.util.Collections;
import java.util.Comparator;
import java.util.Date; import java.util.Date;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
@ -63,6 +67,7 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Map.Entry; import java.util.Map.Entry;
import java.util.Set; import java.util.Set;
import java.util.TreeMap;
import org.eclipse.jgit.dircache.DirCacheIterator; import org.eclipse.jgit.dircache.DirCacheIterator;
import org.eclipse.jgit.errors.CorruptObjectException; import org.eclipse.jgit.errors.CorruptObjectException;
@ -83,6 +88,7 @@ import org.eclipse.jgit.lib.RefDatabase;
import org.eclipse.jgit.revwalk.ObjectWalk; import org.eclipse.jgit.revwalk.ObjectWalk;
import org.eclipse.jgit.revwalk.RevObject; import org.eclipse.jgit.revwalk.RevObject;
import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.revwalk.RevWalk;
import org.eclipse.jgit.storage.pack.PackExt;
import org.eclipse.jgit.storage.pack.PackWriter; import org.eclipse.jgit.storage.pack.PackWriter;
import org.eclipse.jgit.storage.pack.PackWriter.ObjectIdSet; import org.eclipse.jgit.storage.pack.PackWriter.ObjectIdSet;
import org.eclipse.jgit.treewalk.TreeWalk; import org.eclipse.jgit.treewalk.TreeWalk;
@ -192,8 +198,10 @@ public class GC {
if (!oldPack.shouldBeKept()) { if (!oldPack.shouldBeKept()) {
oldPack.close(); oldPack.close();
FileUtils.delete(nameFor(oldName, ".pack"), deleteOptions); //$NON-NLS-1$ for (PackExt ext : PackExt.values()) {
FileUtils.delete(nameFor(oldName, ".idx"), deleteOptions); //$NON-NLS-1$ File f = nameFor(oldName, "." + ext.getExtension()); //$NON-NLS-1$
FileUtils.delete(f, deleteOptions);
}
} }
} }
// close the complete object database. Thats my only chance to force // close the complete object database. Thats my only chance to force
@ -636,7 +644,22 @@ public class GC {
Set<? extends ObjectId> have, Set<ObjectId> tagTargets, Set<? extends ObjectId> have, Set<ObjectId> tagTargets,
List<ObjectIdSet> excludeObjects) throws IOException { List<ObjectIdSet> excludeObjects) throws IOException {
File tmpPack = null; File tmpPack = null;
File tmpIdx = null; Map<PackExt, File> tmpExts = new TreeMap<PackExt, File>(
new Comparator<PackExt>() {
public int compare(PackExt o1, PackExt o2) {
// INDEX entries must be returned last, so the pack
// scanner does pick up the new pack until all the
// PackExt entries have been written.
if (o1 == o2)
return 0;
if (o1 == PackExt.INDEX)
return 1;
if (o2 == PackExt.INDEX)
return -1;
return Integer.signum(o1.hashCode() - o2.hashCode());
}
});
PackWriter pw = new PackWriter(repo); PackWriter pw = new PackWriter(repo);
try { try {
// prepare the PackWriter // prepare the PackWriter
@ -655,9 +678,10 @@ public class GC {
String id = pw.computeName().getName(); String id = pw.computeName().getName();
File packdir = new File(repo.getObjectsDirectory(), "pack"); //$NON-NLS-1$ File packdir = new File(repo.getObjectsDirectory(), "pack"); //$NON-NLS-1$
tmpPack = File.createTempFile("gc_", ".pack_tmp", packdir); //$NON-NLS-1$ //$NON-NLS-2$ tmpPack = File.createTempFile("gc_", ".pack_tmp", packdir); //$NON-NLS-1$ //$NON-NLS-2$
tmpIdx = new File(packdir, tmpPack.getName().substring(0, final String tmpBase = tmpPack.getName()
tmpPack.getName().lastIndexOf('.')) .substring(0, tmpPack.getName().lastIndexOf('.'));
+ ".idx_tmp"); //$NON-NLS-1$ File tmpIdx = new File(packdir, tmpBase + ".idx_tmp"); //$NON-NLS-1$
tmpExts.put(INDEX, tmpIdx);
if (!tmpIdx.createNewFile()) if (!tmpIdx.createNewFile())
throw new IOException(MessageFormat.format( throw new IOException(MessageFormat.format(
@ -687,38 +711,70 @@ public class GC {
idxChannel.close(); idxChannel.close();
} }
if (pw.prepareBitmapIndex(pm)) {
File tmpBitmapIdx = new File(packdir, tmpBase + ".bitmap_tmp"); //$NON-NLS-1$
tmpExts.put(BITMAP_INDEX, tmpBitmapIdx);
if (!tmpBitmapIdx.createNewFile())
throw new IOException(MessageFormat.format(
JGitText.get().cannotCreateIndexfile,
tmpBitmapIdx.getPath()));
idxChannel = new FileOutputStream(tmpBitmapIdx).getChannel();
idxStream = Channels.newOutputStream(idxChannel);
try {
pw.writeBitmapIndex(idxStream);
} finally {
idxChannel.force(true);
idxStream.close();
idxChannel.close();
}
}
// rename the temporary files to real files // rename the temporary files to real files
File realPack = nameFor(id, ".pack"); //$NON-NLS-1$ File realPack = nameFor(id, ".pack"); //$NON-NLS-1$
tmpPack.setReadOnly(); tmpPack.setReadOnly();
File realIdx = nameFor(id, ".idx"); //$NON-NLS-1$
realIdx.setReadOnly();
boolean delete = true; boolean delete = true;
try { try {
if (!tmpPack.renameTo(realPack)) if (!tmpPack.renameTo(realPack))
return null; return null;
delete = false; delete = false;
if (!tmpIdx.renameTo(realIdx)) { for (Map.Entry<PackExt, File> tmpEntry : tmpExts.entrySet()) {
File newIdx = new File(realIdx.getParentFile(), File tmpExt = tmpEntry.getValue();
realIdx.getName() + ".new"); //$NON-NLS-1$ tmpExt.setReadOnly();
if (!tmpIdx.renameTo(newIdx))
newIdx = tmpIdx; File realExt = nameFor(
id, "." + tmpEntry.getKey().getExtension()); //$NON-NLS-1$
if (!tmpExt.renameTo(realExt)) {
File newExt = new File(realExt.getParentFile(),
realExt.getName() + ".new"); //$NON-NLS-1$
if (!tmpExt.renameTo(newExt))
newExt = tmpExt;
throw new IOException(MessageFormat.format( throw new IOException(MessageFormat.format(
JGitText.get().panicCantRenameIndexFile, newIdx, JGitText.get().panicCantRenameIndexFile, newExt,
realIdx)); realExt));
}
} }
} finally { } finally {
if (delete && tmpPack.exists()) if (delete) {
if (tmpPack.exists())
tmpPack.delete(); tmpPack.delete();
if (delete && tmpIdx.exists()) for (File tmpExt : tmpExts.values()) {
tmpIdx.delete(); if (tmpExt.exists())
tmpExt.delete();
}
}
} }
return repo.getObjectDatabase().openPack(realPack); return repo.getObjectDatabase().openPack(realPack);
} finally { } finally {
pw.release(); pw.release();
if (tmpPack != null && tmpPack.exists()) if (tmpPack != null && tmpPack.exists())
tmpPack.delete(); tmpPack.delete();
if (tmpIdx != null && tmpIdx.exists()) for (File tmpExt : tmpExts.values()) {
tmpIdx.delete(); if (tmpExt.exists())
tmpExt.delete();
}
} }
} }

13
org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java

@ -866,7 +866,7 @@ public class PackWriter {
* Create a bitmap index file to match the pack file just written. * Create a bitmap index file to match the pack file just written.
* <p> * <p>
* This method can only be invoked after * This method can only be invoked after
* {@link #prepareBitmapIndex(ProgressMonitor, Set)} has been invoked and * {@link #prepareBitmapIndex(ProgressMonitor)} has been invoked and
* completed successfully. Writing a corresponding bitmap index is an * completed successfully. Writing a corresponding bitmap index is an
* optional feature that not all pack users may require. * optional feature that not all pack users may require.
* *
@ -2051,16 +2051,11 @@ public class PackWriter {
* *
* @param pm * @param pm
* progress monitor to report bitmap building work. * progress monitor to report bitmap building work.
* @param want
* collection of objects to be marked as interesting (start
* points of graph traversal).
* @return whether a bitmap index may be written. * @return whether a bitmap index may be written.
* @throws IOException * @throws IOException
* when some I/O problem occur during reading objects. * when some I/O problem occur during reading objects.
*/ */
public boolean prepareBitmapIndex( public boolean prepareBitmapIndex(ProgressMonitor pm) throws IOException {
ProgressMonitor pm, Set<? extends ObjectId> want)
throws IOException {
if (!canBuildBitmaps || getObjectCount() > Integer.MAX_VALUE if (!canBuildBitmaps || getObjectCount() > Integer.MAX_VALUE
|| !cachedPacks.isEmpty()) || !cachedPacks.isEmpty())
return false; return false;
@ -2069,8 +2064,8 @@ public class PackWriter {
pm = NullProgressMonitor.INSTANCE; pm = NullProgressMonitor.INSTANCE;
writeBitmaps = new PackBitmapIndexBuilder(sortByName()); writeBitmaps = new PackBitmapIndexBuilder(sortByName());
PackWriterBitmapPreparer bitmapPreparer = PackWriterBitmapPreparer bitmapPreparer = new PackWriterBitmapPreparer(
new PackWriterBitmapPreparer(reader, writeBitmaps, pm, want); reader, writeBitmaps, pm, stats.interestingObjects);
int numCommits = objectsLists[Constants.OBJ_COMMIT].size(); int numCommits = objectsLists[Constants.OBJ_COMMIT].size();
Collection<PackWriterBitmapPreparer.BitmapCommit> selectedCommits = Collection<PackWriterBitmapPreparer.BitmapCommit> selectedCommits =

Loading…
Cancel
Save