github
/
jgit
mirror of https://gitee.com/github-mirror-repo/jgit.git
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

Merge changes from topic 'dfs-reftable' 

* changes:
  dfs: write reftable from DfsGarbageCollector
  dfs: compact reftables during DfsPackCompactor
stable-4.9
Shawn Pearce 7 years ago committed by Gerrit Code Review @ Eclipse.org
parent
a0558b7094 d13dfac9dc
commit
d26309c4d9
4 changed files with 542 additions and 76 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 190
      org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollectorTest.java
  2. 163
      org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java
  3. 244
      org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java
  4. 21
      org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableCompactor.java

190
org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollectorTest.java
Unescape View File

@ -5,6 +5,7 @@ import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.INSERT;
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE;
import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;
import static org.eclipse.jgit.internal.storage.pack.PackExt.REFTABLE;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
@ -13,19 +14,29 @@ import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.concurrent.TimeUnit;
import org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource;
import org.eclipse.jgit.internal.storage.reftable.RefCursor;
import org.eclipse.jgit.internal.storage.reftable.ReftableConfig;
import org.eclipse.jgit.internal.storage.reftable.ReftableReader;
import org.eclipse.jgit.internal.storage.reftable.ReftableWriter;
import org.eclipse.jgit.junit.MockSystemReader;
import org.eclipse.jgit.junit.TestRepository;
import org.eclipse.jgit.lib.AnyObjectId;
import org.eclipse.jgit.lib.BatchRefUpdate;
import org.eclipse.jgit.lib.NullProgressMonitor;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectIdRef;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.revwalk.RevBlob;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.RevWalk;
import org.eclipse.jgit.storage.pack.PackConfig;
import org.eclipse.jgit.transport.ReceiveCommand;
import org.eclipse.jgit.util.SystemReader;
import org.junit.After;
import org.junit.Before;
@ -653,6 +664,185 @@ public class DfsGarbageCollectorTest {
assertEquals(2, odb.getPacks().length);
}
@SuppressWarnings("boxing")
@Test
public void producesNewReftable() throws Exception {
String master = "refs/heads/master";
RevCommit commit0 = commit().message("0").create();
RevCommit commit1 = commit().message("1").parent(commit0).create();
BatchRefUpdate bru = git.getRepository().getRefDatabase()
.newBatchUpdate();
bru.addCommand(new ReceiveCommand(ObjectId.zeroId(), commit1, master));
for (int i = 1; i <= 5100; i++) {
bru.addCommand(new ReceiveCommand(ObjectId.zeroId(), commit0,
String.format("refs/pulls/%04d", i)));
}
try (RevWalk rw = new RevWalk(git.getRepository())) {
bru.execute(rw, NullProgressMonitor.INSTANCE);
}
DfsGarbageCollector gc = new DfsGarbageCollector(repo);
gc.setReftableConfig(new ReftableConfig());
run(gc);
// Single GC pack present with all objects.
assertEquals(1, odb.getPacks().length);
DfsPackFile pack = odb.getPacks()[0];
DfsPackDescription desc = pack.getPackDescription();
assertEquals(GC, desc.getPackSource());
assertTrue("commit0 in pack", isObjectInPack(commit0, pack));
assertTrue("commit1 in pack", isObjectInPack(commit1, pack));
// Sibling REFTABLE is also present.
assertTrue(desc.hasFileExt(REFTABLE));
ReftableWriter.Stats stats = desc.getReftableStats();
assertNotNull(stats);
assertTrue(stats.totalBytes() > 0);
assertEquals(5101, stats.refCount());
assertEquals(1, stats.minUpdateIndex());
assertEquals(1, stats.maxUpdateIndex());
DfsReftable table = new DfsReftable(DfsBlockCache.getInstance(), desc);
try (DfsReader ctx = odb.newReader();
ReftableReader rr = table.open(ctx);
RefCursor rc = rr.seekRef("refs/pulls/5100")) {
assertTrue(rc.next());
assertEquals(commit0, rc.getRef().getObjectId());
assertFalse(rc.next());
}
}
@Test
public void leavesNonGcReftablesIfNotConfigured() throws Exception {
String master = "refs/heads/master";
RevCommit commit0 = commit().message("0").create();
RevCommit commit1 = commit().message("1").parent(commit0).create();
git.update(master, commit1);
DfsPackDescription t1 = odb.newPack(INSERT);
try (DfsOutputStream out = odb.writeFile(t1, REFTABLE)) {
out.write("ignored".getBytes(StandardCharsets.UTF_8));
t1.addFileExt(REFTABLE);
}
odb.commitPack(Collections.singleton(t1), null);
DfsGarbageCollector gc = new DfsGarbageCollector(repo);
gc.setReftableConfig(null);
run(gc);
// Single GC pack present with all objects.
assertEquals(1, odb.getPacks().length);
DfsPackFile pack = odb.getPacks()[0];
DfsPackDescription desc = pack.getPackDescription();
assertEquals(GC, desc.getPackSource());
assertTrue("commit0 in pack", isObjectInPack(commit0, pack));
assertTrue("commit1 in pack", isObjectInPack(commit1, pack));
// Only INSERT REFTABLE above is present.
DfsReftable[] tables = odb.getReftables();
assertEquals(1, tables.length);
assertEquals(t1, tables[0].getPackDescription());
}
@Test
public void prunesNonGcReftables() throws Exception {
String master = "refs/heads/master";
RevCommit commit0 = commit().message("0").create();
RevCommit commit1 = commit().message("1").parent(commit0).create();
git.update(master, commit1);
DfsPackDescription t1 = odb.newPack(INSERT);
try (DfsOutputStream out = odb.writeFile(t1, REFTABLE)) {
out.write("ignored".getBytes(StandardCharsets.UTF_8));
t1.addFileExt(REFTABLE);
}
odb.commitPack(Collections.singleton(t1), null);
odb.clearCache();
DfsGarbageCollector gc = new DfsGarbageCollector(repo);
gc.setReftableConfig(new ReftableConfig());
run(gc);
// Single GC pack present with all objects.
assertEquals(1, odb.getPacks().length);
DfsPackFile pack = odb.getPacks()[0];
DfsPackDescription desc = pack.getPackDescription();
assertEquals(GC, desc.getPackSource());
assertTrue("commit0 in pack", isObjectInPack(commit0, pack));
assertTrue("commit1 in pack", isObjectInPack(commit1, pack));
// Only sibling GC REFTABLE is present.
DfsReftable[] tables = odb.getReftables();
assertEquals(1, tables.length);
assertEquals(desc, tables[0].getPackDescription());
assertTrue(desc.hasFileExt(REFTABLE));
}
@Test
public void compactsReftables() throws Exception {
String master = "refs/heads/master";
RevCommit commit0 = commit().message("0").create();
RevCommit commit1 = commit().message("1").parent(commit0).create();
git.update(master, commit1);
DfsGarbageCollector gc = new DfsGarbageCollector(repo);
gc.setReftableConfig(new ReftableConfig());
run(gc);
DfsPackDescription t1 = odb.newPack(INSERT);
Ref next = new ObjectIdRef.PeeledNonTag(Ref.Storage.LOOSE,
"refs/heads/next", commit0.copy());
try (DfsOutputStream out = odb.writeFile(t1, REFTABLE)) {
ReftableWriter w = new ReftableWriter();
w.setMinUpdateIndex(42);
w.setMaxUpdateIndex(42);
w.begin(out);
w.sortAndWriteRefs(Collections.singleton(next));
w.finish();
t1.addFileExt(REFTABLE);
t1.setReftableStats(w.getStats());
}
odb.commitPack(Collections.singleton(t1), null);
gc = new DfsGarbageCollector(repo);
gc.setReftableConfig(new ReftableConfig());
run(gc);
// Single GC pack present with all objects.
assertEquals(1, odb.getPacks().length);
DfsPackFile pack = odb.getPacks()[0];
DfsPackDescription desc = pack.getPackDescription();
assertEquals(GC, desc.getPackSource());
assertTrue("commit0 in pack", isObjectInPack(commit0, pack));
assertTrue("commit1 in pack", isObjectInPack(commit1, pack));
// Only sibling GC REFTABLE is present.
DfsReftable[] tables = odb.getReftables();
assertEquals(1, tables.length);
assertEquals(desc, tables[0].getPackDescription());
assertTrue(desc.hasFileExt(REFTABLE));
// GC reftable contains the compaction.
DfsReftable table = new DfsReftable(DfsBlockCache.getInstance(), desc);
try (DfsReader ctx = odb.newReader();
ReftableReader rr = table.open(ctx);
RefCursor rc = rr.allRefs()) {
assertEquals(1, rr.minUpdateIndex());
assertEquals(42, rr.maxUpdateIndex());
assertTrue(rc.next());
assertEquals(master, rc.getRef().getName());
assertEquals(commit1, rc.getRef().getObjectId());
assertTrue(rc.next());
assertEquals(next.getName(), rc.getRef().getName());
assertEquals(commit0, rc.getRef().getObjectId());
assertFalse(rc.next());
}
}
private TestRepository<InMemoryRepository>.CommitBuilder commit() {
return git.commit();
}

163
org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java
Unescape View File

@ -50,13 +50,16 @@ import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.INSERT;
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.RECEIVE;
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE;
import static org.eclipse.jgit.internal.storage.dfs.DfsPackCompactor.configureReftable;
import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX;
import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;
import static org.eclipse.jgit.internal.storage.pack.PackExt.REFTABLE;
import static org.eclipse.jgit.internal.storage.pack.PackWriter.NONE;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Collection;
import java.util.EnumSet;
@ -72,6 +75,9 @@ import org.eclipse.jgit.internal.storage.file.PackIndex;
import org.eclipse.jgit.internal.storage.file.PackReverseIndex;
import org.eclipse.jgit.internal.storage.pack.PackExt;
import org.eclipse.jgit.internal.storage.pack.PackWriter;
import org.eclipse.jgit.internal.storage.reftable.ReftableCompactor;
import org.eclipse.jgit.internal.storage.reftable.ReftableConfig;
import org.eclipse.jgit.internal.storage.reftable.ReftableWriter;
import org.eclipse.jgit.internal.storage.reftree.RefTreeNames;
import org.eclipse.jgit.lib.AnyObjectId;
import org.eclipse.jgit.lib.Constants;
@ -94,14 +100,15 @@ public class DfsGarbageCollector {
private final DfsObjDatabase objdb;
private final List<DfsPackDescription> newPackDesc;
private final List<PackStatistics> newPackStats;
private final List<ObjectIdSet> newPackObj;
private DfsReader ctx;
private PackConfig packConfig;
private ReftableConfig reftableConfig;
private long reftableInitialMinUpdateIndex = 1;
private long reftableInitialMaxUpdateIndex = 1;
// See packIsCoalesceableGarbage(), below, for how these two variables
// interact.
@ -110,8 +117,10 @@ public class DfsGarbageCollector {
private long startTimeMillis;
private List<DfsPackFile> packsBefore;
private List<DfsReftable> reftablesBefore;
private List<DfsPackFile> expiredGarbagePacks;
private Collection<Ref> refsBefore;
private Set<ObjectId> allHeadsAndTags;
private Set<ObjectId> allTags;
private Set<ObjectId> nonHeads;
@ -151,6 +160,57 @@ public class DfsGarbageCollector {
return this;
}
/**
* @param cfg
* configuration to write a reftable. Reftable writing is
* disabled (default) when {@code cfg} is {@code null}.
* @return {@code this}
*/
public DfsGarbageCollector setReftableConfig(ReftableConfig cfg) {
reftableConfig = cfg;
return this;
}
/**
* Set minUpdateIndex for the initial reftable created during conversion.
* <p>
* <b>Warning:</b> A setting {@code != 1} <b>disables cache refreshes</b>
* normally performed at the start of {@link #pack(ProgressMonitor)}.
* Callers must ensure the reference cache is current and will have been
* read before the pack list.
*
* @param u
* minUpdateIndex for the initial reftable created by scanning
* {@link DfsRefDatabase#getRefs(String)}. Ignored unless caller
* has also set {@link #setReftableConfig(ReftableConfig)}.
* Defaults to {@code 1}. Must be {@code u >= 0}.
* @return {@code this}
*/
public DfsGarbageCollector setReftableInitialMinUpdateIndex(long u) {
reftableInitialMinUpdateIndex = Math.max(u, 0);
return this;
}
/**
* Set maxUpdateIndex for the initial reftable created during conversion.
* <p>
* <b>Warning:</b> A setting {@code != 1} <b>disables cache refreshes</b>
* normally performed at the start of {@link #pack(ProgressMonitor)}.
* Callers must ensure the reference cache is current and will have been
* read before the pack list.
*
* @param u
* maxUpdateIndex for the initial reftable created by scanning
* {@link DfsRefDatabase#getRefs(String)}. Ignored unless caller
* has also set {@link #setReftableConfig(ReftableConfig)}.
* Defaults to {@code 1}. Must be {@code u >= 0}.
* @return {@code this}
*/
public DfsGarbageCollector setReftableInitialMaxUpdateIndex(long u) {
reftableInitialMaxUpdateIndex = Math.max(0, u);
return this;
}
/** @return garbage packs smaller than this size will be repacked. */
public long getCoalesceGarbageLimit() {
return coalesceGarbageLimit;
@ -240,8 +300,9 @@ public class DfsGarbageCollector {
refdb.refresh();
objdb.clearCache();
Collection<Ref> refsBefore = getAllRefs();
refsBefore = getAllRefs();
readPacksBefore();
readReftablesBefore();
Set<ObjectId> allHeads = new HashSet<>();
allHeadsAndTags = new HashSet<>();
@ -333,6 +394,11 @@ public class DfsGarbageCollector {
}
}
private void readReftablesBefore() throws IOException {
DfsReftable[] tables = objdb.getReftables();
reftablesBefore = new ArrayList<>(Arrays.asList(tables));
}
private boolean packIsExpiredGarbage(DfsPackDescription d, long now) {
// Consider the garbage pack as expired when it's older than
// garbagePackTtl. This check gives concurrent inserter threads
@ -407,7 +473,7 @@ public class DfsGarbageCollector {
}
/** @return all of the source packs that fed into this compaction. */
public List<DfsPackDescription> getSourcePacks() {
public Set<DfsPackDescription> getSourcePacks() {
return toPrune();
}
@ -421,28 +487,37 @@ public class DfsGarbageCollector {
return newPackStats;
}
private List<DfsPackDescription> toPrune() {
int cnt = packsBefore.size();
List<DfsPackDescription> all = new ArrayList<>(cnt);
private Set<DfsPackDescription> toPrune() {
Set<DfsPackDescription> toPrune = new HashSet<>();
for (DfsPackFile pack : packsBefore) {
all.add(pack.getPackDescription());
toPrune.add(pack.getPackDescription());
}
if (reftableConfig != null) {
for (DfsReftable table : reftablesBefore) {
toPrune.add(table.getPackDescription());
}
}
for (DfsPackFile pack : expiredGarbagePacks) {
all.add(pack.getPackDescription());
toPrune.add(pack.getPackDescription());
}
return all;
return toPrune;
}
private void packHeads(ProgressMonitor pm) throws IOException {
if (allHeadsAndTags.isEmpty())
if (allHeadsAndTags.isEmpty()) {
writeReftable();
return;
}
try (PackWriter pw = newPackWriter()) {
pw.setTagTargets(tagTargets);
pw.preparePack(pm, allHeadsAndTags, NONE, NONE, allTags);
if (0 < pw.getObjectCount())
writePack(GC, pw, pm,
estimateGcPackSize(INSERT, RECEIVE, COMPACT, GC));
if (0 < pw.getObjectCount()) {
long estSize = estimateGcPackSize(INSERT, RECEIVE, COMPACT, GC);
writePack(GC, pw, pm, estSize);
} else {
writeReftable();
}
}
}
@ -560,6 +635,10 @@ public class DfsGarbageCollector {
estimatedPackSize);
newPackDesc.add(pack);
if (source == GC && reftableConfig != null) {
writeReftable(pack);
}
try (DfsOutputStream out = objdb.writeFile(pack, PACK)) {
pw.writePack(pm, pm, out);
pack.addFileExt(PACK);
@ -592,4 +671,60 @@ public class DfsGarbageCollector {
newPackObj.add(pw.getObjectSet());
return pack;
}
private void writeReftable() throws IOException {
if (reftableConfig != null) {
DfsPackDescription pack = objdb.newPack(GC);
newPackDesc.add(pack);
writeReftable(pack);
}
}
private void writeReftable(DfsPackDescription pack) throws IOException {
if (!hasGcReftable()) {
writeReftable(pack, refsBefore);
return;
}
try (ReftableStack stack = ReftableStack.open(ctx, reftablesBefore)) {
ReftableCompactor compact = new ReftableCompactor();
compact.addAll(stack.readers());
compact.setIncludeDeletes(false);
compactReftable(pack, compact);
}
}
private boolean hasGcReftable() {
for (DfsReftable table : reftablesBefore) {
if (table.getPackDescription().getPackSource() == GC) {
return true;
}
}
return false;
}
private void writeReftable(DfsPackDescription pack, Collection<Ref> refs)
throws IOException {
try (DfsOutputStream out = objdb.writeFile(pack, REFTABLE)) {
ReftableConfig cfg = configureReftable(reftableConfig, out);
ReftableWriter writer = new ReftableWriter(cfg)
.setMinUpdateIndex(reftableInitialMinUpdateIndex)
.setMaxUpdateIndex(reftableInitialMaxUpdateIndex)
.begin(out)
.sortAndWriteRefs(refs)
.finish();
pack.addFileExt(REFTABLE);
pack.setReftableStats(writer.getStats());
}
}
private void compactReftable(DfsPackDescription pack,
ReftableCompactor compact) throws IOException {
try (DfsOutputStream out = objdb.writeFile(pack, REFTABLE)) {
compact.setConfig(configureReftable(reftableConfig, out));
compact.compact(out);
pack.addFileExt(REFTABLE);
pack.setReftableStats(compact.getStats());
}
}
}

244
org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java
Unescape View File

@ -44,21 +44,29 @@
package org.eclipse.jgit.internal.storage.dfs;
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.COMPACT;
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC;
import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;
import static org.eclipse.jgit.internal.storage.pack.PackExt.REFTABLE;
import static org.eclipse.jgit.internal.storage.pack.StoredObjectRepresentation.PACK_DELTA;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import org.eclipse.jgit.errors.IncorrectObjectTypeException;
import org.eclipse.jgit.internal.JGitText;
import org.eclipse.jgit.internal.storage.file.PackIndex;
import org.eclipse.jgit.internal.storage.file.PackReverseIndex;
import org.eclipse.jgit.internal.storage.pack.PackWriter;
import org.eclipse.jgit.internal.storage.reftable.ReftableCompactor;
import org.eclipse.jgit.internal.storage.reftable.ReftableConfig;
import org.eclipse.jgit.lib.AnyObjectId;
import org.eclipse.jgit.lib.NullProgressMonitor;
import org.eclipse.jgit.lib.ObjectId;
@ -89,16 +97,15 @@ import org.eclipse.jgit.util.io.CountingOutputStream;
*/
public class DfsPackCompactor {
private final DfsRepository repo;
private final List<DfsPackFile> srcPacks;
private final List<DfsReftable> srcReftables;
private final List<ObjectIdSet> exclude;
private final List<DfsPackDescription> newPacks;
private final List<PackStatistics> newStats;
private PackStatistics newStats;
private DfsPackDescription outDesc;
private int autoAddSize;
private ReftableConfig reftableConfig;
private RevWalk rw;
private RevFlag added;
@ -114,9 +121,19 @@ public class DfsPackCompactor {
repo = repository;
autoAddSize = 5 * 1024 * 1024; // 5 MiB
srcPacks = new ArrayList<>();
srcReftables = new ArrayList<>();
exclude = new ArrayList<>(4);
newPacks = new ArrayList<>(1);
newStats = new ArrayList<>(1);
}
/**
* @param cfg
* configuration to write a reftable. Reftable compacting is
* disabled (default) when {@code cfg} is {@code null}.
* @return {@code this}
*/
public DfsPackCompactor setReftableConfig(ReftableConfig cfg) {
reftableConfig = cfg;
return this;
}
/**
@ -137,7 +154,19 @@ public class DfsPackCompactor {
}
/**
* Automatically select packs to be included, and add them.
* Add a reftable to be compacted.
*
* @param table
* a reftable to combine.
* @return {@code this}
*/
public DfsPackCompactor add(DfsReftable table) {
srcReftables.add(table);
return this;
}
/**
* Automatically select pack and reftables to be included, and add them.
* <p>
* Packs are selected based on size, smaller packs get included while bigger
* ones are omitted.
@ -155,6 +184,16 @@ public class DfsPackCompactor {
else
exclude(pack);
}
if (reftableConfig != null) {
for (DfsReftable table : objdb.getReftables()) {
DfsPackDescription d = table.getPackDescription();
if (d.getPackSource() != GC
&& d.getFileSize(REFTABLE) < autoAddSize) {
add(table);
}
}
}
return this;
}
@ -197,58 +236,68 @@ public class DfsPackCompactor {
* the packs cannot be compacted.
*/
public void compact(ProgressMonitor pm) throws IOException {
if (pm == null)
if (pm == null) {
pm = NullProgressMonitor.INSTANCE;
}
DfsObjDatabase objdb = repo.getObjectDatabase();
try (DfsReader ctx = objdb.newReader()) {
PackConfig pc = new PackConfig(repo);
pc.setIndexVersion(2);
pc.setDeltaCompress(false);
pc.setReuseDeltas(true);
pc.setReuseObjects(true);
if (reftableConfig != null && !srcReftables.isEmpty()) {
compactReftables(ctx);
}
compactPacks(ctx, pm);
List<DfsPackDescription> commit = getNewPacks();
Collection<DfsPackDescription> remove = toPrune();
if (!commit.isEmpty() || !remove.isEmpty()) {
objdb.commitPack(commit, remove);
}
} finally {
rw = null;
}
}
PackWriter pw = new PackWriter(pc, ctx);
private void compactPacks(DfsReader ctx, ProgressMonitor pm)
throws IOException, IncorrectObjectTypeException {
DfsObjDatabase objdb = repo.getObjectDatabase();
PackConfig pc = new PackConfig(repo);
pc.setIndexVersion(2);
pc.setDeltaCompress(false);
pc.setReuseDeltas(true);
pc.setReuseObjects(true);
PackWriter pw = new PackWriter(pc, ctx);
try {
pw.setDeltaBaseAsOffset(true);
pw.setReuseDeltaCommits(false);
addObjectsToPack(pw, ctx, pm);
if (pw.getObjectCount() == 0) {
return;
}
boolean rollback = true;
initOutDesc(objdb);
try {
pw.setDeltaBaseAsOffset(true);
pw.setReuseDeltaCommits(false);
addObjectsToPack(pw, ctx, pm);
if (pw.getObjectCount() == 0) {
List<DfsPackDescription> remove = toPrune();
if (remove.size() > 0)
objdb.commitPack(
Collections.<DfsPackDescription>emptyList(),
remove);
return;
}
writePack(objdb, outDesc, pw, pm);
writeIndex(objdb, outDesc, pw);
boolean rollback = true;
DfsPackDescription pack = objdb.newPack(COMPACT,
estimatePackSize());
try {
writePack(objdb, pack, pw, pm);
writeIndex(objdb, pack, pw);
PackStatistics stats = pw.getStatistics();
pw.close();
pw = null;
pack.setPackStats(stats);
objdb.commitPack(Collections.singletonList(pack), toPrune());
newPacks.add(pack);
newStats.add(stats);
rollback = false;
} finally {
if (rollback)
objdb.rollbackPack(Collections.singletonList(pack));
}
PackStatistics stats = pw.getStatistics();
pw.close();
pw = null;
outDesc.setPackStats(stats);
newStats = stats;
rollback = false;
} finally {
if (pw != null)
pw.close();
if (rollback) {
objdb.rollbackPack(Collections.singletonList(outDesc));
}
}
} finally {
rw = null;
if (pw != null) {
pw.close();
}
}
}
@ -263,27 +312,81 @@ public class DfsPackCompactor {
return size;
}
private void compactReftables(DfsReader ctx) throws IOException {
DfsObjDatabase objdb = repo.getObjectDatabase();
Collections.sort(srcReftables, objdb.reftableComparator());
try (ReftableStack stack = ReftableStack.open(ctx, srcReftables)) {
initOutDesc(objdb);
ReftableCompactor compact = new ReftableCompactor();
compact.addAll(stack.readers());
compact.setIncludeDeletes(true);
writeReftable(objdb, outDesc, compact);
}
}
private void initOutDesc(DfsObjDatabase objdb) throws IOException {
if (outDesc == null) {
outDesc = objdb.newPack(COMPACT, estimatePackSize());
}
}
/** @return all of the source packs that fed into this compaction. */
public List<DfsPackDescription> getSourcePacks() {
return toPrune();
public Collection<DfsPackDescription> getSourcePacks() {
Set<DfsPackDescription> src = new HashSet<>();
for (DfsPackFile pack : srcPacks) {
src.add(pack.getPackDescription());
}
for (DfsReftable table : srcReftables) {
src.add(table.getPackDescription());
}
return src;
}
/** @return new packs created by this compaction. */
public List<DfsPackDescription> getNewPacks() {
return newPacks;
return outDesc != null
? Collections.singletonList(outDesc)
: Collections.emptyList();
}
/** @return statistics corresponding to the {@link #getNewPacks()}. */
public List<PackStatistics> getNewPackStatistics() {
return newStats;
return newStats != null
? Collections.singletonList(newStats)
: Collections.emptyList();
}
private List<DfsPackDescription> toPrune() {
int cnt = srcPacks.size();
List<DfsPackDescription> all = new ArrayList<>(cnt);
for (DfsPackFile pack : srcPacks)
all.add(pack.getPackDescription());
return all;
private Collection<DfsPackDescription> toPrune() {
Set<DfsPackDescription> packs = new HashSet<>();
for (DfsPackFile pack : srcPacks) {
packs.add(pack.getPackDescription());
}
Set<DfsPackDescription> reftables = new HashSet<>();
for (DfsReftable table : srcReftables) {
reftables.add(table.getPackDescription());
}
for (Iterator<DfsPackDescription> i = packs.iterator(); i.hasNext();) {
DfsPackDescription d = i.next();
if (d.hasFileExt(REFTABLE) && !reftables.contains(d)) {
i.remove();
}
}
for (Iterator<DfsPackDescription> i = reftables.iterator();
i.hasNext();) {
DfsPackDescription d = i.next();
if (d.hasFileExt(PACK) && !packs.contains(d)) {
i.remove();
}
}
Set<DfsPackDescription> toPrune = new HashSet<>();
toPrune.addAll(packs);
toPrune.addAll(reftables);
return toPrune;
}
private void addObjectsToPack(PackWriter pw, DfsReader ctx,
@ -390,6 +493,27 @@ public class DfsPackCompactor {
}
}
private void writeReftable(DfsObjDatabase objdb, DfsPackDescription pack,
ReftableCompactor compact) throws IOException {
try (DfsOutputStream out = objdb.writeFile(pack, REFTABLE)) {
compact.setConfig(configureReftable(reftableConfig, out));
compact.compact(out);
pack.addFileExt(REFTABLE);
pack.setReftableStats(compact.getStats());
}
}
static ReftableConfig configureReftable(ReftableConfig cfg,
DfsOutputStream out) {
int bs = out.blockSize();
if (bs > 0) {
cfg = new ReftableConfig(cfg);
cfg.setRefBlockSize(bs);
cfg.setAlignBlocks(true);
}
return cfg;
}
private static class ObjectIdWithOffset extends ObjectId {
final long offset;

21
org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableCompactor.java
Unescape View File

@ -159,9 +159,16 @@ public class ReftableCompactor {
* tables to compact. Tables should be ordered oldest first/most
* recent last so that the more recent tables can shadow the
* older results. Caller is responsible for closing the readers.
* @throws IOException
* update indexes of a reader cannot be accessed.
*/
public void addAll(List<? extends Reftable> readers) {
public void addAll(List<? extends Reftable> readers) throws IOException {
tables.addAll(readers);
for (Reftable r : readers) {
if (r instanceof ReftableReader) {
adjustUpdateIndexes((ReftableReader) r);
}
}
}
/**
@ -178,7 +185,7 @@ public class ReftableCompactor {
* @return {@code true} if the compactor accepted this table; {@code false}
* if the compactor has reached its limit.
* @throws IOException
* if size of {@code reader} cannot be read.
* if size of {@code reader}, or its update indexes cannot be read.
*/
public boolean tryAddFirst(ReftableReader reader) throws IOException {
long sz = reader.size();
@ -186,10 +193,20 @@ public class ReftableCompactor {
return false;
}
bytesToCompact += sz;
adjustUpdateIndexes(reader);
tables.addFirst(reader);
return true;
}
private void adjustUpdateIndexes(ReftableReader reader) throws IOException {
if (minUpdateIndex == 0) {
minUpdateIndex = reader.minUpdateIndex();
} else {
minUpdateIndex = Math.min(minUpdateIndex, reader.minUpdateIndex());
}
maxUpdateIndex = Math.max(maxUpdateIndex, reader.maxUpdateIndex());
}
/**
* Write a compaction to {@code out}.
*

Write Preview
Loading…
Cancel
Save