|
|
@ -28,22 +28,20 @@ import org.eclipse.jgit.lib.ReflogEntry; |
|
|
|
* to shadow any lower reftable that may have the reference present. |
|
|
|
* to shadow any lower reftable that may have the reference present. |
|
|
|
* <p> |
|
|
|
* <p> |
|
|
|
* By default all log entries within the range defined by |
|
|
|
* By default all log entries within the range defined by |
|
|
|
* {@link #setMinUpdateIndex(long)} and {@link #setMaxUpdateIndex(long)} are |
|
|
|
* {@link #setReflogExpireMinUpdateIndex(long)} and {@link #setReflogExpireMaxUpdateIndex(long)} are |
|
|
|
* copied, even if no references in the output file match the log records. |
|
|
|
* copied, even if no references in the output file match the log records. |
|
|
|
* Callers may truncate the log to a more recent time horizon with |
|
|
|
* Callers may truncate the log to a more recent time horizon with |
|
|
|
* {@link #setOldestReflogTimeMillis(long)}, or disable the log altogether with |
|
|
|
* {@link #setReflogExpireOldestReflogTimeMillis(long)}, or disable the log altogether with |
|
|
|
* {@code setOldestReflogTimeMillis(Long.MAX_VALUE)}. |
|
|
|
* {@code setOldestReflogTimeMillis(Long.MAX_VALUE)}. |
|
|
|
*/ |
|
|
|
*/ |
|
|
|
public class ReftableCompactor { |
|
|
|
public class ReftableCompactor { |
|
|
|
private final ReftableWriter writer; |
|
|
|
private final ReftableWriter writer; |
|
|
|
private final ArrayDeque<ReftableReader> tables = new ArrayDeque<>(); |
|
|
|
private final ArrayDeque<ReftableReader> tables = new ArrayDeque<>(); |
|
|
|
|
|
|
|
|
|
|
|
private long compactBytesLimit; |
|
|
|
|
|
|
|
private long bytesToCompact; |
|
|
|
|
|
|
|
private boolean includeDeletes; |
|
|
|
private boolean includeDeletes; |
|
|
|
private long minUpdateIndex = -1; |
|
|
|
private long reflogExpireMinUpdateIndex = 0; |
|
|
|
private long maxUpdateIndex; |
|
|
|
private long reflogExpireMaxUpdateIndex = Long.MAX_VALUE; |
|
|
|
private long oldestReflogTimeMillis; |
|
|
|
private long reflogExpireOldestReflogTimeMillis; |
|
|
|
private Stats stats; |
|
|
|
private Stats stats; |
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
/** |
|
|
@ -69,18 +67,6 @@ public class ReftableCompactor { |
|
|
|
return this; |
|
|
|
return this; |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
|
|
|
* Set limit on number of bytes from source tables to compact. |
|
|
|
|
|
|
|
* |
|
|
|
|
|
|
|
* @param bytes |
|
|
|
|
|
|
|
* limit on number of bytes from source tables to compact. |
|
|
|
|
|
|
|
* @return {@code this} |
|
|
|
|
|
|
|
*/ |
|
|
|
|
|
|
|
public ReftableCompactor setCompactBytesLimit(long bytes) { |
|
|
|
|
|
|
|
compactBytesLimit = bytes; |
|
|
|
|
|
|
|
return this; |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
/** |
|
|
|
* Whether to include deletions in the output, which may be necessary for |
|
|
|
* Whether to include deletions in the output, which may be necessary for |
|
|
|
* partial compaction. |
|
|
|
* partial compaction. |
|
|
@ -106,8 +92,8 @@ public class ReftableCompactor { |
|
|
|
* in a stack. |
|
|
|
* in a stack. |
|
|
|
* @return {@code this} |
|
|
|
* @return {@code this} |
|
|
|
*/ |
|
|
|
*/ |
|
|
|
public ReftableCompactor setMinUpdateIndex(long min) { |
|
|
|
public ReftableCompactor setReflogExpireMinUpdateIndex(long min) { |
|
|
|
minUpdateIndex = min; |
|
|
|
reflogExpireMinUpdateIndex = min; |
|
|
|
return this; |
|
|
|
return this; |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
@ -122,8 +108,8 @@ public class ReftableCompactor { |
|
|
|
* used in a stack. |
|
|
|
* used in a stack. |
|
|
|
* @return {@code this} |
|
|
|
* @return {@code this} |
|
|
|
*/ |
|
|
|
*/ |
|
|
|
public ReftableCompactor setMaxUpdateIndex(long max) { |
|
|
|
public ReftableCompactor setReflogExpireMaxUpdateIndex(long max) { |
|
|
|
maxUpdateIndex = max; |
|
|
|
reflogExpireMaxUpdateIndex = max; |
|
|
|
return this; |
|
|
|
return this; |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
@ -137,16 +123,13 @@ public class ReftableCompactor { |
|
|
|
* Specified in Java standard milliseconds since the epoch. |
|
|
|
* Specified in Java standard milliseconds since the epoch. |
|
|
|
* @return {@code this} |
|
|
|
* @return {@code this} |
|
|
|
*/ |
|
|
|
*/ |
|
|
|
public ReftableCompactor setOldestReflogTimeMillis(long timeMillis) { |
|
|
|
public ReftableCompactor setReflogExpireOldestReflogTimeMillis(long timeMillis) { |
|
|
|
oldestReflogTimeMillis = timeMillis; |
|
|
|
reflogExpireOldestReflogTimeMillis = timeMillis; |
|
|
|
return this; |
|
|
|
return this; |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
/** |
|
|
|
* Add all of the tables, in the specified order. |
|
|
|
* Add all of the tables, in the specified order. |
|
|
|
* <p> |
|
|
|
|
|
|
|
* Unconditionally adds all tables, ignoring the |
|
|
|
|
|
|
|
* {@link #setCompactBytesLimit(long)}. |
|
|
|
|
|
|
|
* |
|
|
|
* |
|
|
|
* @param readers |
|
|
|
* @param readers |
|
|
|
* tables to compact. Tables should be ordered oldest first/most |
|
|
|
* tables to compact. Tables should be ordered oldest first/most |
|
|
@ -158,44 +141,7 @@ public class ReftableCompactor { |
|
|
|
public void addAll(List<ReftableReader> readers) throws IOException { |
|
|
|
public void addAll(List<ReftableReader> readers) throws IOException { |
|
|
|
for (ReftableReader r : readers) { |
|
|
|
for (ReftableReader r : readers) { |
|
|
|
tables.add(r); |
|
|
|
tables.add(r); |
|
|
|
adjustUpdateIndexes(r); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
|
|
|
* Try to add this reader at the bottom of the stack. |
|
|
|
|
|
|
|
* <p> |
|
|
|
|
|
|
|
* A reader may be rejected by returning {@code false} if the compactor is |
|
|
|
|
|
|
|
* already rewriting its {@link #setCompactBytesLimit(long)}. When this |
|
|
|
|
|
|
|
* happens the caller should stop trying to add tables, and execute the |
|
|
|
|
|
|
|
* compaction. |
|
|
|
|
|
|
|
* |
|
|
|
|
|
|
|
* @param reader |
|
|
|
|
|
|
|
* the reader to insert at the bottom of the stack. Caller is |
|
|
|
|
|
|
|
* responsible for closing the reader. |
|
|
|
|
|
|
|
* @return {@code true} if the compactor accepted this table; {@code false} |
|
|
|
|
|
|
|
* if the compactor has reached its limit. |
|
|
|
|
|
|
|
* @throws java.io.IOException |
|
|
|
|
|
|
|
* if size of {@code reader}, or its update indexes cannot be read. |
|
|
|
|
|
|
|
*/ |
|
|
|
|
|
|
|
public boolean tryAddFirst(ReftableReader reader) throws IOException { |
|
|
|
|
|
|
|
long sz = reader.size(); |
|
|
|
|
|
|
|
if (compactBytesLimit > 0 && bytesToCompact + sz > compactBytesLimit) { |
|
|
|
|
|
|
|
return false; |
|
|
|
|
|
|
|
} |
|
|
|
} |
|
|
|
bytesToCompact += sz; |
|
|
|
|
|
|
|
adjustUpdateIndexes(reader); |
|
|
|
|
|
|
|
tables.addFirst(reader); |
|
|
|
|
|
|
|
return true; |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
private void adjustUpdateIndexes(ReftableReader reader) throws IOException { |
|
|
|
|
|
|
|
if (minUpdateIndex == -1) { |
|
|
|
|
|
|
|
minUpdateIndex = reader.minUpdateIndex(); |
|
|
|
|
|
|
|
} else { |
|
|
|
|
|
|
|
minUpdateIndex = Math.min(minUpdateIndex, reader.minUpdateIndex()); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
maxUpdateIndex = Math.max(maxUpdateIndex, reader.maxUpdateIndex()); |
|
|
|
|
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
/** |
|
|
@ -208,8 +154,9 @@ public class ReftableCompactor { |
|
|
|
MergedReftable mr = new MergedReftable(new ArrayList<>(tables)); |
|
|
|
MergedReftable mr = new MergedReftable(new ArrayList<>(tables)); |
|
|
|
mr.setIncludeDeletes(includeDeletes); |
|
|
|
mr.setIncludeDeletes(includeDeletes); |
|
|
|
|
|
|
|
|
|
|
|
writer.setMinUpdateIndex(Math.max(minUpdateIndex, 0)); |
|
|
|
writer.setMaxUpdateIndex(mr.maxUpdateIndex()); |
|
|
|
writer.setMaxUpdateIndex(maxUpdateIndex); |
|
|
|
writer.setMinUpdateIndex(mr.minUpdateIndex()); |
|
|
|
|
|
|
|
|
|
|
|
writer.begin(); |
|
|
|
writer.begin(); |
|
|
|
mergeRefs(mr); |
|
|
|
mergeRefs(mr); |
|
|
|
mergeLogs(mr); |
|
|
|
mergeLogs(mr); |
|
|
@ -235,16 +182,14 @@ public class ReftableCompactor { |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
private void mergeLogs(MergedReftable mr) throws IOException { |
|
|
|
private void mergeLogs(MergedReftable mr) throws IOException { |
|
|
|
if (oldestReflogTimeMillis == Long.MAX_VALUE) { |
|
|
|
if (reflogExpireOldestReflogTimeMillis == Long.MAX_VALUE) { |
|
|
|
return; |
|
|
|
return; |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
try (LogCursor lc = mr.allLogs()) { |
|
|
|
try (LogCursor lc = mr.allLogs()) { |
|
|
|
while (lc.next()) { |
|
|
|
while (lc.next()) { |
|
|
|
long updateIndex = lc.getUpdateIndex(); |
|
|
|
long updateIndex = lc.getUpdateIndex(); |
|
|
|
if (updateIndex < minUpdateIndex |
|
|
|
if (updateIndex > reflogExpireMaxUpdateIndex || updateIndex < reflogExpireMinUpdateIndex) { |
|
|
|
|| updateIndex > maxUpdateIndex) { |
|
|
|
|
|
|
|
// Cannot merge log records outside the header's range.
|
|
|
|
|
|
|
|
continue; |
|
|
|
continue; |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
@ -258,7 +203,7 @@ public class ReftableCompactor { |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
PersonIdent who = log.getWho(); |
|
|
|
PersonIdent who = log.getWho(); |
|
|
|
if (who.getWhen().getTime() >= oldestReflogTimeMillis) { |
|
|
|
if (who.getWhen().getTime() >= reflogExpireOldestReflogTimeMillis) { |
|
|
|
writer.writeLog( |
|
|
|
writer.writeLog( |
|
|
|
refName, |
|
|
|
refName, |
|
|
|
updateIndex, |
|
|
|
updateIndex, |
|
|
|