diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/reftable/MergedReftableTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/reftable/MergedReftableTest.java index b6306d09e..f9ebaf692 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/reftable/MergedReftableTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/reftable/MergedReftableTest.java @@ -261,6 +261,33 @@ public class MergedReftableTest { } } + @Test + public void compaction() throws IOException { + List delta1 = Arrays.asList( + ref("refs/heads/next", 4), + ref("refs/heads/master", 1)); + List delta2 = Arrays.asList(delete("refs/heads/next")); + List delta3 = Arrays.asList(ref("refs/heads/master", 8)); + + ReftableCompactor compactor = new ReftableCompactor(); + compactor.addAll(Arrays.asList( + read(write(delta1)), + read(write(delta2)), + read(write(delta3)))); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + compactor.compact(out); + byte[] table = out.toByteArray(); + + ReftableReader reader = read(table); + try (RefCursor rc = reader.allRefs()) { + assertTrue(rc.next()); + Ref r = rc.getRef(); + assertEquals("refs/heads/master", r.getName()); + assertEquals(id(8), r.getObjectId()); + assertFalse(rc.next()); + } + } + private static MergedReftable merge(byte[]... table) { List stack = new ArrayList<>(table.length); for (byte[] b : table) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableCompactor.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableCompactor.java new file mode 100644 index 000000000..a91cdc8f1 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableCompactor.java @@ -0,0 +1,223 @@ +/* + * Copyright (C) 2017, Google Inc. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.eclipse.jgit.internal.storage.reftable; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.List; + +import org.eclipse.jgit.lib.PersonIdent; +import org.eclipse.jgit.internal.storage.reftable.ReftableWriter.Stats; +import org.eclipse.jgit.lib.ReflogEntry; + +/** + * Merges reftables and compacts them into a single output. + *

+ * For a partial compaction callers should {@link #setIncludeDeletes(boolean)} + * to {@code true} to ensure the new reftable continues to use a delete marker + * to shadow any lower reftable that may have the reference present. + *

+ * By default all log entries within the range defined by + * {@link #setMinUpdateIndex(long)} and {@link #setMaxUpdateIndex(long)} are + * copied, even if no references in the output file match the log records. + * Callers may truncate the log to a more recent time horizon with + * {@link #setOldestReflogTimeMillis(long)}, or disable the log altogether with + * {@code setOldestReflogTimeMillis(Long.MAX_VALUE)}. + */ +public class ReftableCompactor { + private final ReftableWriter writer = new ReftableWriter(); + private final ArrayDeque tables = new ArrayDeque<>(); + + private boolean includeDeletes; + private long minUpdateIndex; + private long maxUpdateIndex; + private long oldestReflogTimeMillis; + private Stats stats; + + /** + * @param cfg + * configuration for the reftable. + * @return {@code this} + */ + public ReftableCompactor setConfig(ReftableConfig cfg) { + writer.setConfig(cfg); + return this; + } + + /** + * @param deletes + * {@code true} to include deletions in the output, which may be + * necessary for partial compaction. + * @return {@code this} + */ + public ReftableCompactor setIncludeDeletes(boolean deletes) { + includeDeletes = deletes; + return this; + } + + /** + * @param min + * the minimum update index for log entries that appear in the + * compacted reftable. This should be 1 higher than the prior + * reftable's {@code maxUpdateIndex} if this table will be used + * in a stack. + * @return {@code this} + */ + public ReftableCompactor setMinUpdateIndex(long min) { + minUpdateIndex = min; + return this; + } + + /** + * @param max + * the maximum update index for log entries that appear in the + * compacted reftable. This should be at least 1 higher than the + * prior reftable's {@code maxUpdateIndex} if this table will be + * used in a stack. + * @return {@code this} + */ + public ReftableCompactor setMaxUpdateIndex(long max) { + maxUpdateIndex = max; + return this; + } + + /** + * @param timeMillis + * oldest log time to preserve. Entries whose timestamps are + * {@code >= timeMillis} will be copied into the output file. Log + * entries that predate {@code timeMillis} will be discarded. + * Specified in Java standard milliseconds since the epoch. + * @return {@code this} + */ + public ReftableCompactor setOldestReflogTimeMillis(long timeMillis) { + oldestReflogTimeMillis = timeMillis; + return this; + } + + /** + * Add all of the tables, in the specified order. + * + * @param readers + * tables to compact. Tables should be ordered oldest first/most + * recent last so that the more recent tables can shadow the + * older results. Caller is responsible for closing the readers. + */ + public void addAll(List readers) { + tables.addAll(readers); + } + + /** + * Write a compaction to {@code out}. + * + * @param out + * stream to write the compacted tables to. Caller is responsible + * for closing {@code out}. + * @throws IOException + * if tables cannot be read, or cannot be written. + */ + public void compact(OutputStream out) throws IOException { + MergedReftable mr = new MergedReftable(new ArrayList<>(tables)); + mr.setIncludeDeletes(includeDeletes); + + writer.setMinUpdateIndex(minUpdateIndex); + writer.setMaxUpdateIndex(maxUpdateIndex); + writer.begin(out); + mergeRefs(mr); + mergeLogs(mr); + writer.finish(); + stats = writer.getStats(); + } + + /** @return statistics of the last written reftable. */ + public Stats getStats() { + return stats; + } + + private void mergeRefs(MergedReftable mr) throws IOException { + try (RefCursor rc = mr.allRefs()) { + while (rc.next()) { + writer.writeRef(rc.getRef()); + } + } + } + + private void mergeLogs(MergedReftable mr) throws IOException { + if (oldestReflogTimeMillis == Long.MAX_VALUE) { + return; + } + + try (LogCursor lc = mr.allLogs()) { + while (lc.next()) { + long updateIndex = lc.getUpdateIndex(); + if (updateIndex < minUpdateIndex + || updateIndex > maxUpdateIndex) { + // Cannot merge log records outside the header's range. + continue; + } + + String refName = lc.getRefName(); + ReflogEntry log = lc.getReflogEntry(); + if (log == null) { + if (includeDeletes) { + writer.deleteLog(refName, updateIndex); + } + continue; + } + + PersonIdent who = log.getWho(); + if (who.getWhen().getTime() >= oldestReflogTimeMillis) { + writer.writeLog( + refName, + updateIndex, + who, + log.getOldId(), + log.getNewId(), + log.getComment()); + } + } + } + } +}