From 5185d288c11171a21a3057f0df906782493d0b39 Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Sun, 17 Nov 2019 16:29:16 -0800 Subject: [PATCH] Introduce RefDatabase#hasFastTipsWithSha1 The reftable format supports fast inverse (SHA1 => ref) queries. If the ref database does not support fast inverse queries, it may be advantageous to build a complete SHA1 to ref map in advance for multiple uses. To let applications decide, this function indicates whether the inverse map is available. Signed-off-by: Han-Wen Nienhuys Change-Id: Idaf7e01075906972ec21332cade285289619c2b3 --- .../storage/file/FileReftableTest.java | 4 ++ .../storage/reftable/ReftableTest.java | 66 +++++++++++++++++++ .../storage/dfs/DfsReftableDatabase.java | 6 ++ .../storage/file/FileReftableDatabase.java | 6 ++ .../storage/reftable/MergedReftable.java | 10 +++ .../internal/storage/reftable/Reftable.java | 6 ++ .../storage/reftable/ReftableDatabase.java | 13 ++++ .../storage/reftable/ReftableReader.java | 10 +++ .../src/org/eclipse/jgit/lib/RefDatabase.java | 14 ++++ 9 files changed, 135 insertions(+) diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileReftableTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileReftableTest.java index 0e06d459a..cdc64fa1b 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileReftableTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileReftableTest.java @@ -155,6 +155,8 @@ public class FileReftableTest extends SampleDataRepositoryTestCase { assertFalse(b.isSymbolic()); assertTrue(b.isPeeled()); assertEquals(bCommit, b.getObjectId().name()); + + assertTrue(db.getRefDatabase().hasFastTipsWithSha1()); } @Test @@ -169,6 +171,8 @@ public class FileReftableTest extends SampleDataRepositoryTestCase { assertFalse(b.isSymbolic()); assertTrue(b.isPeeled()); assertEquals(bCommit, b.getObjectId().name()); + + assertFalse(db.getRefDatabase().hasFastTipsWithSha1()); } @Test diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/reftable/ReftableTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/reftable/ReftableTest.java index 45e6c7d12..daf540c41 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/reftable/ReftableTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/reftable/ReftableTest.java @@ -176,6 +176,67 @@ public class ReftableTest { assertEquals(expBytes, table.length); } + @Test + public void hasObjMapRefs() throws IOException { + ArrayList refs = new ArrayList<>(); + refs.add(ref(MASTER, 1)); + byte[] table = write(refs); + ReftableReader t = read(table); + assertTrue(t.hasObjectMap()); + } + + @Test + public void hasObjMapRefsSmallTable() throws IOException { + ArrayList refs = new ArrayList<>(); + ReftableConfig cfg = new ReftableConfig(); + cfg.setIndexObjects(false); + refs.add(ref(MASTER, 1)); + byte[] table = write(refs); + ReftableReader t = read(table); + assertTrue(t.hasObjectMap()); + } + + @Test + public void hasObjLogs() throws IOException { + PersonIdent who = new PersonIdent("Log", "Ger", 1500079709, -8 * 60); + String msg = "test"; + ReftableConfig cfg = new ReftableConfig(); + cfg.setIndexObjects(false); + + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + ReftableWriter writer = new ReftableWriter(buffer) + .setMinUpdateIndex(1) + .setConfig(cfg) + .setMaxUpdateIndex(1) + .begin(); + + writer.writeLog("master", 1, who, ObjectId.zeroId(), id(1), msg); + writer.finish(); + byte[] table = buffer.toByteArray(); + + ReftableReader t = read(table); + assertTrue(t.hasObjectMap()); + } + + @Test + public void hasObjMapRefsNoIndexObjects() throws IOException { + ArrayList refs = new ArrayList<>(); + ReftableConfig cfg = new ReftableConfig(); + cfg.setIndexObjects(false); + cfg.setRefBlockSize(256); + cfg.setAlignBlocks(true); + + // Fill up 5 blocks. + int N = 256 * 5 / 25; + for (int i= 0; i < N; i++) { + refs.add(ref(String.format("%02d/xxxxxxxxxx", i), i)); + } + byte[] table = write(refs, cfg); + + ReftableReader t = read(table); + assertFalse(t.hasObjectMap()); + } + @Test public void oneIdRef() throws IOException { Ref exp = ref(MASTER, 1); @@ -936,8 +997,13 @@ public class ReftableTest { } private byte[] write(Collection refs) throws IOException { + return write(refs, new ReftableConfig()); + } + + private byte[] write(Collection refs, ReftableConfig cfg) throws IOException { ByteArrayOutputStream buffer = new ByteArrayOutputStream(); stats = new ReftableWriter(buffer) + .setConfig(cfg) .begin() .sortAndWriteRefs(refs) .finish() diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReftableDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReftableDatabase.java index 2ad3ae7e6..124131d1d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReftableDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReftableDatabase.java @@ -216,6 +216,12 @@ public class DfsReftableDatabase extends DfsRefDatabase { return reftableDatabase.getTipsWithSha1(id); } + /** {@inheritDoc} */ + @Override + public boolean hasFastTipsWithSha1() throws IOException { + return reftableDatabase.hasFastTipsWithSha1(); + } + /** {@inheritDoc} */ @Override public Ref peel(Ref ref) throws IOException { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java index b06fd30b3..fd80ad9ec 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java @@ -125,6 +125,12 @@ public class FileReftableDatabase extends RefDatabase { && new File(repoDir, Constants.REFTABLE).isDirectory(); } + /** {@inheritDoc} */ + @Override + public boolean hasFastTipsWithSha1() throws IOException { + return reftableDatabase.hasFastTipsWithSha1(); + } + /** * Runs a full compaction for GC purposes. * @throws IOException on I/O errors diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/MergedReftable.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/MergedReftable.java index 4de5e392f..6c312ca7a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/MergedReftable.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/MergedReftable.java @@ -98,6 +98,16 @@ public class MergedReftable extends Reftable { : 0; } + /** {@inheritDoc} */ + @Override + public boolean hasObjectMap() throws IOException { + boolean has = true; + for (int i = 0; has && i < tables.length; i++) { + has = has && tables[i].hasObjectMap(); + }; + return has; + } + /** {@inheritDoc} */ @Override public RefCursor allRefs() throws IOException { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/Reftable.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/Reftable.java index 8f6426176..8a8a14375 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/Reftable.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/Reftable.java @@ -161,6 +161,12 @@ public abstract class Reftable { */ public abstract RefCursor byObjectId(AnyObjectId id) throws IOException; + /** + * @return whether this reftable can do a fast SHA1 => ref lookup. + * @throws IOException on I/O problems. + */ + public abstract boolean hasObjectMap() throws IOException; + /** * Seek reader to read log records. * diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableDatabase.java index 5a32d2914..c08f1814d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableDatabase.java @@ -298,6 +298,19 @@ public abstract class ReftableDatabase { return Collections.unmodifiableList(all); } + /** + * @return whether there is a fast SHA1 to ref map. + * @throws IOException in case of I/O problems. + */ + public boolean hasFastTipsWithSha1() throws IOException { + lock.lock(); + try { + return reader().hasObjectMap(); + } finally { + lock.unlock(); + } + } + /** * Returns all refs that resolve directly to the given {@link ObjectId}. * Includes peeled {@linkObjectId}s. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableReader.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableReader.java index c5e667449..14b821a1e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableReader.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableReader.java @@ -128,6 +128,16 @@ public class ReftableReader extends Reftable implements AutoCloseable { return blockSize; } + @Override + public boolean hasObjectMap() throws IOException { + if (objIndexPosition == -1) { + readFileFooter(); + } + + // We have the map, we have no refs, or the table is small. + return (objPosition > 0 || refEnd == 24 || refIndexPosition == 0); + } + /** * Get the minimum update index for log entries that appear in this * reftable. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefDatabase.java index 4d9450e75..9b5a1fdc2 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefDatabase.java @@ -496,6 +496,20 @@ public abstract class RefDatabase { || id.equals(r.getPeeledObjectId())).collect(toSet()); } + /** + * If the ref database does not support fast inverse queries, it may + * be advantageous to build a complete SHA1 to ref map in advance for + * multiple uses. To let applications decide on this decision, + * this function indicates whether the inverse map is available. + * + * @return whether this RefDatabase supports fast inverse ref queries. + * @throws IOException on I/O problems. + * @since 5.6 + */ + public boolean hasFastTipsWithSha1() throws IOException { + return false; + } + /** * Check if any refs exist in the ref database. *