From 1222f345063bf989f35920b5e06091d9ea3b6ab0 Mon Sep 17 00:00:00 2001 From: Shawn Pearce Date: Mon, 17 Jul 2017 09:35:14 -0700 Subject: [PATCH] dfs: support reading reftables through DfsBlockCache DfsBlockCache directly shares its internal byte[] with ReftableReader, avoding copying between the DfsBlockCache and the BlockReader instances used by ReftableReader. Change-Id: Icaa4f40052b26f952681414653a8b5314b7c2c23 --- .../internal/storage/dfs/BlockBasedFile.java | 6 +- .../jgit/internal/storage/dfs/DfsBlock.java | 10 +- .../internal/storage/dfs/DfsPackFile.java | 4 - .../jgit/internal/storage/dfs/DfsReader.java | 18 +- .../internal/storage/dfs/DfsReftable.java | 178 ++++++++++++++++++ 5 files changed, 199 insertions(+), 17 deletions(-) create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReftable.java diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/BlockBasedFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/BlockBasedFile.java index 813e7f4cd..b9758bd64 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/BlockBasedFile.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/BlockBasedFile.java @@ -53,7 +53,7 @@ import org.eclipse.jgit.errors.PackInvalidException; import org.eclipse.jgit.internal.storage.pack.PackExt; /** Block based file stored in {@link DfsBlockCache}. */ -public abstract class BlockBasedFile { +abstract class BlockBasedFile { /** Cache that owns this file and its data. */ final DfsBlockCache cache; @@ -129,6 +129,10 @@ public abstract class BlockBasedFile { return size; } + DfsBlock getOrLoadBlock(long pos, DfsReader ctx) throws IOException { + return cache.getOrLoad(this, pos, ctx, null); + } + DfsBlock readOneBlock(long pos, DfsReader ctx, @Nullable ReadableChannel fileChannel) throws IOException { if (invalid) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlock.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlock.java index dae922eb4..62a9be3e5 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlock.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlock.java @@ -46,6 +46,7 @@ package org.eclipse.jgit.internal.storage.dfs; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.zip.CRC32; import java.util.zip.DataFormatException; import java.util.zip.Inflater; @@ -55,11 +56,8 @@ import org.eclipse.jgit.internal.storage.pack.PackOutputStream; /** A cached slice of a {@link BlockBasedFile}. */ final class DfsBlock { final DfsStreamKey stream; - final long start; - final long end; - private final byte[] block; DfsBlock(DfsStreamKey p, long pos, byte[] buf) { @@ -73,6 +71,12 @@ final class DfsBlock { return block.length; } + ByteBuffer zeroCopyByteBuffer(int n) { + ByteBuffer b = ByteBuffer.wrap(block); + b.position(n); + return b; + } + boolean contains(DfsStreamKey want, long pos) { return stream.equals(want) && start <= pos && pos < end; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java index 2326219fd..81b9980e2 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java @@ -739,10 +739,6 @@ public final class DfsPackFile extends BlockBasedFile { throw new EOFException(); } - DfsBlock getOrLoadBlock(long pos, DfsReader ctx) throws IOException { - return cache.getOrLoad(this, pos, ctx, null); - } - ObjectLoader load(DfsReader ctx, long pos) throws IOException { try { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java index 4b0d58343..3c8422077 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java @@ -655,7 +655,7 @@ public class DfsReader extends ObjectReader implements ObjectReuseAsIs { /** * Copy bytes from the window to a caller supplied buffer. * - * @param pack + * @param file * the file the desired window is stored within. * @param position * position within the file to read from. @@ -674,24 +674,24 @@ public class DfsReader extends ObjectReader implements ObjectReuseAsIs { * this cursor does not match the provider or id and the proper * window could not be acquired through the provider's cache. */ - int copy(DfsPackFile pack, long position, byte[] dstbuf, int dstoff, int cnt) - throws IOException { + int copy(BlockBasedFile file, long position, byte[] dstbuf, int dstoff, + int cnt) throws IOException { if (cnt == 0) return 0; - long length = pack.length; + long length = file.length; if (0 <= length && length <= position) return 0; int need = cnt; do { - pin(pack, position); + pin(file, position); int r = block.copy(position, dstbuf, dstoff, need); position += r; dstoff += r; need -= r; if (length < 0) - length = pack.length; + length = file.length; } while (0 < need && position < length); return cnt - need; } @@ -756,14 +756,14 @@ public class DfsReader extends ObjectReader implements ObjectReuseAsIs { inf.reset(); } - void pin(DfsPackFile pack, long position) throws IOException { - if (block == null || !block.contains(pack.key, position)) { + void pin(BlockBasedFile file, long position) throws IOException { + if (block == null || !block.contains(file.key, position)) { // If memory is low, we may need what is in our window field to // be cleaned up by the GC during the get for the next window. // So we always clear it, even though we are just going to set // it again. block = null; - block = pack.getOrLoadBlock(position, this); + block = file.getOrLoadBlock(position, this); } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReftable.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReftable.java new file mode 100644 index 000000000..5a8ea92a8 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReftable.java @@ -0,0 +1,178 @@ +/* + * Copyright (C) 2017, Google Inc. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.eclipse.jgit.internal.storage.dfs; + +import static org.eclipse.jgit.internal.storage.pack.PackExt.REFTABLE; + +import java.io.IOException; +import java.nio.ByteBuffer; + +import org.eclipse.jgit.internal.storage.io.BlockSource; +import org.eclipse.jgit.internal.storage.reftable.ReftableReader; + +/** A reftable stored in {@link DfsBlockCache}. */ +public class DfsReftable extends BlockBasedFile { + /** + * Construct a reader for an existing reftable. + * + * @param desc + * description of the reftable within the DFS. + */ + public DfsReftable(DfsPackDescription desc) { + this(DfsBlockCache.getInstance(), desc); + } + + /** + * Construct a reader for an existing reftable. + * + * @param cache + * cache that will store the reftable data. + * @param desc + * description of the reftable within the DFS. + */ + public DfsReftable(DfsBlockCache cache, DfsPackDescription desc) { + super(cache, desc, REFTABLE); + + int bs = desc.getBlockSize(REFTABLE); + if (bs > 0) { + setBlockSize(bs); + } + + long sz = desc.getFileSize(REFTABLE); + length = sz > 0 ? sz : -1; + } + + /** @return description that was originally used to configure this file. */ + public DfsPackDescription getPackDescription() { + return desc; + } + + /** + * Open reader on the reftable. + *

+ * The returned reader is not thread safe. + * + * @param ctx + * reader to access the DFS storage. + * @return cursor to read the table; caller must close. + * @throws IOException + * table cannot be opened. + */ + public ReftableReader open(DfsReader ctx) throws IOException { + return new ReftableReader(new CacheSource(this, cache, ctx)); + } + + private static final class CacheSource extends BlockSource { + private final DfsReftable file; + private final DfsBlockCache cache; + private final DfsReader ctx; + private ReadableChannel ch; + private int readAhead; + + CacheSource(DfsReftable file, DfsBlockCache cache, DfsReader ctx) { + this.file = file; + this.cache = cache; + this.ctx = ctx; + } + + @Override + public ByteBuffer read(long pos, int cnt) throws IOException { + if (ch == null && readAhead > 0 && notInCache(pos)) { + open().setReadAheadBytes(readAhead); + } + + DfsBlock block = cache.getOrLoad(file, pos, ctx, ch); + if (block.start == pos && block.size() >= cnt) { + return block.zeroCopyByteBuffer(cnt); + } + + byte[] dst = new byte[cnt]; + ByteBuffer buf = ByteBuffer.wrap(dst); + buf.position(ctx.copy(file, pos, dst, 0, cnt)); + return buf; + } + + private boolean notInCache(long pos) { + return cache.get(file.key, file.alignToBlock(pos)) == null; + } + + @Override + public long size() throws IOException { + long n = file.length; + if (n < 0) { + n = open().size(); + file.length = n; + } + return n; + } + + @Override + public void adviseSequentialRead(long start, long end) { + int sz = ctx.getOptions().getStreamPackBufferSize(); + if (sz > 0) { + readAhead = (int) Math.min(sz, end - start); + } + } + + private ReadableChannel open() throws IOException { + if (ch == null) { + ch = ctx.db.openFile(file.desc, file.ext); + } + return ch; + } + + @Override + public void close() { + if (ch != null) { + try { + ch.close(); + } catch (IOException e) { + // Ignore read close failures. + } finally { + ch = null; + } + } + } + } +}