|
|
@ -195,44 +195,7 @@ public final class DfsPackFile extends BlockBasedFile { |
|
|
|
try { |
|
|
|
try { |
|
|
|
DfsStreamKey idxKey = desc.getStreamKey(INDEX); |
|
|
|
DfsStreamKey idxKey = desc.getStreamKey(INDEX); |
|
|
|
DfsBlockCache.Ref<PackIndex> idxref = cache.getOrLoadRef(idxKey, |
|
|
|
DfsBlockCache.Ref<PackIndex> idxref = cache.getOrLoadRef(idxKey, |
|
|
|
() -> { |
|
|
|
() -> loadPackIndex(ctx, idxKey)); |
|
|
|
try { |
|
|
|
|
|
|
|
ctx.stats.readIdx++; |
|
|
|
|
|
|
|
long start = System.nanoTime(); |
|
|
|
|
|
|
|
try (ReadableChannel rc = ctx.db.openFile(desc, |
|
|
|
|
|
|
|
INDEX)) { |
|
|
|
|
|
|
|
InputStream in = Channels |
|
|
|
|
|
|
|
.newInputStream(rc); |
|
|
|
|
|
|
|
int wantSize = 8192; |
|
|
|
|
|
|
|
int bs = rc.blockSize(); |
|
|
|
|
|
|
|
if (0 < bs && bs < wantSize) { |
|
|
|
|
|
|
|
bs = (wantSize / bs) * bs; |
|
|
|
|
|
|
|
} else if (bs <= 0) { |
|
|
|
|
|
|
|
bs = wantSize; |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
PackIndex idx = PackIndex.read( |
|
|
|
|
|
|
|
new BufferedInputStream(in, bs)); |
|
|
|
|
|
|
|
int sz = (int) Math.min( |
|
|
|
|
|
|
|
idx.getObjectCount() * REC_SIZE, |
|
|
|
|
|
|
|
Integer.MAX_VALUE); |
|
|
|
|
|
|
|
ctx.stats.readIdxBytes += rc.position(); |
|
|
|
|
|
|
|
index = idx; |
|
|
|
|
|
|
|
return new DfsBlockCache.Ref<>(idxKey, 0, |
|
|
|
|
|
|
|
sz, idx); |
|
|
|
|
|
|
|
} finally { |
|
|
|
|
|
|
|
ctx.stats.readIdxMicros += elapsedMicros( |
|
|
|
|
|
|
|
start); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
} catch (EOFException e) { |
|
|
|
|
|
|
|
throw new IOException(MessageFormat.format( |
|
|
|
|
|
|
|
DfsText.get().shortReadOfIndex, |
|
|
|
|
|
|
|
desc.getFileName(INDEX)), e); |
|
|
|
|
|
|
|
} catch (IOException e) { |
|
|
|
|
|
|
|
throw new IOException(MessageFormat.format( |
|
|
|
|
|
|
|
DfsText.get().cannotReadIndex, |
|
|
|
|
|
|
|
desc.getFileName(INDEX)), e); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
}); |
|
|
|
|
|
|
|
PackIndex idx = idxref.get(); |
|
|
|
PackIndex idx = idxref.get(); |
|
|
|
if (index == null && idx != null) { |
|
|
|
if (index == null && idx != null) { |
|
|
|
index = idx; |
|
|
|
index = idx; |
|
|
@ -267,44 +230,9 @@ public final class DfsPackFile extends BlockBasedFile { |
|
|
|
PackIndex idx = idx(ctx); |
|
|
|
PackIndex idx = idx(ctx); |
|
|
|
PackReverseIndex revidx = getReverseIdx(ctx); |
|
|
|
PackReverseIndex revidx = getReverseIdx(ctx); |
|
|
|
DfsStreamKey bitmapKey = desc.getStreamKey(BITMAP_INDEX); |
|
|
|
DfsStreamKey bitmapKey = desc.getStreamKey(BITMAP_INDEX); |
|
|
|
DfsBlockCache.Ref<PackBitmapIndex> idxref = cache |
|
|
|
DfsBlockCache.Ref<PackBitmapIndex> idxref = cache.getOrLoadRef( |
|
|
|
.getOrLoadRef(bitmapKey, () -> { |
|
|
|
bitmapKey, |
|
|
|
ctx.stats.readBitmap++; |
|
|
|
() -> loadBitmapIndex(ctx, bitmapKey, idx, revidx)); |
|
|
|
long start = System.nanoTime(); |
|
|
|
|
|
|
|
try (ReadableChannel rc = ctx.db.openFile(desc, |
|
|
|
|
|
|
|
BITMAP_INDEX)) { |
|
|
|
|
|
|
|
long size; |
|
|
|
|
|
|
|
PackBitmapIndex bmidx; |
|
|
|
|
|
|
|
try { |
|
|
|
|
|
|
|
InputStream in = Channels.newInputStream(rc); |
|
|
|
|
|
|
|
int wantSize = 8192; |
|
|
|
|
|
|
|
int bs = rc.blockSize(); |
|
|
|
|
|
|
|
if (0 < bs && bs < wantSize) { |
|
|
|
|
|
|
|
bs = (wantSize / bs) * bs; |
|
|
|
|
|
|
|
} else if (bs <= 0) { |
|
|
|
|
|
|
|
bs = wantSize; |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
in = new BufferedInputStream(in, bs); |
|
|
|
|
|
|
|
bmidx = PackBitmapIndex.read(in, idx, revidx); |
|
|
|
|
|
|
|
} finally { |
|
|
|
|
|
|
|
size = rc.position(); |
|
|
|
|
|
|
|
ctx.stats.readIdxBytes += size; |
|
|
|
|
|
|
|
ctx.stats.readIdxMicros += elapsedMicros(start); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
int sz = (int) Math.min(size, Integer.MAX_VALUE); |
|
|
|
|
|
|
|
bitmapIndex = bmidx; |
|
|
|
|
|
|
|
return new DfsBlockCache.Ref<>(bitmapKey, 0, sz, |
|
|
|
|
|
|
|
bmidx); |
|
|
|
|
|
|
|
} catch (EOFException e) { |
|
|
|
|
|
|
|
throw new IOException(MessageFormat.format( |
|
|
|
|
|
|
|
DfsText.get().shortReadOfIndex, |
|
|
|
|
|
|
|
desc.getFileName(BITMAP_INDEX)), e); |
|
|
|
|
|
|
|
} catch (IOException e) { |
|
|
|
|
|
|
|
throw new IOException(MessageFormat.format( |
|
|
|
|
|
|
|
DfsText.get().cannotReadIndex, |
|
|
|
|
|
|
|
desc.getFileName(BITMAP_INDEX)), e); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
}); |
|
|
|
|
|
|
|
PackBitmapIndex bmidx = idxref.get(); |
|
|
|
PackBitmapIndex bmidx = idxref.get(); |
|
|
|
if (bitmapIndex == null && bmidx != null) { |
|
|
|
if (bitmapIndex == null && bmidx != null) { |
|
|
|
bitmapIndex = bmidx; |
|
|
|
bitmapIndex = bmidx; |
|
|
@ -326,14 +254,8 @@ public final class DfsPackFile extends BlockBasedFile { |
|
|
|
PackIndex idx = idx(ctx); |
|
|
|
PackIndex idx = idx(ctx); |
|
|
|
DfsStreamKey revKey = new DfsStreamKey.ForReverseIndex( |
|
|
|
DfsStreamKey revKey = new DfsStreamKey.ForReverseIndex( |
|
|
|
desc.getStreamKey(INDEX)); |
|
|
|
desc.getStreamKey(INDEX)); |
|
|
|
DfsBlockCache.Ref<PackReverseIndex> revref = cache |
|
|
|
DfsBlockCache.Ref<PackReverseIndex> revref = cache.getOrLoadRef( |
|
|
|
.getOrLoadRef(revKey, () -> { |
|
|
|
revKey, () -> loadReverseIdx(ctx, revKey, idx)); |
|
|
|
PackReverseIndex revidx = new PackReverseIndex(idx); |
|
|
|
|
|
|
|
int sz = (int) Math.min(idx.getObjectCount() * 8, |
|
|
|
|
|
|
|
Integer.MAX_VALUE); |
|
|
|
|
|
|
|
reverseIndex = revidx; |
|
|
|
|
|
|
|
return new DfsBlockCache.Ref<>(revKey, 0, sz, revidx); |
|
|
|
|
|
|
|
}); |
|
|
|
|
|
|
|
PackReverseIndex revidx = revref.get(); |
|
|
|
PackReverseIndex revidx = revref.get(); |
|
|
|
if (reverseIndex == null && revidx != null) { |
|
|
|
if (reverseIndex == null && revidx != null) { |
|
|
|
reverseIndex = revidx; |
|
|
|
reverseIndex = revidx; |
|
|
@ -1091,4 +1013,87 @@ public final class DfsPackFile extends BlockBasedFile { |
|
|
|
list.add(offset); |
|
|
|
list.add(offset); |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
private DfsBlockCache.Ref<PackIndex> loadPackIndex( |
|
|
|
|
|
|
|
DfsReader ctx, DfsStreamKey idxKey) throws IOException { |
|
|
|
|
|
|
|
try { |
|
|
|
|
|
|
|
ctx.stats.readIdx++; |
|
|
|
|
|
|
|
long start = System.nanoTime(); |
|
|
|
|
|
|
|
try (ReadableChannel rc = ctx.db.openFile(desc, INDEX)) { |
|
|
|
|
|
|
|
InputStream in = Channels.newInputStream(rc); |
|
|
|
|
|
|
|
int wantSize = 8192; |
|
|
|
|
|
|
|
int bs = rc.blockSize(); |
|
|
|
|
|
|
|
if (0 < bs && bs < wantSize) { |
|
|
|
|
|
|
|
bs = (wantSize / bs) * bs; |
|
|
|
|
|
|
|
} else if (bs <= 0) { |
|
|
|
|
|
|
|
bs = wantSize; |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
PackIndex idx = PackIndex.read(new BufferedInputStream(in, bs)); |
|
|
|
|
|
|
|
int sz = (int) Math.min( |
|
|
|
|
|
|
|
idx.getObjectCount() * REC_SIZE, |
|
|
|
|
|
|
|
Integer.MAX_VALUE); |
|
|
|
|
|
|
|
ctx.stats.readIdxBytes += rc.position(); |
|
|
|
|
|
|
|
index = idx; |
|
|
|
|
|
|
|
return new DfsBlockCache.Ref<>(idxKey, 0, sz, idx); |
|
|
|
|
|
|
|
} finally { |
|
|
|
|
|
|
|
ctx.stats.readIdxMicros += elapsedMicros(start); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
} catch (EOFException e) { |
|
|
|
|
|
|
|
throw new IOException(MessageFormat.format( |
|
|
|
|
|
|
|
DfsText.get().shortReadOfIndex, |
|
|
|
|
|
|
|
desc.getFileName(INDEX)), e); |
|
|
|
|
|
|
|
} catch (IOException e) { |
|
|
|
|
|
|
|
throw new IOException(MessageFormat.format( |
|
|
|
|
|
|
|
DfsText.get().cannotReadIndex, |
|
|
|
|
|
|
|
desc.getFileName(INDEX)), e); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
private DfsBlockCache.Ref<PackReverseIndex> loadReverseIdx( |
|
|
|
|
|
|
|
DfsReader ctx, DfsStreamKey revKey, PackIndex idx) { |
|
|
|
|
|
|
|
PackReverseIndex revidx = new PackReverseIndex(idx); |
|
|
|
|
|
|
|
int sz = (int) Math.min(idx.getObjectCount() * 8, Integer.MAX_VALUE); |
|
|
|
|
|
|
|
reverseIndex = revidx; |
|
|
|
|
|
|
|
return new DfsBlockCache.Ref<>(revKey, 0, sz, revidx); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
private DfsBlockCache.Ref<PackBitmapIndex> loadBitmapIndex( |
|
|
|
|
|
|
|
DfsReader ctx, |
|
|
|
|
|
|
|
DfsStreamKey bitmapKey, |
|
|
|
|
|
|
|
PackIndex idx, |
|
|
|
|
|
|
|
PackReverseIndex revidx) throws IOException { |
|
|
|
|
|
|
|
ctx.stats.readBitmap++; |
|
|
|
|
|
|
|
long start = System.nanoTime(); |
|
|
|
|
|
|
|
try (ReadableChannel rc = ctx.db.openFile(desc, BITMAP_INDEX)) { |
|
|
|
|
|
|
|
long size; |
|
|
|
|
|
|
|
PackBitmapIndex bmidx; |
|
|
|
|
|
|
|
try { |
|
|
|
|
|
|
|
InputStream in = Channels.newInputStream(rc); |
|
|
|
|
|
|
|
int wantSize = 8192; |
|
|
|
|
|
|
|
int bs = rc.blockSize(); |
|
|
|
|
|
|
|
if (0 < bs && bs < wantSize) { |
|
|
|
|
|
|
|
bs = (wantSize / bs) * bs; |
|
|
|
|
|
|
|
} else if (bs <= 0) { |
|
|
|
|
|
|
|
bs = wantSize; |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
in = new BufferedInputStream(in, bs); |
|
|
|
|
|
|
|
bmidx = PackBitmapIndex.read(in, idx, revidx); |
|
|
|
|
|
|
|
} finally { |
|
|
|
|
|
|
|
size = rc.position(); |
|
|
|
|
|
|
|
ctx.stats.readIdxBytes += size; |
|
|
|
|
|
|
|
ctx.stats.readIdxMicros += elapsedMicros(start); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
int sz = (int) Math.min(size, Integer.MAX_VALUE); |
|
|
|
|
|
|
|
bitmapIndex = bmidx; |
|
|
|
|
|
|
|
return new DfsBlockCache.Ref<>(bitmapKey, 0, sz, bmidx); |
|
|
|
|
|
|
|
} catch (EOFException e) { |
|
|
|
|
|
|
|
throw new IOException(MessageFormat.format( |
|
|
|
|
|
|
|
DfsText.get().shortReadOfIndex, |
|
|
|
|
|
|
|
desc.getFileName(BITMAP_INDEX)), e); |
|
|
|
|
|
|
|
} catch (IOException e) { |
|
|
|
|
|
|
|
throw new IOException(MessageFormat.format( |
|
|
|
|
|
|
|
DfsText.get().cannotReadIndex, |
|
|
|
|
|
|
|
desc.getFileName(BITMAP_INDEX)), e); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|