From 5b5c3a2e1da86819c5176766003d820c000d609e Mon Sep 17 00:00:00 2001 From: Dave Borowitz Date: Wed, 1 Nov 2017 10:19:38 -0400 Subject: [PATCH 1/5] ObjectDirectory: Remove last modified check in insertPack GC explicitly handles the case where a new pack has the same name as an existing pack due to it containing the exact same set of objects. In this case, the pack passed to insertPack will have the same name as an existing pack, but it will also almost certainly have a later mtime than the existing pack. The loop in insertPack tried to short-circuit when inserting a new pack, to avoid walking more of the pack list than necessary. Unfortunately, this means it will never get to the check for an identical name, resulting in a duplicate entry for the same PackFile in the pack list. Remove the short-circuit so that insertPack does not insert a duplicate entry. Change-Id: I00711b28594622ad3bd104332334e8a3592cda7f --- .../org/eclipse/jgit/internal/storage/file/ObjectDirectory.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java index d953b87c4..9629e36fa 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java @@ -814,8 +814,6 @@ public class ObjectDirectory extends FileObjectDatabase { final PackFile[] oldList = o.packs; final String name = pf.getPackFile().getName(); for (PackFile p : oldList) { - if (PackFile.SORT.compare(pf, p) < 0) - break; if (name.equals(p.getPackFile().getName())) return; } From 5ce1cc3d43a85acfaa2237968dd0d2a29beb07b0 Mon Sep 17 00:00:00 2001 From: Dave Borowitz Date: Wed, 1 Nov 2017 10:42:46 -0400 Subject: [PATCH 2/5] ObjectDirectory: Factor a method to close open pack handles Change-Id: I9490c4583fdf801de9c0bb09595ebb8fb4926988 --- .../eclipse/jgit/internal/storage/file/GC.java | 11 +---------- .../internal/storage/file/ObjectDirectory.java | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java index 8cd8ee152..f992a3340 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java @@ -1173,16 +1173,7 @@ public class GC { // rename the temporary files to real files File realPack = nameFor(id, ".pack"); //$NON-NLS-1$ - // if the packfile already exists (because we are rewriting a - // packfile for the same set of objects maybe with different - // PackConfig) then make sure we get rid of all handles on the file. - // Windows will not allow for rename otherwise. - if (realPack.exists()) - for (PackFile p : repo.getObjectDatabase().getPacks()) - if (realPack.getPath().equals(p.getPackFile().getPath())) { - p.close(); - break; - } + repo.getObjectDatabase().closeAllPackHandles(realPack); tmpPack.setReadOnly(); FileUtils.rename(tmpPack, realPack, StandardCopyOption.ATOMIC_MOVE); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java index 9629e36fa..7d351704d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java @@ -969,6 +969,21 @@ public class ObjectDirectory extends FileObjectDatabase { return nameSet; } + void closeAllPackHandles(File packFile) { + // if the packfile already exists (because we are rewriting a + // packfile for the same set of objects maybe with different + // PackConfig) then make sure we get rid of all handles on the file. + // Windows will not allow for rename otherwise. + if (packFile.exists()) { + for (PackFile p : getPacks()) { + if (packFile.getPath().equals(p.getPackFile().getPath())) { + p.close(); + break; + } + } + } + } + AlternateHandle[] myAlternates() { AlternateHandle[] alt = alternates.get(); if (alt == null) { From f7ceeaa23f98f671848a8446c324d0d96800d5db Mon Sep 17 00:00:00 2001 From: Dave Borowitz Date: Tue, 31 Oct 2017 10:05:29 -0400 Subject: [PATCH 3/5] FileRepository: Add pack-based inserter implementation Applications that use ObjectInserters to create lots of individual objects may prefer to avoid cluttering up the object directory with loose objects. Add a specialized inserter implementation that produces a single pack file no matter how many objects. This inserter is loosely based on the existing DfsInserter implementation, but is simpler since we don't need to buffer blocks in memory before writing to storage. An alternative for such applications would be to write out the loose objects and then repack just those objects later. This operation is not currently supported with the GC class, which always repacks existing packs when compacting loose objects. This in turn requires more CPU-intensive reachability checks and extra I/O to copy objects from old packs to new packs. So, the choice was between implementing a new variant of repack, or not writing loose objects in the first place. The latter approach is likely less code overall, and avoids unnecessary I/O at runtime. The current implementation does not yet support newReader() for reading back objects. Change-Id: I2074418f4e65853b7113de5eaced3a6b037d1a17 --- .../storage/file/PackInserterTest.java | 421 ++++++++++++++++++ .../storage/file/ObjectDirectory.java | 10 + .../internal/storage/file/PackInserter.java | 408 +++++++++++++++++ 3 files changed, 839 insertions(+) create mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackInserterTest.java create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackInserterTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackInserterTest.java new file mode 100644 index 000000000..1e9dd45b2 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackInserterTest.java @@ -0,0 +1,421 @@ +/* + * Copyright (C) 2017, Google Inc. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.eclipse.jgit.internal.storage.file; + +import static java.util.Comparator.comparing; +import static java.util.stream.Collectors.toList; +import static org.eclipse.jgit.lib.Constants.OBJ_BLOB; +import static org.eclipse.jgit.lib.Constants.OBJ_COMMIT; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.lessThan; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.function.Predicate; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.eclipse.jgit.dircache.DirCache; +import org.eclipse.jgit.dircache.DirCacheBuilder; +import org.eclipse.jgit.dircache.DirCacheEntry; +import org.eclipse.jgit.junit.RepositoryTestCase; +import org.eclipse.jgit.lib.CommitBuilder; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.FileMode; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectLoader; +import org.eclipse.jgit.lib.ObjectReader; +import org.eclipse.jgit.treewalk.CanonicalTreeParser; +import org.junit.Before; +import org.junit.Test; + +@SuppressWarnings("boxing") +public class PackInserterTest extends RepositoryTestCase { + @Before + public void emptyAtSetUp() throws Exception { + assertEquals(0, listPacks().size()); + assertNoObjects(); + } + + @Test + public void noFlush() throws Exception { + try (PackInserter ins = newInserter()) { + ins.insert(OBJ_BLOB, Constants.encode("foo contents")); + // No flush. + } + assertNoObjects(); + } + + @Test + public void flushEmptyPack() throws Exception { + try (PackInserter ins = newInserter()) { + ins.flush(); + } + assertNoObjects(); + } + + @Test + public void singlePack() throws Exception { + ObjectId blobId; + byte[] blob = Constants.encode("foo contents"); + ObjectId treeId; + ObjectId commitId; + byte[] commit; + try (PackInserter ins = newInserter()) { + blobId = ins.insert(OBJ_BLOB, blob); + + DirCache dc = DirCache.newInCore(); + DirCacheBuilder b = dc.builder(); + DirCacheEntry dce = new DirCacheEntry("foo"); + dce.setFileMode(FileMode.REGULAR_FILE); + dce.setObjectId(blobId); + b.add(dce); + b.finish(); + treeId = dc.writeTree(ins); + + CommitBuilder cb = new CommitBuilder(); + cb.setTreeId(treeId); + cb.setAuthor(author); + cb.setCommitter(committer); + cb.setMessage("Commit message"); + commit = cb.toByteArray(); + commitId = ins.insert(cb); + ins.flush(); + } + + assertPacksOnly(); + List packs = listPacks(); + assertEquals(1, packs.size()); + assertEquals(3, packs.get(0).getObjectCount()); + + try (ObjectReader reader = db.newObjectReader()) { + assertBlob(reader, blobId, blob); + + CanonicalTreeParser treeParser = + new CanonicalTreeParser(null, reader, treeId); + assertEquals("foo", treeParser.getEntryPathString()); + assertEquals(blobId, treeParser.getEntryObjectId()); + + ObjectLoader commitLoader = reader.open(commitId); + assertEquals(OBJ_COMMIT, commitLoader.getType()); + assertArrayEquals(commit, commitLoader.getBytes()); + } + } + + @Test + public void multiplePacks() throws Exception { + ObjectId blobId1; + ObjectId blobId2; + byte[] blob1 = Constants.encode("blob1"); + byte[] blob2 = Constants.encode("blob2"); + + try (PackInserter ins = newInserter()) { + blobId1 = ins.insert(OBJ_BLOB, blob1); + ins.flush(); + blobId2 = ins.insert(OBJ_BLOB, blob2); + ins.flush(); + } + + assertPacksOnly(); + List packs = listPacks(); + assertEquals(2, packs.size()); + assertEquals(1, packs.get(0).getObjectCount()); + assertEquals(1, packs.get(1).getObjectCount()); + + try (ObjectReader reader = db.newObjectReader()) { + assertBlob(reader, blobId1, blob1); + assertBlob(reader, blobId2, blob2); + } + } + + @Test + public void largeBlob() throws Exception { + ObjectId blobId; + byte[] blob = newLargeBlob(); + try (PackInserter ins = newInserter()) { + assertThat(blob.length, greaterThan(ins.getBufferSize())); + blobId = + ins.insert(OBJ_BLOB, blob.length, new ByteArrayInputStream(blob)); + ins.flush(); + } + + assertPacksOnly(); + Collection packs = listPacks(); + assertEquals(1, packs.size()); + PackFile p = packs.iterator().next(); + assertEquals(1, p.getObjectCount()); + + try (ObjectReader reader = db.newObjectReader()) { + assertBlob(reader, blobId, blob); + } + } + + @Test + public void overwriteExistingPack() throws Exception { + ObjectId blobId; + byte[] blob = Constants.encode("foo contents"); + + try (PackInserter ins = newInserter()) { + blobId = ins.insert(OBJ_BLOB, blob); + ins.flush(); + } + + assertPacksOnly(); + List packs = listPacks(); + assertEquals(1, packs.size()); + PackFile pack = packs.get(0); + assertEquals(1, pack.getObjectCount()); + + String inode = getInode(pack.getPackFile()); + + try (PackInserter ins = newInserter()) { + ins.checkExisting(false); + assertEquals(blobId, ins.insert(OBJ_BLOB, blob)); + ins.flush(); + } + + assertPacksOnly(); + packs = listPacks(); + assertEquals(1, packs.size()); + pack = packs.get(0); + assertEquals(1, pack.getObjectCount()); + + if (inode != null) { + // Old file was overwritten with new file, although objects were + // equivalent. + assertNotEquals(inode, getInode(pack.getPackFile())); + } + } + + @Test + public void checkExisting() throws Exception { + ObjectId blobId; + byte[] blob = Constants.encode("foo contents"); + + try (PackInserter ins = newInserter()) { + blobId = ins.insert(OBJ_BLOB, blob); + ins.insert(OBJ_BLOB, Constants.encode("another blob")); + ins.flush(); + } + + assertPacksOnly(); + assertEquals(1, listPacks().size()); + + try (PackInserter ins = newInserter()) { + assertEquals(blobId, ins.insert(OBJ_BLOB, blob)); + ins.flush(); + } + + assertPacksOnly(); + assertEquals(1, listPacks().size()); + + try (PackInserter ins = newInserter()) { + ins.checkExisting(false); + assertEquals(blobId, ins.insert(OBJ_BLOB, blob)); + ins.flush(); + } + + assertPacksOnly(); + assertEquals(2, listPacks().size()); + + try (ObjectReader reader = db.newObjectReader()) { + assertBlob(reader, blobId, blob); + } + } + + @Test + public void insertSmallInputStreamRespectsCheckExisting() throws Exception { + ObjectId blobId; + byte[] blob = Constants.encode("foo contents"); + try (PackInserter ins = newInserter()) { + assertThat(blob.length, lessThan(ins.getBufferSize())); + blobId = ins.insert(OBJ_BLOB, blob); + ins.insert(OBJ_BLOB, Constants.encode("another blob")); + ins.flush(); + } + + assertPacksOnly(); + assertEquals(1, listPacks().size()); + + try (PackInserter ins = newInserter()) { + assertEquals(blobId, + ins.insert(OBJ_BLOB, blob.length, new ByteArrayInputStream(blob))); + ins.flush(); + } + + assertPacksOnly(); + assertEquals(1, listPacks().size()); + } + + @Test + public void insertLargeInputStreamBypassesCheckExisting() throws Exception { + ObjectId blobId; + byte[] blob = newLargeBlob(); + + try (PackInserter ins = newInserter()) { + assertThat(blob.length, greaterThan(ins.getBufferSize())); + blobId = ins.insert(OBJ_BLOB, blob); + ins.insert(OBJ_BLOB, Constants.encode("another blob")); + ins.flush(); + } + + assertPacksOnly(); + assertEquals(1, listPacks().size()); + + try (PackInserter ins = newInserter()) { + assertEquals(blobId, + ins.insert(OBJ_BLOB, blob.length, new ByteArrayInputStream(blob))); + ins.flush(); + } + + assertPacksOnly(); + assertEquals(2, listPacks().size()); + } + + private List listPacks() throws Exception { + List fromOpenDb = listPacks(db); + List reopened; + try (FileRepository db2 = new FileRepository(db.getDirectory())) { + reopened = listPacks(db2); + } + assertEquals(fromOpenDb.size(), reopened.size()); + for (int i = 0 ; i < fromOpenDb.size(); i++) { + PackFile a = fromOpenDb.get(i); + PackFile b = reopened.get(i); + assertEquals(a.getPackName(), b.getPackName()); + assertEquals( + a.getPackFile().getAbsolutePath(), b.getPackFile().getAbsolutePath()); + assertEquals(a.getObjectCount(), b.getObjectCount()); + a.getObjectCount(); + } + return fromOpenDb; + } + + private static List listPacks(FileRepository db) throws Exception { + return db.getObjectDatabase().getPacks().stream() + .sorted(comparing(PackFile::getPackName)).collect(toList()); + } + + private PackInserter newInserter() { + return db.getObjectDatabase().newPackInserter(); + } + + private static byte[] newLargeBlob() { + byte[] blob = new byte[10240]; + for (int i = 0; i < blob.length; i++) { + blob[i] = (byte) ('0' + (i % 10)); + } + return blob; + } + + private static String getInode(File f) throws Exception { + BasicFileAttributes attrs = Files.readAttributes( + f.toPath(), BasicFileAttributes.class); + Object k = attrs.fileKey(); + if (k == null) { + return null; + } + Pattern p = Pattern.compile("^\\(dev=[^,]*,ino=(\\d+)\\)$"); + Matcher m = p.matcher(k.toString()); + return m.matches() ? m.group(1) : null; + } + + private static void assertBlob(ObjectReader reader, ObjectId id, + byte[] expected) throws Exception { + ObjectLoader loader = reader.open(id); + assertEquals(OBJ_BLOB, loader.getType()); + assertEquals(expected.length, loader.getSize()); + assertArrayEquals(expected, loader.getBytes()); + } + + private void assertPacksOnly() throws Exception { + new BadFileCollector(f -> !f.endsWith(".pack") && !f.endsWith(".idx")) + .assertNoBadFiles(db.getObjectDatabase().getDirectory()); + } + + private void assertNoObjects() throws Exception { + new BadFileCollector(f -> true) + .assertNoBadFiles(db.getObjectDatabase().getDirectory()); + } + + private static class BadFileCollector extends SimpleFileVisitor { + private final Predicate badName; + private List bad; + + BadFileCollector(Predicate badName) { + this.badName = badName; + } + + void assertNoBadFiles(File f) throws IOException { + bad = new ArrayList<>(); + Files.walkFileTree(f.toPath(), this); + if (!bad.isEmpty()) { + fail("unexpected files in object directory: " + bad); + } + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { + String name = file.getFileName().toString(); + if (!attrs.isDirectory() && badName.test(name)) { + bad.add(name); + } + return FileVisitResult.CONTINUE; + } + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java index 7d351704d..153c7dd92 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java @@ -220,6 +220,16 @@ public class ObjectDirectory extends FileObjectDatabase { return new ObjectDirectoryInserter(this, config); } + /** + * Create a new inserter that inserts all objects as pack files, not loose + * objects. + * + * @return new inserter. + */ + public PackInserter newPackInserter() { + return new PackInserter(this); + } + @Override public void close() { unpackedObjectCache.clear(); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java new file mode 100644 index 000000000..25baad04f --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java @@ -0,0 +1,408 @@ +/* + * Copyright (C) 2017, Google Inc. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.eclipse.jgit.internal.storage.file; + +import static java.nio.file.StandardCopyOption.ATOMIC_MOVE; +import static org.eclipse.jgit.lib.Constants.OBJECT_ID_LENGTH; + +import java.io.EOFException; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.RandomAccessFile; +import java.text.MessageFormat; +import java.util.Collections; +import java.util.List; +import java.util.zip.CRC32; +import java.util.zip.Deflater; +import java.util.zip.DeflaterOutputStream; + +import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectIdOwnerMap; +import org.eclipse.jgit.lib.ObjectInserter; +import org.eclipse.jgit.lib.ObjectReader; +import org.eclipse.jgit.transport.PackParser; +import org.eclipse.jgit.transport.PackedObjectInfo; +import org.eclipse.jgit.util.BlockList; +import org.eclipse.jgit.util.FileUtils; +import org.eclipse.jgit.util.IO; +import org.eclipse.jgit.util.NB; +import org.eclipse.jgit.util.io.CountingOutputStream; +import org.eclipse.jgit.util.sha1.SHA1; + +/** + * Object inserter that inserts one pack per call to {@link #flush()}, and never + * inserts loose objects. + */ +class PackInserter extends ObjectInserter { + /** Always produce version 2 indexes, to get CRC data. */ + private static final int INDEX_VERSION = 2; + + private final ObjectDirectory db; + + private List objectList; + private ObjectIdOwnerMap objectMap; + private boolean rollback; + private boolean checkExisting = true; + + private int compression = Deflater.BEST_COMPRESSION; + private File tmpPack; + private PackStream packOut; + + PackInserter(ObjectDirectory db) { + this.db = db; + } + + /** + * @param check + * if false, will write out possibly-duplicate objects without + * first checking whether they exist in the repo; default is true. + */ + public void checkExisting(boolean check) { + checkExisting = check; + } + + /** + * @param compression + * compression level for zlib deflater. + */ + public void setCompressionLevel(int compression) { + this.compression = compression; + } + + int getBufferSize() { + return buffer().length; + } + + @Override + public ObjectId insert(int type, byte[] data, int off, int len) + throws IOException { + ObjectId id = idFor(type, data, off, len); + if (objectMap != null && objectMap.contains(id)) { + return id; + } + // Ignore loose objects, which are potentially unreachable. + if (checkExisting && db.hasPackedObject(id)) { + return id; + } + + long offset = beginObject(type, len); + packOut.compress.write(data, off, len); + packOut.compress.finish(); + return endObject(id, offset); + } + + @Override + public ObjectId insert(int type, long len, InputStream in) + throws IOException { + byte[] buf = buffer(); + if (len <= buf.length) { + IO.readFully(in, buf, 0, (int) len); + return insert(type, buf, 0, (int) len); + } + + long offset = beginObject(type, len); + SHA1 md = digest(); + md.update(Constants.encodedTypeString(type)); + md.update((byte) ' '); + md.update(Constants.encodeASCII(len)); + md.update((byte) 0); + + while (0 < len) { + int n = in.read(buf, 0, (int) Math.min(buf.length, len)); + if (n <= 0) { + throw new EOFException(); + } + md.update(buf, 0, n); + packOut.compress.write(buf, 0, n); + len -= n; + } + packOut.compress.finish(); + return endObject(md.toObjectId(), offset); + } + + private long beginObject(int type, long len) throws IOException { + if (packOut == null) { + beginPack(); + } + long offset = packOut.getOffset(); + packOut.beginObject(type, len); + return offset; + } + + private ObjectId endObject(ObjectId id, long offset) { + PackedObjectInfo obj = new PackedObjectInfo(id); + obj.setOffset(offset); + obj.setCRC((int) packOut.crc32.getValue()); + objectList.add(obj); + objectMap.addIfAbsent(obj); + return id; + } + + private static File idxFor(File packFile) { + String p = packFile.getName(); + return new File( + packFile.getParentFile(), + p.substring(0, p.lastIndexOf('.')) + ".idx"); //$NON-NLS-1$ + } + + private void beginPack() throws IOException { + objectList = new BlockList<>(); + objectMap = new ObjectIdOwnerMap<>(); + + rollback = true; + tmpPack = File.createTempFile("insert_", ".pack", db.getDirectory()); //$NON-NLS-1$ //$NON-NLS-2$ + packOut = new PackStream(tmpPack); + + // Write the header as though it were a single object pack. + packOut.write(packOut.hdrBuf, 0, writePackHeader(packOut.hdrBuf, 1)); + } + + private static int writePackHeader(byte[] buf, int objectCount) { + System.arraycopy(Constants.PACK_SIGNATURE, 0, buf, 0, 4); + NB.encodeInt32(buf, 4, 2); // Always use pack version 2. + NB.encodeInt32(buf, 8, objectCount); + return 12; + } + + @Override + public PackParser newPackParser(InputStream in) { + throw new UnsupportedOperationException(); + } + + @Override + public ObjectReader newReader() { + throw new UnsupportedOperationException(); + } + + @Override + public void flush() throws IOException { + if (tmpPack == null) { + return; + } + + if (packOut == null) { + throw new IOException(); + } + + byte[] packHash; + try { + packHash = packOut.finishPack(); + } finally { + packOut.close(); + packOut = null; + } + + Collections.sort(objectList); + File tmpIdx = idxFor(tmpPack); + writePackIndex(tmpIdx, packHash, objectList); + + File realPack = new File( + new File(db.getDirectory(), "pack"), //$NON-NLS-1$ + "pack-" + computeName(objectList).name() + ".pack"); //$NON-NLS-1$ //$NON-NLS-2$ + db.closeAllPackHandles(realPack); + tmpPack.setReadOnly(); + FileUtils.rename(tmpPack, realPack, ATOMIC_MOVE); + + File realIdx = idxFor(realPack); + tmpIdx.setReadOnly(); + try { + FileUtils.rename(tmpIdx, realIdx, ATOMIC_MOVE); + } catch (IOException e) { + File newIdx = new File( + realIdx.getParentFile(), realIdx.getName() + ".new"); //$NON-NLS-1$ + try { + FileUtils.rename(tmpIdx, newIdx, ATOMIC_MOVE); + } catch (IOException e2) { + newIdx = tmpIdx; + e = e2; + } + throw new IOException(MessageFormat.format( + JGitText.get().panicCantRenameIndexFile, newIdx, + realIdx), e); + } + + db.openPack(realPack); + rollback = false; + clear(); + } + + private static void writePackIndex(File idx, byte[] packHash, + List list) throws IOException { + try (OutputStream os = new FileOutputStream(idx)) { + PackIndexWriter w = PackIndexWriter.createVersion(os, INDEX_VERSION); + w.write(list, packHash); + } + } + + private ObjectId computeName(List list) { + SHA1 md = digest().reset(); + byte[] buf = buffer(); + for (PackedObjectInfo otp : list) { + otp.copyRawTo(buf, 0); + md.update(buf, 0, OBJECT_ID_LENGTH); + } + return ObjectId.fromRaw(md.digest()); + } + + @Override + public void close() { + try { + if (packOut != null) { + try { + packOut.close(); + } catch (IOException err) { + // Ignore a close failure, the pack should be removed. + } + } + if (rollback && tmpPack != null) { + try { + FileUtils.delete(tmpPack); + } catch (IOException e) { + // Still delete idx. + } + try { + FileUtils.delete(idxFor(tmpPack)); + } catch (IOException e) { + // Ignore error deleting temp idx. + } + rollback = false; + } + } finally { + clear(); + } + } + + private void clear() { + objectList = null; + objectMap = null; + tmpPack = null; + packOut = null; + } + + private class PackStream extends OutputStream { + final byte[] hdrBuf; + final CRC32 crc32; + final DeflaterOutputStream compress; + + private final RandomAccessFile file; + private final CountingOutputStream out; + private final Deflater deflater; + + PackStream(File pack) throws IOException { + file = new RandomAccessFile(pack, "rw"); //$NON-NLS-1$ + out = new CountingOutputStream(new FileOutputStream(file.getFD())); + deflater = new Deflater(compression); + compress = new DeflaterOutputStream(this, deflater, 8192); + hdrBuf = new byte[32]; + crc32 = new CRC32(); + } + + long getOffset() { + return out.getCount(); + } + + void beginObject(int objectType, long length) throws IOException { + crc32.reset(); + deflater.reset(); + write(hdrBuf, 0, encodeTypeSize(objectType, length)); + } + + private int encodeTypeSize(int type, long rawLength) { + long nextLength = rawLength >>> 4; + hdrBuf[0] = (byte) ((nextLength > 0 ? 0x80 : 0x00) | (type << 4) | (rawLength & 0x0F)); + rawLength = nextLength; + int n = 1; + while (rawLength > 0) { + nextLength >>>= 7; + hdrBuf[n++] = (byte) ((nextLength > 0 ? 0x80 : 0x00) | (rawLength & 0x7F)); + rawLength = nextLength; + } + return n; + } + + @Override + public void write(final int b) throws IOException { + hdrBuf[0] = (byte) b; + write(hdrBuf, 0, 1); + } + + @Override + public void write(byte[] data, int off, int len) throws IOException { + crc32.update(data, off, len); + out.write(data, off, len); + } + + byte[] finishPack() throws IOException { + // Overwrite placeholder header with actual object count, then hash. + file.seek(0); + write(hdrBuf, 0, writePackHeader(hdrBuf, objectList.size())); + + byte[] buf = buffer(); + SHA1 md = digest().reset(); + file.seek(0); + while (true) { + int r = file.read(buf); + if (r < 0) { + break; + } + md.update(buf, 0, r); + } + byte[] packHash = md.digest(); + out.write(packHash, 0, packHash.length); + return packHash; + } + + @Override + public void close() throws IOException { + deflater.end(); + out.close(); + file.close(); + } + } +} From 080b4770e7212ddb7cc45f41c65138a8b3e36eb0 Mon Sep 17 00:00:00 2001 From: Dave Borowitz Date: Wed, 1 Nov 2017 11:45:22 -0400 Subject: [PATCH 4/5] Move some strings from DfsText to JGitText Change-Id: I60050e5127d12b6139d81859dba929fcfaabe504 --- .../org/eclipse/jgit/internal/JGitText.properties | 2 ++ .../jgit/internal/storage/dfs/DfsText.properties | 2 -- .../src/org/eclipse/jgit/internal/JGitText.java | 2 ++ .../eclipse/jgit/internal/storage/dfs/DfsInserter.java | 10 +++++----- .../org/eclipse/jgit/internal/storage/dfs/DfsText.java | 2 -- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties index 43dd9482d..0b9a24779 100644 --- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties +++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties @@ -88,6 +88,7 @@ cannotParseDate=The date specification "{0}" could not be parsed with the follow cannotParseGitURIish=Cannot parse Git URI-ish cannotPullOnARepoWithState=Cannot pull into a repository with state: {0} cannotRead=Cannot read {0} +cannotReadBackDelta=Cannot read delta type {0} cannotReadBlob=Cannot read blob {0} cannotReadCommit=Cannot read commit {0} cannotReadFile=Cannot read file {0} @@ -684,6 +685,7 @@ unencodeableFile=Unencodable file: {0} unexpectedCompareResult=Unexpected metadata comparison result: {0} unexpectedEndOfConfigFile=Unexpected end of config file unexpectedEndOfInput=Unexpected end of input +unexpectedEofInPack=Unexpected EOF in partially created pack unexpectedHunkTrailer=Unexpected hunk trailer unexpectedOddResult=odd: {0} + {1} - {2} unexpectedRefReport={0}: unexpected ref report: {1} diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/storage/dfs/DfsText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/storage/dfs/DfsText.properties index 4bbc4cc71..2c4bd06a3 100644 --- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/storage/dfs/DfsText.properties +++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/storage/dfs/DfsText.properties @@ -1,6 +1,4 @@ cannotReadIndex=Cannot read index {0} -cannotReadBackDelta=Cannot read delta type {0} shortReadOfBlock=Short read of block at {0} in pack {1}; expected {2} bytes, received only {3} shortReadOfIndex=Short read of index {0} -unexpectedEofInPack=Unexpected EOF in partially created pack willNotStoreEmptyPack=Cannot store empty pack diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java index 6b3631601..4225dba09 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java @@ -147,6 +147,7 @@ public class JGitText extends TranslationBundle { /***/ public String cannotParseGitURIish; /***/ public String cannotPullOnARepoWithState; /***/ public String cannotRead; + /***/ public String cannotReadBackDelta; /***/ public String cannotReadBlob; /***/ public String cannotReadCommit; /***/ public String cannotReadFile; @@ -743,6 +744,7 @@ public class JGitText extends TranslationBundle { /***/ public String unexpectedCompareResult; /***/ public String unexpectedEndOfConfigFile; /***/ public String unexpectedEndOfInput; + /***/ public String unexpectedEofInPack; /***/ public String unexpectedHunkTrailer; /***/ public String unexpectedOddResult; /***/ public String unexpectedRefReport; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsInserter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsInserter.java index 01654d45d..19e86522c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsInserter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsInserter.java @@ -500,7 +500,7 @@ public class DfsInserter extends ObjectInserter { inf.setInput(currBuf, s, n); return n; } - throw new EOFException(DfsText.get().unexpectedEofInPack); + throw new EOFException(JGitText.get().unexpectedEofInPack); } private DfsBlock getOrLoadBlock(long pos) throws IOException { @@ -513,7 +513,7 @@ public class DfsInserter extends ObjectInserter { for (int p = 0; p < blockSize;) { int n = out.read(s + p, ByteBuffer.wrap(d, p, blockSize - p)); if (n <= 0) - throw new EOFException(DfsText.get().unexpectedEofInPack); + throw new EOFException(JGitText.get().unexpectedEofInPack); p += n; } b = new DfsBlock(packKey, s, d); @@ -569,13 +569,13 @@ public class DfsInserter extends ObjectInserter { byte[] buf = buffer(); int cnt = packOut.read(obj.getOffset(), buf, 0, 20); if (cnt <= 0) - throw new EOFException(DfsText.get().unexpectedEofInPack); + throw new EOFException(JGitText.get().unexpectedEofInPack); int c = buf[0] & 0xff; int type = (c >> 4) & 7; if (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) throw new IOException(MessageFormat.format( - DfsText.get().cannotReadBackDelta, Integer.toString(type))); + JGitText.get().cannotReadBackDelta, Integer.toString(type))); if (typeHint != OBJ_ANY && type != typeHint) { throw new IncorrectObjectTypeException(objectId.copy(), typeHint); } @@ -585,7 +585,7 @@ public class DfsInserter extends ObjectInserter { int shift = 4; while ((c & 0x80) != 0) { if (ptr >= cnt) - throw new EOFException(DfsText.get().unexpectedEofInPack); + throw new EOFException(JGitText.get().unexpectedEofInPack); c = buf[ptr++] & 0xff; sz += ((long) (c & 0x7f)) << shift; shift += 7; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsText.java index 862454759..dedcab04d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsText.java @@ -55,9 +55,7 @@ public class DfsText extends TranslationBundle { // @formatter:off /***/ public String cannotReadIndex; - /***/ public String cannotReadBackDelta; /***/ public String shortReadOfBlock; /***/ public String shortReadOfIndex; - /***/ public String unexpectedEofInPack; /***/ public String willNotStoreEmptyPack; } From 678c99c057786e5d285a222f7dcb6250ccc5a433 Mon Sep 17 00:00:00 2001 From: Dave Borowitz Date: Wed, 1 Nov 2017 11:46:48 -0400 Subject: [PATCH 5/5] PackInserter: Implement newReader() Change-Id: Ib9e7f6439332eaed3d936f895a5271a7d514d3e9 --- .../storage/file/PackInserterTest.java | 120 ++++++++- .../internal/storage/file/PackInserter.java | 233 +++++++++++++++++- 2 files changed, 351 insertions(+), 2 deletions(-) diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackInserterTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackInserterTest.java index 1e9dd45b2..17f04c854 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackInserterTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackInserterTest.java @@ -45,6 +45,7 @@ package org.eclipse.jgit.internal.storage.file; import static java.util.Comparator.comparing; import static java.util.stream.Collectors.toList; + import static org.eclipse.jgit.lib.Constants.OBJ_BLOB; import static org.eclipse.jgit.lib.Constants.OBJ_COMMIT; import static org.hamcrest.Matchers.greaterThan; @@ -73,6 +74,7 @@ import java.util.regex.Pattern; import org.eclipse.jgit.dircache.DirCache; import org.eclipse.jgit.dircache.DirCacheBuilder; import org.eclipse.jgit.dircache.DirCacheEntry; +import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.junit.RepositoryTestCase; import org.eclipse.jgit.lib.CommitBuilder; import org.eclipse.jgit.lib.Constants; @@ -80,12 +82,29 @@ import org.eclipse.jgit.lib.FileMode; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectLoader; import org.eclipse.jgit.lib.ObjectReader; +import org.eclipse.jgit.lib.ObjectStream; +import org.eclipse.jgit.storage.file.WindowCacheConfig; import org.eclipse.jgit.treewalk.CanonicalTreeParser; +import org.eclipse.jgit.util.IO; +import org.junit.After; import org.junit.Before; import org.junit.Test; @SuppressWarnings("boxing") public class PackInserterTest extends RepositoryTestCase { + private WindowCacheConfig origWindowCacheConfig; + + @Before + public void setWindowCacheConfig() { + origWindowCacheConfig = new WindowCacheConfig(); + origWindowCacheConfig.install(); + } + + @After + public void resetWindowCacheConfig() { + origWindowCacheConfig.install(); + } + @Before public void emptyAtSetUp() throws Exception { assertEquals(0, listPacks().size()); @@ -327,6 +346,100 @@ public class PackInserterTest extends RepositoryTestCase { assertEquals(2, listPacks().size()); } + @Test + public void readBackSmallFiles() throws Exception { + ObjectId blobId1; + ObjectId blobId2; + ObjectId blobId3; + byte[] blob1 = Constants.encode("blob1"); + byte[] blob2 = Constants.encode("blob2"); + byte[] blob3 = Constants.encode("blob3"); + try (PackInserter ins = newInserter()) { + assertThat(blob1.length, lessThan(ins.getBufferSize())); + blobId1 = ins.insert(OBJ_BLOB, blob1); + + try (ObjectReader reader = ins.newReader()) { + assertBlob(reader, blobId1, blob1); + } + + // Read-back should not mess up the file pointer. + blobId2 = ins.insert(OBJ_BLOB, blob2); + ins.flush(); + + blobId3 = ins.insert(OBJ_BLOB, blob3); + } + + assertPacksOnly(); + List packs = listPacks(); + assertEquals(1, packs.size()); + assertEquals(2, packs.get(0).getObjectCount()); + + try (ObjectReader reader = db.newObjectReader()) { + assertBlob(reader, blobId1, blob1); + assertBlob(reader, blobId2, blob2); + + try { + reader.open(blobId3); + fail("Expected MissingObjectException"); + } catch (MissingObjectException expected) { + // Expected. + } + } + } + + @Test + public void readBackLargeFile() throws Exception { + ObjectId blobId; + byte[] blob = newLargeBlob(); + + WindowCacheConfig wcc = new WindowCacheConfig(); + wcc.setStreamFileThreshold(1024); + wcc.install(); + try (ObjectReader reader = db.newObjectReader()) { + assertThat(blob.length, greaterThan(reader.getStreamFileThreshold())); + } + + try (PackInserter ins = newInserter()) { + blobId = ins.insert(OBJ_BLOB, blob); + + try (ObjectReader reader = ins.newReader()) { + // Double-check threshold is propagated. + assertThat(blob.length, greaterThan(reader.getStreamFileThreshold())); + assertBlob(reader, blobId, blob); + } + } + + assertPacksOnly(); + // Pack was streamed out to disk and read back from the temp file, but + // ultimately rolled back and deleted. + assertEquals(0, listPacks().size()); + + try (ObjectReader reader = db.newObjectReader()) { + try { + reader.open(blobId); + fail("Expected MissingObjectException"); + } catch (MissingObjectException expected) { + // Expected. + } + } + } + + @Test + public void readBackFallsBackToRepo() throws Exception { + ObjectId blobId; + byte[] blob = Constants.encode("foo contents"); + try (PackInserter ins = newInserter()) { + assertThat(blob.length, lessThan(ins.getBufferSize())); + blobId = ins.insert(OBJ_BLOB, blob); + ins.flush(); + } + + try (PackInserter ins = newInserter(); + ObjectReader reader = ins.newReader()) { + assertBlob(reader, blobId, blob); + } + } + private List listPacks() throws Exception { List fromOpenDb = listPacks(db); List reopened; @@ -380,7 +493,12 @@ public class PackInserterTest extends RepositoryTestCase { ObjectLoader loader = reader.open(id); assertEquals(OBJ_BLOB, loader.getType()); assertEquals(expected.length, loader.getSize()); - assertArrayEquals(expected, loader.getBytes()); + try (ObjectStream s = loader.openStream()) { + int n = (int) s.getSize(); + byte[] actual = new byte[n]; + assertEquals(n, IO.readFully(s, actual, 0)); + assertArrayEquals(expected, actual); + } } private void assertPacksOnly() throws Exception { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java index 25baad04f..dd83e251f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java @@ -45,7 +45,10 @@ package org.eclipse.jgit.internal.storage.file; import static java.nio.file.StandardCopyOption.ATOMIC_MOVE; import static org.eclipse.jgit.lib.Constants.OBJECT_ID_LENGTH; +import static org.eclipse.jgit.lib.Constants.OBJ_OFS_DELTA; +import static org.eclipse.jgit.lib.Constants.OBJ_REF_DELTA; +import java.io.BufferedInputStream; import java.io.EOFException; import java.io.File; import java.io.FileOutputStream; @@ -53,19 +56,35 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.RandomAccessFile; +import java.nio.channels.Channels; import java.text.MessageFormat; +import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.zip.CRC32; +import java.util.zip.DataFormatException; import java.util.zip.Deflater; import java.util.zip.DeflaterOutputStream; +import java.util.zip.Inflater; +import java.util.zip.InflaterInputStream; +import org.eclipse.jgit.errors.CorruptObjectException; +import org.eclipse.jgit.errors.IncorrectObjectTypeException; +import org.eclipse.jgit.errors.LargeObjectException; +import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.lib.AbbreviatedObjectId; +import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.InflaterCache; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectIdOwnerMap; import org.eclipse.jgit.lib.ObjectInserter; +import org.eclipse.jgit.lib.ObjectLoader; import org.eclipse.jgit.lib.ObjectReader; +import org.eclipse.jgit.lib.ObjectStream; import org.eclipse.jgit.transport.PackParser; import org.eclipse.jgit.transport.PackedObjectInfo; import org.eclipse.jgit.util.BlockList; @@ -93,6 +112,7 @@ class PackInserter extends ObjectInserter { private int compression = Deflater.BEST_COMPRESSION; private File tmpPack; private PackStream packOut; + private Inflater cachedInflater; PackInserter(ObjectDirectory db) { this.db = db; @@ -217,7 +237,7 @@ class PackInserter extends ObjectInserter { @Override public ObjectReader newReader() { - throw new UnsupportedOperationException(); + return new Reader(); } @Override @@ -315,6 +335,11 @@ class PackInserter extends ObjectInserter { } } finally { clear(); + try { + InflaterCache.release(cachedInflater); + } finally { + cachedInflater = null; + } } } @@ -325,6 +350,15 @@ class PackInserter extends ObjectInserter { packOut = null; } + private Inflater inflater() { + if (cachedInflater == null) { + cachedInflater = InflaterCache.get(); + } else { + cachedInflater.reset(); + } + return cachedInflater; + } + private class PackStream extends OutputStream { final byte[] hdrBuf; final CRC32 crc32; @@ -404,5 +438,202 @@ class PackInserter extends ObjectInserter { out.close(); file.close(); } + + byte[] inflate(long filePos, int len) throws IOException, DataFormatException { + byte[] dstbuf; + try { + dstbuf = new byte[len]; + } catch (OutOfMemoryError noMemory) { + return null; // Caller will switch to large object streaming. + } + + byte[] srcbuf = buffer(); + Inflater inf = inflater(); + filePos += setInput(filePos, inf, srcbuf); + for (int dstoff = 0;;) { + int n = inf.inflate(dstbuf, dstoff, dstbuf.length - dstoff); + dstoff += n; + if (inf.finished()) { + return dstbuf; + } + if (inf.needsInput()) { + filePos += setInput(filePos, inf, srcbuf); + } else if (n == 0) { + throw new DataFormatException(); + } + } + } + + private int setInput(long filePos, Inflater inf, byte[] buf) + throws IOException { + if (file.getFilePointer() != filePos) { + file.seek(filePos); + } + int n = file.read(buf); + if (n < 0) { + throw new EOFException(JGitText.get().unexpectedEofInPack); + } + inf.setInput(buf, 0, n); + return n; + } + } + + private class Reader extends ObjectReader { + private final ObjectReader ctx; + + private Reader() { + ctx = db.newReader(); + setStreamFileThreshold(ctx.getStreamFileThreshold()); + } + + @Override + public ObjectReader newReader() { + return db.newReader(); + } + + @Override + public ObjectInserter getCreatedFromInserter() { + return PackInserter.this; + } + + @Override + public Collection resolve(AbbreviatedObjectId id) + throws IOException { + Collection stored = ctx.resolve(id); + if (objectList == null) { + return stored; + } + + Set r = new HashSet<>(stored.size() + 2); + r.addAll(stored); + for (PackedObjectInfo obj : objectList) { + if (id.prefixCompare(obj) == 0) { + r.add(obj.copy()); + } + } + return r; + } + + @Override + public ObjectLoader open(AnyObjectId objectId, int typeHint) + throws MissingObjectException, IncorrectObjectTypeException, + IOException { + if (objectMap == null) { + return ctx.open(objectId, typeHint); + } + + PackedObjectInfo obj = objectMap.get(objectId); + if (obj == null) { + return ctx.open(objectId, typeHint); + } + + byte[] buf = buffer(); + RandomAccessFile f = packOut.file; + f.seek(obj.getOffset()); + int cnt = f.read(buf, 0, 20); + if (cnt <= 0) { + throw new EOFException(JGitText.get().unexpectedEofInPack); + } + + int c = buf[0] & 0xff; + int type = (c >> 4) & 7; + if (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) { + throw new IOException(MessageFormat.format( + JGitText.get().cannotReadBackDelta, Integer.toString(type))); + } + if (typeHint != OBJ_ANY && type != typeHint) { + throw new IncorrectObjectTypeException(objectId.copy(), typeHint); + } + + long sz = c & 0x0f; + int ptr = 1; + int shift = 4; + while ((c & 0x80) != 0) { + if (ptr >= cnt) { + throw new EOFException(JGitText.get().unexpectedEofInPack); + } + c = buf[ptr++] & 0xff; + sz += ((long) (c & 0x7f)) << shift; + shift += 7; + } + + long zpos = obj.getOffset() + ptr; + if (sz < getStreamFileThreshold()) { + byte[] data = inflate(obj, zpos, (int) sz); + if (data != null) { + return new ObjectLoader.SmallObject(type, data); + } + } + return new StreamLoader(f, type, sz, zpos); + } + + private byte[] inflate(PackedObjectInfo obj, long zpos, int sz) + throws IOException, CorruptObjectException { + try { + return packOut.inflate(zpos, sz); + } catch (DataFormatException dfe) { + CorruptObjectException coe = new CorruptObjectException( + MessageFormat.format( + JGitText.get().objectAtHasBadZlibStream, + Long.valueOf(obj.getOffset()), + tmpPack.getAbsolutePath())); + coe.initCause(dfe); + throw coe; + } + } + + @Override + public Set getShallowCommits() throws IOException { + return ctx.getShallowCommits(); + } + + @Override + public void close() { + ctx.close(); + } + + private class StreamLoader extends ObjectLoader { + private final RandomAccessFile file; + private final int type; + private final long size; + private final long pos; + + StreamLoader(RandomAccessFile file, int type, long size, long pos) { + this.file = file; + this.type = type; + this.size = size; + this.pos = pos; + } + + @Override + public ObjectStream openStream() + throws MissingObjectException, IOException { + int bufsz = buffer().length; + file.seek(pos); + return new ObjectStream.Filter( + type, size, + new BufferedInputStream( + new InflaterInputStream( + Channels.newInputStream(packOut.file.getChannel()), + inflater(), bufsz), + bufsz)); + } + + @Override + public int getType() { + return type; + } + + @Override + public long getSize() { + return size; + } + + @Override + public byte[] getCachedBytes() throws LargeObjectException { + throw new LargeObjectException.ExceedsLimit( + getStreamFileThreshold(), size); + } + } } }