From eb64ccad6d0ec1c8dbe4419c6e4ff564d1fac167 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Tue, 7 Sep 2010 18:18:55 -0700 Subject: [PATCH 1/3] Correctly name DeltaBaseCache This class is used only to cache the unpacked form of an object that was used as a base for another object. The theory goes that if an object is used as a delta base for A, it will probably also be a delta base for B, C, D, E, etc. and therefore having an unpacked copy of it on hand will make delta resolution for the others very fast. However since objects are usually only accessed once, we don't want to cache everything we unpack, just things that we are likely to need again. The only things we need again are the delta bases. Hence, its a delta base cache. This gets us the class name UnpackedObjectCache back, so we can use it to actually create a cache of unpacked object information. Change-Id: I121f356cf4eca7b80126497264eac22bd5825a1d Signed-off-by: Shawn O. Pearce --- ...npackedObjectCache.java => DeltaBaseCache.java} | 4 ++-- .../org/eclipse/jgit/storage/file/PackFile.java | 14 +++----------- .../org/eclipse/jgit/storage/file/WindowCache.java | 2 +- .../jgit/storage/file/WindowCacheConfig.java | 4 ++-- 4 files changed, 8 insertions(+), 16 deletions(-) rename org.eclipse.jgit/src/org/eclipse/jgit/storage/file/{UnpackedObjectCache.java => DeltaBaseCache.java} (98%) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/UnpackedObjectCache.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/DeltaBaseCache.java similarity index 98% rename from org.eclipse.jgit/src/org/eclipse/jgit/storage/file/UnpackedObjectCache.java rename to org.eclipse.jgit/src/org/eclipse/jgit/storage/file/DeltaBaseCache.java index 92f482425..8b548242b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/UnpackedObjectCache.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/DeltaBaseCache.java @@ -45,7 +45,7 @@ package org.eclipse.jgit.storage.file; import java.lang.ref.SoftReference; -class UnpackedObjectCache { +class DeltaBaseCache { private static final int CACHE_SZ = 1024; private static final SoftReference DEAD; @@ -164,7 +164,7 @@ class UnpackedObjectCache { e.sz = 0; } - private UnpackedObjectCache() { + private DeltaBaseCache() { throw new UnsupportedOperationException(); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/PackFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/PackFile.java index ed159ef38..523911162 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/PackFile.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/PackFile.java @@ -220,7 +220,7 @@ public class PackFile implements Iterable { * Close the resources utilized by this repository */ public void close() { - UnpackedObjectCache.purge(this); + DeltaBaseCache.purge(this); WindowCache.purge(this); synchronized (this) { loadedIdx = null; @@ -274,14 +274,6 @@ public class PackFile implements Iterable { return getReverseIdx().findObject(offset); } - private final UnpackedObjectCache.Entry readCache(final long position) { - return UnpackedObjectCache.get(this, position); - } - - private final void saveCache(final long position, final byte[] data, final int type) { - UnpackedObjectCache.store(this, position, data, type); - } - private final byte[] decompress(final long position, final long totalSize, final WindowCursor curs) throws IOException, DataFormatException { final byte[] dstbuf = new byte[(int) totalSize]; @@ -700,7 +692,7 @@ public class PackFile implements Iterable { byte[] data; int type; - UnpackedObjectCache.Entry e = readCache(posBase); + DeltaBaseCache.Entry e = DeltaBaseCache.get(this, posBase); if (e != null) { data = e.data; type = e.type; @@ -715,7 +707,7 @@ public class PackFile implements Iterable { } data = p.getCachedBytes(); type = p.getType(); - saveCache(posBase, data, type); + DeltaBaseCache.store(this, posBase, data, type); } // At this point we have the base, and its small, and the delta diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCache.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCache.java index 68fa19120..f533af48c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCache.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCache.java @@ -187,7 +187,7 @@ public class WindowCache { oc.removeAll(); cache = nc; streamFileThreshold = cfg.getStreamFileThreshold(); - UnpackedObjectCache.reconfigure(cfg); + DeltaBaseCache.reconfigure(cfg); } static int getStreamFileThreshold() { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCacheConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCacheConfig.java index 90ea376b5..95ec6f7c7 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCacheConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCacheConfig.java @@ -146,7 +146,7 @@ public class WindowCacheConfig { } /** - * @return maximum number of bytes to cache in {@link UnpackedObjectCache} + * @return maximum number of bytes to cache in {@link DeltaBaseCache} * for inflated, recently accessed objects, without delta chains. * Default 10 MB. */ @@ -157,7 +157,7 @@ public class WindowCacheConfig { /** * @param newLimit * maximum number of bytes to cache in - * {@link UnpackedObjectCache} for inflated, recently accessed + * {@link DeltaBaseCache} for inflated, recently accessed * objects, without delta chains. */ public void setDeltaBaseCacheLimit(final int newLimit) { From 3f66e65e710adf3649b10385f899f3df6d680922 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Tue, 7 Sep 2010 19:16:39 -0700 Subject: [PATCH 2/3] Remember loose objects and fast-track their lookup Recently created objects are usually what branches point to, and are usually written out as loose objects. But due to the high cost of asking the operating system if a file exists, these are the last thing that ObjectDirectory examines when looking for an object by its ObjectId. Caching recently seen loose objects permits the opening code to jump directly to the loose object, accelerating lookup for branch heads that are accessed often. To avoid exploding the cache its limited to approximately 2048 entries. When more ids are added, the table is simply cleared and reset in size. Change-Id: I18f483217412b102f754ffd496c87061d592e535 Signed-off-by: Shawn O. Pearce --- .../jgit/storage/file/ObjectDirectory.java | 21 +++ .../storage/file/ObjectDirectoryInserter.java | 8 +- .../storage/file/UnpackedObjectCache.java | 149 ++++++++++++++++++ 3 files changed, 176 insertions(+), 2 deletions(-) create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/storage/file/UnpackedObjectCache.java diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/ObjectDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/ObjectDirectory.java index 2ad14c804..29a89bc09 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/ObjectDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/ObjectDirectory.java @@ -117,6 +117,8 @@ public class ObjectDirectory extends FileObjectDatabase { private final AtomicReference alternates; + private final UnpackedObjectCache unpackedObjectCache; + /** * Initialize a reference to an on-disk object directory. * @@ -140,6 +142,7 @@ public class ObjectDirectory extends FileObjectDatabase { packDirectory = new File(objects, "pack"); alternatesFile = new File(infoDirectory, "alternates"); packList = new AtomicReference(NO_PACKS); + unpackedObjectCache = new UnpackedObjectCache(); this.fs = fs; alternates = new AtomicReference(); @@ -179,6 +182,8 @@ public class ObjectDirectory extends FileObjectDatabase { @Override public void close() { + unpackedObjectCache.clear(); + final PackList packs = packList.get(); packList.set(NO_PACKS); for (final PackFile p : packs.packs) @@ -255,6 +260,8 @@ public class ObjectDirectory extends FileObjectDatabase { } boolean hasObject1(final AnyObjectId objectId) { + if (unpackedObjectCache.isUnpacked(objectId)) + return true; for (final PackFile p : packList.get().packs) { try { if (p.hasObject(objectId)) { @@ -328,6 +335,14 @@ public class ObjectDirectory extends FileObjectDatabase { ObjectLoader openObject1(final WindowCursor curs, final AnyObjectId objectId) throws IOException { + if (unpackedObjectCache.isUnpacked(objectId)) { + ObjectLoader ldr = openObject2(curs, objectId.name(), objectId); + if (ldr != null) + return ldr; + else + unpackedObjectCache.remove(objectId); + } + PackList pList = packList.get(); SEARCH: for (;;) { for (final PackFile p : pList.packs) { @@ -429,15 +444,21 @@ public class ObjectDirectory extends FileObjectDatabase { File path = fileFor(objectName); FileInputStream in = new FileInputStream(path); try { + unpackedObjectCache.add(objectId); return UnpackedObject.open(in, path, objectId, curs); } finally { in.close(); } } catch (FileNotFoundException noFile) { + unpackedObjectCache.remove(objectId); return null; } } + void addUnpackedObject(ObjectId id) { + unpackedObjectCache.add(id); + } + boolean tryAgain1() { final PackList old = packList.get(); if (old.tryAgain(packDirectory.lastModified())) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/ObjectDirectoryInserter.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/ObjectDirectoryInserter.java index 501667989..a1f224c7b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/ObjectDirectoryInserter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/ObjectDirectoryInserter.java @@ -91,16 +91,20 @@ class ObjectDirectoryInserter extends ObjectInserter { } final File dst = db.fileFor(id); - if (tmp.renameTo(dst)) + if (tmp.renameTo(dst)) { + db.addUnpackedObject(id); return id; + } // Maybe the directory doesn't exist yet as the object // directories are always lazily created. Note that we // try the rename first as the directory likely does exist. // dst.getParentFile().mkdir(); - if (tmp.renameTo(dst)) + if (tmp.renameTo(dst)) { + db.addUnpackedObject(id); return id; + } if (db.has(id)) { tmp.delete(); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/UnpackedObjectCache.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/UnpackedObjectCache.java new file mode 100644 index 000000000..4d05c6f7e --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/UnpackedObjectCache.java @@ -0,0 +1,149 @@ +/* + * Copyright (C) 2010, Google Inc. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.eclipse.jgit.storage.file; + +import java.util.concurrent.atomic.AtomicReferenceArray; + +import org.eclipse.jgit.lib.AnyObjectId; +import org.eclipse.jgit.lib.ObjectId; + +/** Remembers objects that are currently unpacked. */ +class UnpackedObjectCache { + private static final int INITIAL_BITS = 5; // size = 32 + + private static final int MAX_BITS = 11; // size = 2048 + + private volatile Table table; + + UnpackedObjectCache() { + table = new Table(INITIAL_BITS); + } + + boolean isUnpacked(AnyObjectId objectId) { + return table.contains(objectId); + } + + void add(AnyObjectId objectId) { + Table t = table; + if (t.add(objectId)) { + // The object either already exists in the table, or was + // successfully added. Either way leave the table alone. + // + } else { + // The object won't fit into the table. Implement a crude + // cache removal by just dropping the table away, but double + // it in size for the next incarnation. + // + Table n = new Table(Math.min(t.bits + 1, MAX_BITS)); + n.add(objectId); + table = n; + } + } + + void remove(AnyObjectId objectId) { + if (isUnpacked(objectId)) + clear(); + } + + void clear() { + table = new Table(INITIAL_BITS); + } + + private static class Table { + private static final int MAX_CHAIN = 8; + + private final AtomicReferenceArray ids; + + private final int shift; + + final int bits; + + Table(int bits) { + this.ids = new AtomicReferenceArray(1 << bits); + this.shift = 32 - bits; + this.bits = bits; + } + + boolean contains(AnyObjectId toFind) { + int i = index(toFind); + for (int n = 0; n < MAX_CHAIN; n++) { + ObjectId obj = ids.get(i); + if (obj == null) + break; + + if (AnyObjectId.equals(obj, toFind)) + return true; + + if (++i == ids.length()) + i = 0; + } + return false; + } + + boolean add(AnyObjectId toAdd) { + int i = index(toAdd); + for (int n = 0; n < MAX_CHAIN;) { + ObjectId obj = ids.get(i); + if (obj == null) { + if (ids.compareAndSet(i, null, toAdd.copy())) + return true; + else + continue; + } + + if (AnyObjectId.equals(obj, toAdd)) + return true; + + if (++i == ids.length()) + i = 0; + n++; + } + return false; + } + + private int index(AnyObjectId id) { + return id.hashCode() >>> shift; + } + } +} From 41dd9ed1c054f9f9e1ab52fc7bbf1a55a56cf543 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Tue, 7 Sep 2010 19:47:23 -0700 Subject: [PATCH 3/3] Unpack and cache large deltas as loose objects Instead of spooling large delta bases into temporary files and then immediately deleting them afterwards, spool the large delta out to a normal loose object. Later any requests for that large delta can be answered by reading from the loose object, which is much easier to stream efficiently for readers. Since the object is now duplicated, once in the pack as a delta and again as a loose object, any future prune-packed will automatically delete the loose object variant, releasing the wasted disk space. As prune-packed is run automatically during either repack or gc, and gc --auto triggers automatically based on the number of loose objects, we get automatic cache management for free. Large objects that were unpacked will be periodically cleared out, and will simply be restored later if they are needed again. After a short offline discussion with Junio Hamano today, we may want to propose a change to prune-packed to hold onto larger loose objects which also exist in pack files as deltas, if the loose object was recently accessed or modified in the last 2 days. Change-Id: I3668a3967c807010f48cd69f994dcbaaf582337c Signed-off-by: Shawn O. Pearce --- .../storage/file/CachedObjectDirectory.java | 8 ++- .../jgit/storage/file/FileObjectDatabase.java | 5 ++ .../storage/file/LargePackedDeltaObject.java | 52 ++++++++++++------ .../jgit/storage/file/ObjectDirectory.java | 47 ++++++++++++++-- .../storage/file/ObjectDirectoryInserter.java | 54 ++++++------------- 5 files changed, 105 insertions(+), 61 deletions(-) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/CachedObjectDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/CachedObjectDirectory.java index ca0f06fae..f0159f626 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/CachedObjectDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/CachedObjectDirectory.java @@ -54,7 +54,6 @@ import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectDatabase; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectIdSubclassMap; -import org.eclipse.jgit.lib.ObjectInserter; import org.eclipse.jgit.lib.ObjectLoader; import org.eclipse.jgit.storage.pack.ObjectToPack; import org.eclipse.jgit.storage.pack.PackWriter; @@ -113,7 +112,7 @@ class CachedObjectDirectory extends FileObjectDatabase { } @Override - public ObjectInserter newInserter() { + public ObjectDirectoryInserter newInserter() { return wrapped.newInserter(); } @@ -213,6 +212,11 @@ class CachedObjectDirectory extends FileObjectDatabase { throw new UnsupportedOperationException(); } + @Override + boolean insertUnpackedObject(File tmp, ObjectId objectId, boolean force) { + return wrapped.insertUnpackedObject(tmp, objectId, force); + } + @Override void selectObjectRepresentation(PackWriter packer, ObjectToPack otp, WindowCursor curs) throws IOException { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/FileObjectDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/FileObjectDatabase.java index da38887fc..29c7a2531 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/FileObjectDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/FileObjectDatabase.java @@ -62,6 +62,9 @@ abstract class FileObjectDatabase extends ObjectDatabase { return new WindowCursor(this); } + @Override + public abstract ObjectDirectoryInserter newInserter(); + /** * Does the requested object exist in this database? *

@@ -246,6 +249,8 @@ abstract class FileObjectDatabase extends ObjectDatabase { abstract long getObjectSize2(WindowCursor curs, String objectName, AnyObjectId objectId) throws IOException; + abstract boolean insertUnpackedObject(File tmp, ObjectId id, boolean force); + abstract FileObjectDatabase newCachedFileObjectDatabase(); static class AlternateHandle { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/LargePackedDeltaObject.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/LargePackedDeltaObject.java index 2b98f107f..8d15fcf79 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/LargePackedDeltaObject.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/LargePackedDeltaObject.java @@ -44,9 +44,12 @@ package org.eclipse.jgit.storage.file; import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.zip.DataFormatException; +import java.util.zip.DeflaterOutputStream; import java.util.zip.InflaterInputStream; import org.eclipse.jgit.errors.IncorrectObjectTypeException; @@ -58,7 +61,6 @@ import org.eclipse.jgit.lib.ObjectLoader; import org.eclipse.jgit.lib.ObjectStream; import org.eclipse.jgit.storage.pack.BinaryDelta; import org.eclipse.jgit.storage.pack.DeltaStream; -import org.eclipse.jgit.util.TemporaryBuffer; import org.eclipse.jgit.util.io.TeeInputStream; class LargePackedDeltaObject extends ObjectLoader { @@ -165,14 +167,39 @@ class LargePackedDeltaObject extends ObjectLoader { @Override public ObjectStream openStream() throws MissingObjectException, IOException { + // If the object was recently unpacked, its available loose. + // The loose format is going to be faster to access than a + // delta applied on top of a base. Use that whenever we can. + // + final ObjectId myId = getObjectId(); final WindowCursor wc = new WindowCursor(db); + ObjectLoader ldr = db.openObject2(wc, myId.name(), myId); + if (ldr != null) + return ldr.openStream(); + InputStream in = open(wc); in = new BufferedInputStream(in, 8192); - return new ObjectStream.Filter(getType(), size, in) { + + // While we inflate the object, also deflate it back as a loose + // object. This will later be cleaned up by a gc pass, but until + // then we will reuse the loose form by the above code path. + // + int myType = getType(); + long mySize = getSize(); + final ObjectDirectoryInserter odi = db.newInserter(); + final File tmp = odi.newTempFile(); + DeflaterOutputStream dOut = odi.compress(new FileOutputStream(tmp)); + odi.writeHeader(dOut, myType, mySize); + + in = new TeeInputStream(in, dOut); + return new ObjectStream.Filter(myType, mySize, in) { @Override public void close() throws IOException { - wc.release(); super.close(); + + odi.release(); + wc.release(); + db.insertUnpackedObject(tmp, myId, true /* force creation */); } }; } @@ -195,13 +222,9 @@ class LargePackedDeltaObject extends ObjectLoader { final ObjectLoader base = pack.load(wc, baseOffset); DeltaStream ds = new DeltaStream(delta) { private long baseSize = SIZE_UNKNOWN; - private TemporaryBuffer.LocalFile buffer; @Override protected InputStream openBase() throws IOException { - if (buffer != null) - return buffer.openInputStream(); - InputStream in; if (base instanceof LargePackedDeltaObject) in = ((LargePackedDeltaObject) base).open(wc); @@ -213,9 +236,7 @@ class LargePackedDeltaObject extends ObjectLoader { else if (in instanceof ObjectStream) baseSize = ((ObjectStream) in).getSize(); } - - buffer = new TemporaryBuffer.LocalFile(db.getDirectory()); - return new TeeInputStream(in, buffer); + return in; } @Override @@ -228,14 +249,11 @@ class LargePackedDeltaObject extends ObjectLoader { } return baseSize; } - - @Override - public void close() throws IOException { - super.close(); - if (buffer != null) - buffer.destroy(); - } }; + if (type == Constants.OBJ_BAD) { + if (!(base instanceof LargePackedDeltaObject)) + type = base.getType(); + } if (size == SIZE_UNKNOWN) size = ds.getSize(); return ds; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/ObjectDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/ObjectDirectory.java index 29a89bc09..372a97813 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/ObjectDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/ObjectDirectory.java @@ -69,7 +69,6 @@ import org.eclipse.jgit.lib.Config; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectDatabase; import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectInserter; import org.eclipse.jgit.lib.ObjectLoader; import org.eclipse.jgit.lib.RepositoryCache; import org.eclipse.jgit.lib.RepositoryCache.FileKey; @@ -176,7 +175,7 @@ public class ObjectDirectory extends FileObjectDatabase { } @Override - public ObjectInserter newInserter() { + public ObjectDirectoryInserter newInserter() { return new ObjectDirectoryInserter(this, config); } @@ -455,8 +454,48 @@ public class ObjectDirectory extends FileObjectDatabase { } } - void addUnpackedObject(ObjectId id) { - unpackedObjectCache.add(id); + @Override + boolean insertUnpackedObject(File tmp, ObjectId id, boolean force) { + if (!force && has(id)) { + // Object is already in the repository, remove temporary file. + // + tmp.delete(); + return true; + } + tmp.setReadOnly(); + + final File dst = fileFor(id); + if (force && dst.exists()) { + tmp.delete(); + return true; + } + if (tmp.renameTo(dst)) { + unpackedObjectCache.add(id); + return true; + } + + // Maybe the directory doesn't exist yet as the object + // directories are always lazily created. Note that we + // try the rename first as the directory likely does exist. + // + dst.getParentFile().mkdir(); + if (tmp.renameTo(dst)) { + unpackedObjectCache.add(id); + return true; + } + + if (!force && has(id)) { + tmp.delete(); + return true; + } + + // The object failed to be renamed into its proper + // location and it doesn't exist in the repository + // either. We really don't know what went wrong, so + // fail. + // + tmp.delete(); + return false; } boolean tryAgain1() { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/ObjectDirectoryInserter.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/ObjectDirectoryInserter.java index a1f224c7b..d92285de8 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/ObjectDirectoryInserter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/ObjectDirectoryInserter.java @@ -83,40 +83,10 @@ class ObjectDirectoryInserter extends ObjectInserter { final MessageDigest md = digest(); final File tmp = toTemp(md, type, len, is); final ObjectId id = ObjectId.fromRaw(md.digest()); - if (db.has(id)) { - // Object is already in the repository, remove temporary file. - // - tmp.delete(); + if (db.insertUnpackedObject(tmp, id, false /* no duplicate */)) return id; - } final File dst = db.fileFor(id); - if (tmp.renameTo(dst)) { - db.addUnpackedObject(id); - return id; - } - - // Maybe the directory doesn't exist yet as the object - // directories are always lazily created. Note that we - // try the rename first as the directory likely does exist. - // - dst.getParentFile().mkdir(); - if (tmp.renameTo(dst)) { - db.addUnpackedObject(id); - return id; - } - - if (db.has(id)) { - tmp.delete(); - return id; - } - - // The object failed to be renamed into its proper - // location and it doesn't exist in the repository - // either. We really don't know what went wrong, so - // fail. - // - tmp.delete(); throw new ObjectWritingException("Unable to create new object: " + dst); } @@ -140,15 +110,12 @@ class ObjectDirectoryInserter extends ObjectInserter { final InputStream is) throws IOException, FileNotFoundException, Error { boolean delete = true; - File tmp = File.createTempFile("noz", null, db.getDirectory()); + File tmp = newTempFile(); try { DigestOutputStream dOut = new DigestOutputStream( compress(new FileOutputStream(tmp)), md); try { - dOut.write(Constants.encodedTypeString(type)); - dOut.write((byte) ' '); - dOut.write(Constants.encodeASCII(len)); - dOut.write((byte) 0); + writeHeader(dOut, type, len); final byte[] buf = buffer(); while (len > 0) { @@ -162,7 +129,6 @@ class ObjectDirectoryInserter extends ObjectInserter { dOut.close(); } - tmp.setReadOnly(); delete = false; return tmp; } finally { @@ -171,7 +137,19 @@ class ObjectDirectoryInserter extends ObjectInserter { } } - private DeflaterOutputStream compress(final OutputStream out) { + void writeHeader(OutputStream out, final int type, long len) + throws IOException { + out.write(Constants.encodedTypeString(type)); + out.write((byte) ' '); + out.write(Constants.encodeASCII(len)); + out.write((byte) 0); + } + + File newTempFile() throws IOException { + return File.createTempFile("noz", null, db.getDirectory()); + } + + DeflaterOutputStream compress(final OutputStream out) { if (deflate == null) deflate = new Deflater(config.get(CoreConfig.KEY).getCompression()); else