Browse Source

Added caching for loose object lookup during pack indexing

On Windows systems, file system lookup is a slow operation, so
checking each object if it exists during indexing (after receiving
the pack) could take a siginificant time. This patch introduces
CachedObjectDirectory that pre-caches lookup results.

Bug: 300397
Change-Id: I471b93f9bb3ee173eb37cae1d75e9e4eb49985e7
Signed-off-by: Constantine Plotnikov <constantine.plotnikov@gmail.com>
Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
stable-0.7
Constantine Plotnikov 15 years ago committed by Shawn O. Pearce
parent
commit
cc64794b24
  1. 6
      org.eclipse.jgit/src/org/eclipse/jgit/lib/AlternateRepositoryDatabase.java
  2. 132
      org.eclipse.jgit/src/org/eclipse/jgit/lib/CachedObjectDatabase.java
  3. 112
      org.eclipse.jgit/src/org/eclipse/jgit/lib/CachedObjectDirectory.java
  4. 12
      org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectDatabase.java
  5. 5
      org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectDirectory.java
  6. 10
      org.eclipse.jgit/src/org/eclipse/jgit/transport/IndexPack.java

6
org.eclipse.jgit/src/org/eclipse/jgit/lib/AlternateRepositoryDatabase.java

@ -1,4 +1,5 @@
/*
* Copyright (C) 2010, Constantine Plotnikov <constantine.plotnikov@gmail.com>
* Copyright (C) 2009, Google Inc.
* and other copyright owners as documented in the project's IP log.
*
@ -130,4 +131,9 @@ public final class AlternateRepositoryDatabase extends ObjectDatabase {
protected void closeAlternates(final ObjectDatabase[] alt) {
// Do nothing; these belong to odb to close, not us.
}
@Override
public ObjectDatabase newCachedDatabase() {
return odb.newCachedDatabase();
}
}

132
org.eclipse.jgit/src/org/eclipse/jgit/lib/CachedObjectDatabase.java

@ -0,0 +1,132 @@
/*
* Copyright (C) 2010, Constantine Plotnikov <constantine.plotnikov@gmail.com>
* Copyright (C) 2010, JetBrains s.r.o.
* and other copyright owners as documented in the project's IP log.
*
* This program and the accompanying materials are made available
* under the terms of the Eclipse Distribution License v1.0 which
* accompanies this distribution, is reproduced below, and is
* available at http://www.eclipse.org/org/documents/edl-v10.php
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
*
* - Neither the name of the Eclipse Foundation, Inc. nor the
* names of its contributors may be used to endorse or promote
* products derived from this software without specific prior
* written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.eclipse.jgit.lib;
import java.io.IOException;
import java.util.Collection;
/**
* {@link ObjectDatabase} wrapper providing temporary lookup caching.
* <p>
* The base class for {@code ObjectDatabase}s that wrap other database instances
* and optimize querying for objects by caching some database dependent
* information. Instances of this class (or any of its subclasses) can be
* returned from the method {@link ObjectDatabase#newCachedDatabase()}. This
* class can be used in scenarios where the database does not change, or when
* changes in the database while some operation is in progress is an acceptable
* risk.
* <p>
* The default implementation delegates all requests to the wrapped database.
* The instance might be indirectly invalidated if the wrapped instance is
* closed. Closing the delegating instance does not implies closing the wrapped
* instance. For alternative databases, cached instances are used as well.
*/
public class CachedObjectDatabase extends ObjectDatabase {
/**
* The wrapped database instance
*/
protected final ObjectDatabase wrapped;
/**
* Create the delegating database instance
*
* @param wrapped
* the wrapped object database
*/
public CachedObjectDatabase(ObjectDatabase wrapped) {
this.wrapped = wrapped;
}
@Override
protected boolean hasObject1(AnyObjectId objectId) {
return wrapped.hasObject1(objectId);
}
@Override
protected ObjectLoader openObject1(WindowCursor curs, AnyObjectId objectId)
throws IOException {
return wrapped.openObject1(curs, objectId);
}
@Override
protected boolean hasObject2(String objectName) {
return wrapped.hasObject2(objectName);
}
@Override
protected ObjectDatabase[] loadAlternates() throws IOException {
ObjectDatabase[] loaded = wrapped.getAlternates();
ObjectDatabase[] result = new ObjectDatabase[loaded.length];
for (int i = 0; i < loaded.length; i++) {
result[i] = loaded[i].newCachedDatabase();
}
return result;
}
@Override
protected ObjectLoader openObject2(WindowCursor curs, String objectName,
AnyObjectId objectId) throws IOException {
return wrapped.openObject2(curs, objectName, objectId);
}
@Override
void openObjectInAllPacks1(Collection<PackedObjectLoader> out,
WindowCursor curs, AnyObjectId objectId) throws IOException {
wrapped.openObjectInAllPacks1(out, curs, objectId);
}
@Override
protected boolean tryAgain1() {
return wrapped.tryAgain1();
}
@Override
public ObjectDatabase newCachedDatabase() {
// Note that "this" is not returned since subclasses might actually do something,
// on closeSelf() (for example closing database connections or open repositories).
// The situation might become even more tricky if we will consider alternates.
return wrapped.newCachedDatabase();
}
}

112
org.eclipse.jgit/src/org/eclipse/jgit/lib/CachedObjectDirectory.java

@ -0,0 +1,112 @@
/*
* Copyright (C) 2010, Constantine Plotnikov <constantine.plotnikov@gmail.com>
* Copyright (C) 2010, JetBrains s.r.o.
* and other copyright owners as documented in the project's IP log.
*
* This program and the accompanying materials are made available
* under the terms of the Eclipse Distribution License v1.0 which
* accompanies this distribution, is reproduced below, and is
* available at http://www.eclipse.org/org/documents/edl-v10.php
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
*
* - Neither the name of the Eclipse Foundation, Inc. nor the
* names of its contributors may be used to endorse or promote
* products derived from this software without specific prior
* written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.eclipse.jgit.lib;
import java.io.File;
import java.io.IOException;
/**
* The cached instance of an {@link ObjectDirectory}.
* <p>
* This class caches the list of loose objects in memory, so the file system is
* not queried with stat calls.
*/
public class CachedObjectDirectory extends CachedObjectDatabase {
/**
* The set that contains unpacked objects identifiers, it is created when
* the cached instance is created.
*/
private final ObjectIdSubclassMap<ObjectId> unpackedObjects = new ObjectIdSubclassMap<ObjectId>();
/**
* The constructor
*
* @param wrapped
* the wrapped database
*/
public CachedObjectDirectory(ObjectDirectory wrapped) {
super(wrapped);
File objects = wrapped.getDirectory();
String[] fanout = objects.list();
if (fanout == null)
fanout = new String[0];
for (String d : fanout) {
if (d.length() != 2)
continue;
String[] entries = new File(objects, d).list();
if (entries == null)
continue;
for (String e : entries) {
if (e.length() != Constants.OBJECT_ID_STRING_LENGTH - 2)
continue;
try {
unpackedObjects.add(ObjectId.fromString(d + e));
} catch (IllegalArgumentException notAnObject) {
// ignoring the file that does not represent loose object
}
}
}
}
@Override
protected ObjectLoader openObject2(WindowCursor curs, String objectName,
AnyObjectId objectId) throws IOException {
if (unpackedObjects.get(objectId) == null)
return null;
return super.openObject2(curs, objectName, objectId);
}
@Override
protected boolean hasObject1(AnyObjectId objectId) {
if (unpackedObjects.get(objectId) != null)
return true; // known to be loose
return super.hasObject1(objectId);
}
@Override
protected boolean hasObject2(String name) {
return false; // loose objects were tested by hasObject1
}
}

12
org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectDatabase.java

@ -380,4 +380,16 @@ public abstract class ObjectDatabase {
d.close();
}
}
/**
* Create a new cached database instance over this database. This instance might
* optimize queries by caching some information about database. So some modifications
* done after instance creation might fail to be noticed.
*
* @return new cached database instance
* @see CachedObjectDatabase
*/
public ObjectDatabase newCachedDatabase() {
return new CachedObjectDatabase(this);
}
}

5
org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectDirectory.java

@ -541,4 +541,9 @@ public class ObjectDirectory extends ObjectDatabase {
return true;
}
}
@Override
public ObjectDatabase newCachedDatabase() {
return new CachedObjectDirectory(this);
}
}

10
org.eclipse.jgit/src/org/eclipse/jgit/transport/IndexPack.java

@ -68,6 +68,7 @@ import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.InflaterCache;
import org.eclipse.jgit.lib.MutableObjectId;
import org.eclipse.jgit.lib.ObjectChecker;
import org.eclipse.jgit.lib.ObjectDatabase;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectIdSubclassMap;
import org.eclipse.jgit.lib.ObjectLoader;
@ -130,6 +131,11 @@ public class IndexPack {
private final Repository repo;
/**
* Object database used for loading existing objects
*/
private final ObjectDatabase objectDatabase;
private Inflater inflater;
private final MessageDigest objectDigest;
@ -199,6 +205,7 @@ public class IndexPack {
public IndexPack(final Repository db, final InputStream src,
final File dstBase) throws IOException {
repo = db;
objectDatabase = db.getObjectDatabase().newCachedDatabase();
in = src;
inflater = InflaterCache.get();
readCurs = new WindowCursor();
@ -350,6 +357,7 @@ public class IndexPack {
InflaterCache.release(inflater);
} finally {
inflater = null;
objectDatabase.close();
}
readCurs = WindowCursor.release(readCurs);
@ -756,7 +764,7 @@ public class IndexPack {
}
}
final ObjectLoader ldr = repo.openObject(readCurs, id);
final ObjectLoader ldr = objectDatabase.openObject(readCurs, id);
if (ldr != null) {
final byte[] existingData = ldr.getCachedBytes();
if (ldr.getType() != type || !Arrays.equals(data, existingData)) {

Loading…
Cancel
Save