Browse Source
If the attributes of FileSnapshot don't detect modification of a packfile read the packfile's checksum and compare it against the checksum cached in the loaded packfile. Since reading the checksum needs less IO than reloading the complete packfile this may help to reduce the overhead to detect modficiation when a gc completes while ObjectDirectory scans for packfiles in another thread. Bug: 546891 Change-Id: I9811b497eb11b8a85ae689081dc5d949ca8c4be5 Signed-off-by: Matthias Sohn <matthias.sohn@sap.com>stable-5.1
Matthias Sohn
6 years ago
4 changed files with 312 additions and 22 deletions
@ -0,0 +1,143 @@
|
||||
/* |
||||
* Copyright (C) 2019, Matthias Sohn <matthias.sohn@sap.com> |
||||
* and other copyright owners as documented in the project's IP log. |
||||
* |
||||
* This program and the accompanying materials are made available |
||||
* under the terms of the Eclipse Distribution License v1.0 which |
||||
* accompanies this distribution, is reproduced below, and is |
||||
* available at http://www.eclipse.org/org/documents/edl-v10.php
|
||||
* |
||||
* All rights reserved. |
||||
* |
||||
* Redistribution and use in source and binary forms, with or |
||||
* without modification, are permitted provided that the following |
||||
* conditions are met: |
||||
* |
||||
* - Redistributions of source code must retain the above copyright |
||||
* notice, this list of conditions and the following disclaimer. |
||||
* |
||||
* - Redistributions in binary form must reproduce the above |
||||
* copyright notice, this list of conditions and the following |
||||
* disclaimer in the documentation and/or other materials provided |
||||
* with the distribution. |
||||
* |
||||
* - Neither the name of the Eclipse Foundation, Inc. nor the |
||||
* names of its contributors may be used to endorse or promote |
||||
* products derived from this software without specific prior |
||||
* written permission. |
||||
* |
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND |
||||
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, |
||||
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
||||
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR |
||||
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
||||
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
||||
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER |
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, |
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF |
||||
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||||
*/ |
||||
package org.eclipse.jgit.internal.storage.file; |
||||
|
||||
import static org.junit.Assert.assertEquals; |
||||
import static org.junit.Assert.assertTrue; |
||||
|
||||
import java.io.File; |
||||
import java.io.IOException; |
||||
import java.io.Writer; |
||||
import java.nio.file.Files; |
||||
import java.nio.file.StandardOpenOption; |
||||
import java.text.ParseException; |
||||
import java.util.Collection; |
||||
import java.util.Random; |
||||
import java.util.zip.Deflater; |
||||
|
||||
import org.eclipse.jgit.api.GarbageCollectCommand; |
||||
import org.eclipse.jgit.api.Git; |
||||
import org.eclipse.jgit.junit.RepositoryTestCase; |
||||
import org.eclipse.jgit.lib.ConfigConstants; |
||||
import org.eclipse.jgit.storage.file.FileBasedConfig; |
||||
import org.eclipse.jgit.storage.pack.PackConfig; |
||||
import org.junit.Test; |
||||
|
||||
public class PackFileSnapshotTest extends RepositoryTestCase { |
||||
|
||||
@Test |
||||
public void testSamePackDifferentCompressionDetectChecksumChanged() |
||||
throws Exception { |
||||
Git git = Git.wrap(db); |
||||
File f = writeTrashFile("file", "foobar "); |
||||
for (int i = 0; i < 10; i++) { |
||||
appendRandomLine(f); |
||||
git.add().addFilepattern("file").call(); |
||||
git.commit().setMessage("message" + i).call(); |
||||
} |
||||
|
||||
FileBasedConfig c = db.getConfig(); |
||||
c.setInt(ConfigConstants.CONFIG_GC_SECTION, null, |
||||
ConfigConstants.CONFIG_KEY_AUTOPACKLIMIT, 1); |
||||
c.save(); |
||||
Collection<PackFile> packs = gc(Deflater.NO_COMPRESSION); |
||||
assertEquals("expected 1 packfile after gc", 1, packs.size()); |
||||
PackFile p1 = packs.iterator().next(); |
||||
PackFileSnapshot snapshot = p1.getFileSnapshot(); |
||||
|
||||
packs = gc(Deflater.BEST_COMPRESSION); |
||||
assertEquals("expected 1 packfile after gc", 1, packs.size()); |
||||
PackFile p2 = packs.iterator().next(); |
||||
File pf = p2.getPackFile(); |
||||
|
||||
// changing compression level with aggressive gc may change size,
|
||||
// fileKey (on *nix) and checksum. Hence FileSnapshot.isModified can
|
||||
// return true already based on size or fileKey.
|
||||
// So the only thing we can test here is that we ensure that checksum
|
||||
// also changed when we read it here in this test
|
||||
assertTrue("expected snapshot to detect modified pack", |
||||
snapshot.isModified(pf)); |
||||
assertTrue("expected checksum changed", snapshot.isChecksumChanged(pf)); |
||||
} |
||||
|
||||
private void appendRandomLine(File f) throws IOException { |
||||
try (Writer w = Files.newBufferedWriter(f.toPath(), |
||||
StandardOpenOption.APPEND)) { |
||||
w.append(randomLine(20)); |
||||
} |
||||
} |
||||
|
||||
private String randomLine(int len) { |
||||
final int a = 97; // 'a'
|
||||
int z = 122; // 'z'
|
||||
Random random = new Random(); |
||||
StringBuilder buffer = new StringBuilder(len); |
||||
for (int i = 0; i < len; i++) { |
||||
int rnd = a + (int) (random.nextFloat() * (z - a + 1)); |
||||
buffer.append((char) rnd); |
||||
} |
||||
buffer.append('\n'); |
||||
return buffer.toString(); |
||||
} |
||||
|
||||
private Collection<PackFile> gc(int compressionLevel) |
||||
throws IOException, ParseException { |
||||
GC gc = new GC(db); |
||||
PackConfig pc = new PackConfig(db.getConfig()); |
||||
pc.setCompressionLevel(compressionLevel); |
||||
|
||||
pc.setSinglePack(true); |
||||
|
||||
// --aggressive
|
||||
pc.setDeltaSearchWindowSize( |
||||
GarbageCollectCommand.DEFAULT_GC_AGGRESSIVE_WINDOW); |
||||
pc.setMaxDeltaDepth(GarbageCollectCommand.DEFAULT_GC_AGGRESSIVE_DEPTH); |
||||
pc.setReuseObjects(false); |
||||
|
||||
gc.setPackConfig(pc); |
||||
gc.setExpireAgeMillis(0); |
||||
gc.setPackExpireAgeMillis(0); |
||||
return gc.gc(); |
||||
} |
||||
|
||||
} |
||||
@ -0,0 +1,123 @@
|
||||
/* |
||||
* Copyright (C) 2019, Matthias Sohn <matthias.sohn@sap.com> |
||||
* and other copyright owners as documented in the project's IP log. |
||||
* |
||||
* This program and the accompanying materials are made available |
||||
* under the terms of the Eclipse Distribution License v1.0 which |
||||
* accompanies this distribution, is reproduced below, and is |
||||
* available at http://www.eclipse.org/org/documents/edl-v10.php
|
||||
* |
||||
* All rights reserved. |
||||
* |
||||
* Redistribution and use in source and binary forms, with or |
||||
* without modification, are permitted provided that the following |
||||
* conditions are met: |
||||
* |
||||
* - Redistributions of source code must retain the above copyright |
||||
* notice, this list of conditions and the following disclaimer. |
||||
* |
||||
* - Redistributions in binary form must reproduce the above |
||||
* copyright notice, this list of conditions and the following |
||||
* disclaimer in the documentation and/or other materials provided |
||||
* with the distribution. |
||||
* |
||||
* - Neither the name of the Eclipse Foundation, Inc. nor the |
||||
* names of its contributors may be used to endorse or promote |
||||
* products derived from this software without specific prior |
||||
* written permission. |
||||
* |
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND |
||||
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, |
||||
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
||||
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR |
||||
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
||||
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
||||
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER |
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, |
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF |
||||
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||||
*/ |
||||
package org.eclipse.jgit.internal.storage.file; |
||||
|
||||
import java.io.File; |
||||
import java.io.IOException; |
||||
import java.io.RandomAccessFile; |
||||
|
||||
import org.eclipse.jgit.lib.AnyObjectId; |
||||
import org.eclipse.jgit.lib.ObjectId; |
||||
|
||||
class PackFileSnapshot extends FileSnapshot { |
||||
|
||||
private static final ObjectId MISSING_CHECKSUM = ObjectId.zeroId(); |
||||
|
||||
/** |
||||
* Record a snapshot for a specific packfile path. |
||||
* <p> |
||||
* This method should be invoked before the packfile is accessed. |
||||
* |
||||
* @param path |
||||
* the path to later remember. The path's current status |
||||
* information is saved. |
||||
* @return the snapshot. |
||||
*/ |
||||
public static PackFileSnapshot save(File path) { |
||||
return new PackFileSnapshot(path); |
||||
} |
||||
|
||||
private AnyObjectId checksum = MISSING_CHECKSUM; |
||||
|
||||
private boolean wasChecksumChanged; |
||||
|
||||
|
||||
PackFileSnapshot(File packFile) { |
||||
super(packFile); |
||||
} |
||||
|
||||
void setChecksum(AnyObjectId checksum) { |
||||
this.checksum = checksum; |
||||
} |
||||
|
||||
/** {@inheritDoc} */ |
||||
@Override |
||||
public boolean isModified(File packFile) { |
||||
if (!super.isModified(packFile)) { |
||||
return false; |
||||
} |
||||
if (wasSizeChanged() || wasFileKeyChanged() |
||||
|| !wasLastModifiedRacilyClean()) { |
||||
return true; |
||||
} |
||||
return isChecksumChanged(packFile); |
||||
} |
||||
|
||||
boolean isChecksumChanged(File packFile) { |
||||
return wasChecksumChanged = checksum != MISSING_CHECKSUM |
||||
&& !checksum.equals(readChecksum(packFile)); |
||||
} |
||||
|
||||
private AnyObjectId readChecksum(File packFile) { |
||||
try (RandomAccessFile fd = new RandomAccessFile(packFile, "r")) { //$NON-NLS-1$
|
||||
fd.seek(fd.length() - 20); |
||||
final byte[] buf = new byte[20]; |
||||
fd.readFully(buf, 0, 20); |
||||
return ObjectId.fromRaw(buf); |
||||
} catch (IOException e) { |
||||
return MISSING_CHECKSUM; |
||||
} |
||||
} |
||||
|
||||
boolean wasChecksumChanged() { |
||||
return wasChecksumChanged; |
||||
} |
||||
|
||||
@SuppressWarnings("nls") |
||||
@Override |
||||
public String toString() { |
||||
return "PackFileSnapshot [checksum=" + checksum + ", " |
||||
+ super.toString() + "]"; |
||||
} |
||||
|
||||
} |
||||
Loading…
Reference in new issue