Browse Source
JGit already had some fsck-like classes like ObjectChecker which can check for an individual object. The read-only FsckPackParser which will parse all objects within a pack file and check it with ObjectChecker. It will also check the pack index file against the object information from the pack parser. Change-Id: Ifd8e0d28eb68ff0b8edd2b51b2fa3a50a544c855 Signed-off-by: Zhen Chen <czhen@google.com>stable-4.9
Zhen Chen
8 years ago
14 changed files with 1010 additions and 45 deletions
@ -0,0 +1,201 @@ |
|||||||
|
/* |
||||||
|
* Copyright (C) 2017, Google Inc. |
||||||
|
* and other copyright owners as documented in the project's IP log. |
||||||
|
* |
||||||
|
* This program and the accompanying materials are made available |
||||||
|
* under the terms of the Eclipse Distribution License v1.0 which |
||||||
|
* accompanies this distribution, is reproduced below, and is |
||||||
|
* available at http://www.eclipse.org/org/documents/edl-v10.php
|
||||||
|
* |
||||||
|
* All rights reserved. |
||||||
|
* |
||||||
|
* Redistribution and use in source and binary forms, with or |
||||||
|
* without modification, are permitted provided that the following |
||||||
|
* conditions are met: |
||||||
|
* |
||||||
|
* - Redistributions of source code must retain the above copyright |
||||||
|
* notice, this list of conditions and the following disclaimer. |
||||||
|
* |
||||||
|
* - Redistributions in binary form must reproduce the above |
||||||
|
* copyright notice, this list of conditions and the following |
||||||
|
* disclaimer in the documentation and/or other materials provided |
||||||
|
* with the distribution. |
||||||
|
* |
||||||
|
* - Neither the name of the Eclipse Foundation, Inc. nor the |
||||||
|
* names of its contributors may be used to endorse or promote |
||||||
|
* products derived from this software without specific prior |
||||||
|
* written permission. |
||||||
|
* |
||||||
|
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND |
||||||
|
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, |
||||||
|
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
||||||
|
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
||||||
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR |
||||||
|
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
||||||
|
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
||||||
|
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
||||||
|
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER |
||||||
|
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, |
||||||
|
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
||||||
|
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF |
||||||
|
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.eclipse.jgit.internal.storage.dfs; |
||||||
|
|
||||||
|
import static org.eclipse.jgit.junit.JGitTestUtil.concat; |
||||||
|
import static org.eclipse.jgit.lib.Constants.OBJECT_ID_LENGTH; |
||||||
|
import static org.eclipse.jgit.lib.Constants.encodeASCII; |
||||||
|
import static org.junit.Assert.assertEquals; |
||||||
|
import static org.junit.Assert.assertNull; |
||||||
|
import static org.junit.Assert.assertTrue; |
||||||
|
import static org.junit.Assert.fail; |
||||||
|
|
||||||
|
import java.io.IOException; |
||||||
|
|
||||||
|
import org.eclipse.jgit.internal.fsck.FsckError; |
||||||
|
import org.eclipse.jgit.internal.fsck.FsckError.CorruptObject; |
||||||
|
import org.eclipse.jgit.junit.TestRepository; |
||||||
|
import org.eclipse.jgit.lib.Constants; |
||||||
|
import org.eclipse.jgit.lib.ObjectChecker.ErrorType; |
||||||
|
import org.eclipse.jgit.lib.ObjectId; |
||||||
|
import org.eclipse.jgit.lib.ObjectInserter; |
||||||
|
import org.eclipse.jgit.revwalk.RevCommit; |
||||||
|
import org.junit.Before; |
||||||
|
import org.junit.Test; |
||||||
|
|
||||||
|
public class DfsFsckTest { |
||||||
|
private TestRepository<InMemoryRepository> git; |
||||||
|
|
||||||
|
private InMemoryRepository repo; |
||||||
|
|
||||||
|
private ObjectInserter ins; |
||||||
|
|
||||||
|
@Before |
||||||
|
public void setUp() throws IOException { |
||||||
|
DfsRepositoryDescription desc = new DfsRepositoryDescription("test"); |
||||||
|
git = new TestRepository<>(new InMemoryRepository(desc)); |
||||||
|
repo = git.getRepository(); |
||||||
|
ins = repo.newObjectInserter(); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void testHealthyRepo() throws Exception { |
||||||
|
RevCommit commit0 = git.commit().message("0").create(); |
||||||
|
RevCommit commit1 = git.commit().message("1").parent(commit0).create(); |
||||||
|
git.update("master", commit1); |
||||||
|
|
||||||
|
DfsFsck fsck = new DfsFsck(repo); |
||||||
|
FsckError errors = fsck.check(null); |
||||||
|
|
||||||
|
assertEquals(errors.getCorruptObjects().size(), 0); |
||||||
|
assertEquals(errors.getMissingObjects().size(), 0); |
||||||
|
assertEquals(errors.getCorruptIndices().size(), 0); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void testCommitWithCorruptAuthor() throws Exception { |
||||||
|
StringBuilder b = new StringBuilder(); |
||||||
|
b.append("tree be9bfa841874ccc9f2ef7c48d0c76226f89b7189\n"); |
||||||
|
b.append("author b <b@c> <b@c> 0 +0000\n"); |
||||||
|
b.append("committer <> 0 +0000\n"); |
||||||
|
byte[] data = encodeASCII(b.toString()); |
||||||
|
ObjectId id = ins.insert(Constants.OBJ_COMMIT, data); |
||||||
|
ins.flush(); |
||||||
|
|
||||||
|
DfsFsck fsck = new DfsFsck(repo); |
||||||
|
FsckError errors = fsck.check(null); |
||||||
|
|
||||||
|
assertEquals(errors.getCorruptObjects().size(), 1); |
||||||
|
CorruptObject o = errors.getCorruptObjects().iterator().next(); |
||||||
|
assertTrue(o.getId().equals(id)); |
||||||
|
assertEquals(o.getErrorType(), ErrorType.BAD_DATE); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void testCommitWithoutTree() throws Exception { |
||||||
|
StringBuilder b = new StringBuilder(); |
||||||
|
b.append("parent "); |
||||||
|
b.append("be9bfa841874ccc9f2ef7c48d0c76226f89b7189"); |
||||||
|
b.append('\n'); |
||||||
|
byte[] data = encodeASCII(b.toString()); |
||||||
|
ObjectId id = ins.insert(Constants.OBJ_COMMIT, data); |
||||||
|
ins.flush(); |
||||||
|
|
||||||
|
DfsFsck fsck = new DfsFsck(repo); |
||||||
|
FsckError errors = fsck.check(null); |
||||||
|
|
||||||
|
assertEquals(errors.getCorruptObjects().size(), 1); |
||||||
|
CorruptObject o = errors.getCorruptObjects().iterator().next(); |
||||||
|
assertTrue(o.getId().equals(id)); |
||||||
|
assertEquals(o.getErrorType(), ErrorType.MISSING_TREE); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void testTagWithoutObject() throws Exception { |
||||||
|
StringBuilder b = new StringBuilder(); |
||||||
|
b.append("type commit\n"); |
||||||
|
b.append("tag test-tag\n"); |
||||||
|
b.append("tagger A. U. Thor <author@localhost> 1 +0000\n"); |
||||||
|
byte[] data = encodeASCII(b.toString()); |
||||||
|
ObjectId id = ins.insert(Constants.OBJ_TAG, data); |
||||||
|
ins.flush(); |
||||||
|
|
||||||
|
DfsFsck fsck = new DfsFsck(repo); |
||||||
|
FsckError errors = fsck.check(null); |
||||||
|
|
||||||
|
assertEquals(errors.getCorruptObjects().size(), 1); |
||||||
|
CorruptObject o = errors.getCorruptObjects().iterator().next(); |
||||||
|
assertTrue(o.getId().equals(id)); |
||||||
|
assertEquals(o.getErrorType(), ErrorType.MISSING_OBJECT); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void testTreeWithNullSha() throws Exception { |
||||||
|
byte[] data = concat(encodeASCII("100644 A"), new byte[] { '\0' }, |
||||||
|
new byte[OBJECT_ID_LENGTH]); |
||||||
|
ObjectId id = ins.insert(Constants.OBJ_TREE, data); |
||||||
|
ins.flush(); |
||||||
|
|
||||||
|
DfsFsck fsck = new DfsFsck(repo); |
||||||
|
FsckError errors = fsck.check(null); |
||||||
|
|
||||||
|
assertEquals(errors.getCorruptObjects().size(), 1); |
||||||
|
CorruptObject o = errors.getCorruptObjects().iterator().next(); |
||||||
|
assertTrue(o.getId().equals(id)); |
||||||
|
assertEquals(o.getErrorType(), ErrorType.NULL_SHA1); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void testMultipleInvalidObjects() throws Exception { |
||||||
|
StringBuilder b = new StringBuilder(); |
||||||
|
b.append("tree "); |
||||||
|
b.append("be9bfa841874ccc9f2ef7c48d0c76226f89b7189"); |
||||||
|
b.append('\n'); |
||||||
|
b.append("parent "); |
||||||
|
b.append("\n"); |
||||||
|
byte[] data = encodeASCII(b.toString()); |
||||||
|
ObjectId id1 = ins.insert(Constants.OBJ_COMMIT, data); |
||||||
|
|
||||||
|
b = new StringBuilder(); |
||||||
|
b.append("100644"); |
||||||
|
data = encodeASCII(b.toString()); |
||||||
|
ObjectId id2 = ins.insert(Constants.OBJ_TREE, data); |
||||||
|
|
||||||
|
ins.flush(); |
||||||
|
|
||||||
|
DfsFsck fsck = new DfsFsck(repo); |
||||||
|
FsckError errors = fsck.check(null); |
||||||
|
|
||||||
|
assertEquals(errors.getCorruptObjects().size(), 2); |
||||||
|
for (CorruptObject o : errors.getCorruptObjects()) { |
||||||
|
if (o.getId().equals(id1)) { |
||||||
|
assertEquals(o.getErrorType(), ErrorType.BAD_PARENT_SHA1); |
||||||
|
} else if (o.getId().equals(id2)) { |
||||||
|
assertNull(o.getErrorType()); |
||||||
|
} else { |
||||||
|
fail(); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,94 @@ |
|||||||
|
/* |
||||||
|
* Copyright (C) 2017, Google Inc. |
||||||
|
* and other copyright owners as documented in the project's IP log. |
||||||
|
* |
||||||
|
* This program and the accompanying materials are made available |
||||||
|
* under the terms of the Eclipse Distribution License v1.0 which |
||||||
|
* accompanies this distribution, is reproduced below, and is |
||||||
|
* available at http://www.eclipse.org/org/documents/edl-v10.php
|
||||||
|
* |
||||||
|
* All rights reserved. |
||||||
|
* |
||||||
|
* Redistribution and use in source and binary forms, with or |
||||||
|
* without modification, are permitted provided that the following |
||||||
|
* conditions are met: |
||||||
|
* |
||||||
|
* - Redistributions of source code must retain the above copyright |
||||||
|
* notice, this list of conditions and the following disclaimer. |
||||||
|
* |
||||||
|
* - Redistributions in binary form must reproduce the above |
||||||
|
* copyright notice, this list of conditions and the following |
||||||
|
* disclaimer in the documentation and/or other materials provided |
||||||
|
* with the distribution. |
||||||
|
* |
||||||
|
* - Neither the name of the Eclipse Foundation, Inc. nor the |
||||||
|
* names of its contributors may be used to endorse or promote |
||||||
|
* products derived from this software without specific prior |
||||||
|
* written permission. |
||||||
|
* |
||||||
|
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND |
||||||
|
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, |
||||||
|
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
||||||
|
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
||||||
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR |
||||||
|
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
||||||
|
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
||||||
|
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
||||||
|
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER |
||||||
|
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, |
||||||
|
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
||||||
|
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF |
||||||
|
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.eclipse.jgit.errors; |
||||||
|
|
||||||
|
import org.eclipse.jgit.annotations.Nullable; |
||||||
|
|
||||||
|
/** |
||||||
|
* Exception thrown when encounters a corrupt pack index file. |
||||||
|
* |
||||||
|
* @since 4.9 |
||||||
|
*/ |
||||||
|
public class CorruptPackIndexException extends Exception { |
||||||
|
private static final long serialVersionUID = 1L; |
||||||
|
|
||||||
|
/** The error type of a corrupt index file. */ |
||||||
|
public enum ErrorType { |
||||||
|
/** Offset does not match index in pack file. */ |
||||||
|
MISMATCH_OFFSET, |
||||||
|
/** CRC does not match CRC of the object data in pack file. */ |
||||||
|
MISMATCH_CRC, |
||||||
|
/** CRC is not present in index file. */ |
||||||
|
MISSING_CRC, |
||||||
|
/** Object in pack is not present in index file. */ |
||||||
|
MISSING_OBJ, |
||||||
|
/** Object in index file is not present in pack file. */ |
||||||
|
UNKNOWN_OBJ, |
||||||
|
} |
||||||
|
|
||||||
|
private ErrorType errorType; |
||||||
|
|
||||||
|
/** |
||||||
|
* Report a specific error condition discovered in an index file. |
||||||
|
* |
||||||
|
* @param message |
||||||
|
* the error message. |
||||||
|
* @param errorType |
||||||
|
* the error type of corruption. |
||||||
|
*/ |
||||||
|
public CorruptPackIndexException(String message, ErrorType errorType) { |
||||||
|
super(message); |
||||||
|
this.errorType = errorType; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Specific the reason of the corrupt index file. |
||||||
|
* |
||||||
|
* @return error condition or null. |
||||||
|
*/ |
||||||
|
@Nullable |
||||||
|
public ErrorType getErrorType() { |
||||||
|
return errorType; |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,145 @@ |
|||||||
|
/* |
||||||
|
* Copyright (C) 2017, Google Inc. |
||||||
|
* and other copyright owners as documented in the project's IP log. |
||||||
|
* |
||||||
|
* This program and the accompanying materials are made available |
||||||
|
* under the terms of the Eclipse Distribution License v1.0 which |
||||||
|
* accompanies this distribution, is reproduced below, and is |
||||||
|
* available at http://www.eclipse.org/org/documents/edl-v10.php
|
||||||
|
* |
||||||
|
* All rights reserved. |
||||||
|
* |
||||||
|
* Redistribution and use in source and binary forms, with or |
||||||
|
* without modification, are permitted provided that the following |
||||||
|
* conditions are met: |
||||||
|
* |
||||||
|
* - Redistributions of source code must retain the above copyright |
||||||
|
* notice, this list of conditions and the following disclaimer. |
||||||
|
* |
||||||
|
* - Redistributions in binary form must reproduce the above |
||||||
|
* copyright notice, this list of conditions and the following |
||||||
|
* disclaimer in the documentation and/or other materials provided |
||||||
|
* with the distribution. |
||||||
|
* |
||||||
|
* - Neither the name of the Eclipse Foundation, Inc. nor the |
||||||
|
* names of its contributors may be used to endorse or promote |
||||||
|
* products derived from this software without specific prior |
||||||
|
* written permission. |
||||||
|
* |
||||||
|
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND |
||||||
|
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, |
||||||
|
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
||||||
|
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
||||||
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR |
||||||
|
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
||||||
|
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
||||||
|
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
||||||
|
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER |
||||||
|
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, |
||||||
|
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
||||||
|
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF |
||||||
|
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||||||
|
*/ |
||||||
|
package org.eclipse.jgit.internal.fsck; |
||||||
|
|
||||||
|
import java.util.HashSet; |
||||||
|
import java.util.Set; |
||||||
|
|
||||||
|
import org.eclipse.jgit.annotations.Nullable; |
||||||
|
import org.eclipse.jgit.errors.CorruptPackIndexException; |
||||||
|
import org.eclipse.jgit.errors.CorruptPackIndexException.ErrorType; |
||||||
|
import org.eclipse.jgit.lib.ObjectChecker; |
||||||
|
import org.eclipse.jgit.lib.ObjectId; |
||||||
|
|
||||||
|
/** Holds all fsck errors of a git repository. */ |
||||||
|
public class FsckError { |
||||||
|
/** Represents a corrupt object. */ |
||||||
|
public static class CorruptObject { |
||||||
|
final ObjectId id; |
||||||
|
|
||||||
|
final int type; |
||||||
|
|
||||||
|
ObjectChecker.ErrorType errorType; |
||||||
|
|
||||||
|
/** |
||||||
|
* @param id |
||||||
|
* the object identifier. |
||||||
|
* @param type |
||||||
|
* type of the object. |
||||||
|
*/ |
||||||
|
public CorruptObject(ObjectId id, int type) { |
||||||
|
this.id = id; |
||||||
|
this.type = type; |
||||||
|
} |
||||||
|
|
||||||
|
void setErrorType(ObjectChecker.ErrorType errorType) { |
||||||
|
this.errorType = errorType; |
||||||
|
} |
||||||
|
|
||||||
|
/** @return identifier of the object. */ |
||||||
|
public ObjectId getId() { |
||||||
|
return id; |
||||||
|
} |
||||||
|
|
||||||
|
/** @return type of the object. */ |
||||||
|
public int getType() { |
||||||
|
return type; |
||||||
|
} |
||||||
|
|
||||||
|
/** @return error type of the corruption. */ |
||||||
|
@Nullable |
||||||
|
public ObjectChecker.ErrorType getErrorType() { |
||||||
|
return errorType; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/** Represents a corrupt pack index file. */ |
||||||
|
public static class CorruptIndex { |
||||||
|
String fileName; |
||||||
|
|
||||||
|
CorruptPackIndexException.ErrorType errorType; |
||||||
|
|
||||||
|
/** |
||||||
|
* @param fileName |
||||||
|
* the file name of the pack index. |
||||||
|
* @param errorType |
||||||
|
* the type of error as reported in |
||||||
|
* {@link CorruptPackIndexException}. |
||||||
|
*/ |
||||||
|
public CorruptIndex(String fileName, ErrorType errorType) { |
||||||
|
this.fileName = fileName; |
||||||
|
this.errorType = errorType; |
||||||
|
} |
||||||
|
|
||||||
|
/** @return the file name of the index file. */ |
||||||
|
public String getFileName() { |
||||||
|
return fileName; |
||||||
|
} |
||||||
|
|
||||||
|
/** @return the error type of the corruption. */ |
||||||
|
public ErrorType getErrorType() { |
||||||
|
return errorType; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
private final Set<CorruptObject> corruptObjects = new HashSet<>(); |
||||||
|
|
||||||
|
private final Set<ObjectId> missingObjects = new HashSet<>(); |
||||||
|
|
||||||
|
private final Set<CorruptIndex> corruptIndices = new HashSet<>(); |
||||||
|
|
||||||
|
/** @return corrupt objects from all pack files. */ |
||||||
|
public Set<CorruptObject> getCorruptObjects() { |
||||||
|
return corruptObjects; |
||||||
|
} |
||||||
|
|
||||||
|
/** @return missing objects that should present in pack files. */ |
||||||
|
public Set<ObjectId> getMissingObjects() { |
||||||
|
return missingObjects; |
||||||
|
} |
||||||
|
|
||||||
|
/** @return corrupt index files associated with the packs. */ |
||||||
|
public Set<CorruptIndex> getCorruptIndices() { |
||||||
|
return corruptIndices; |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,326 @@ |
|||||||
|
/* |
||||||
|
* Copyright (C) 2017, Google Inc. |
||||||
|
* and other copyright owners as documented in the project's IP log. |
||||||
|
* |
||||||
|
* This program and the accompanying materials are made available |
||||||
|
* under the terms of the Eclipse Distribution License v1.0 which |
||||||
|
* accompanies this distribution, is reproduced below, and is |
||||||
|
* available at http://www.eclipse.org/org/documents/edl-v10.php
|
||||||
|
* |
||||||
|
* All rights reserved. |
||||||
|
* |
||||||
|
* Redistribution and use in source and binary forms, with or |
||||||
|
* without modification, are permitted provided that the following |
||||||
|
* conditions are met: |
||||||
|
* |
||||||
|
* - Redistributions of source code must retain the above copyright |
||||||
|
* notice, this list of conditions and the following disclaimer. |
||||||
|
* |
||||||
|
* - Redistributions in binary form must reproduce the above |
||||||
|
* copyright notice, this list of conditions and the following |
||||||
|
* disclaimer in the documentation and/or other materials provided |
||||||
|
* with the distribution. |
||||||
|
* |
||||||
|
* - Neither the name of the Eclipse Foundation, Inc. nor the |
||||||
|
* names of its contributors may be used to endorse or promote |
||||||
|
* products derived from this software without specific prior |
||||||
|
* written permission. |
||||||
|
* |
||||||
|
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND |
||||||
|
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, |
||||||
|
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
||||||
|
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
||||||
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR |
||||||
|
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
||||||
|
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
||||||
|
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
||||||
|
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER |
||||||
|
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, |
||||||
|
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
||||||
|
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF |
||||||
|
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.eclipse.jgit.internal.fsck; |
||||||
|
|
||||||
|
import java.io.IOException; |
||||||
|
import java.nio.ByteBuffer; |
||||||
|
import java.nio.channels.Channels; |
||||||
|
import java.text.MessageFormat; |
||||||
|
import java.util.Arrays; |
||||||
|
import java.util.HashSet; |
||||||
|
import java.util.List; |
||||||
|
import java.util.Set; |
||||||
|
import java.util.zip.CRC32; |
||||||
|
|
||||||
|
import org.eclipse.jgit.errors.CorruptObjectException; |
||||||
|
import org.eclipse.jgit.errors.CorruptPackIndexException; |
||||||
|
import org.eclipse.jgit.errors.CorruptPackIndexException.ErrorType; |
||||||
|
import org.eclipse.jgit.errors.MissingObjectException; |
||||||
|
import org.eclipse.jgit.internal.JGitText; |
||||||
|
import org.eclipse.jgit.internal.fsck.FsckError.CorruptObject; |
||||||
|
import org.eclipse.jgit.internal.storage.dfs.ReadableChannel; |
||||||
|
import org.eclipse.jgit.internal.storage.file.PackIndex; |
||||||
|
import org.eclipse.jgit.internal.storage.file.PackIndex.MutableEntry; |
||||||
|
import org.eclipse.jgit.lib.AnyObjectId; |
||||||
|
import org.eclipse.jgit.lib.ObjectChecker; |
||||||
|
import org.eclipse.jgit.lib.ObjectDatabase; |
||||||
|
import org.eclipse.jgit.transport.PackParser; |
||||||
|
import org.eclipse.jgit.transport.PackedObjectInfo; |
||||||
|
|
||||||
|
/** A read-only pack parser for object validity checking. */ |
||||||
|
public class FsckPackParser extends PackParser { |
||||||
|
private final CRC32 crc; |
||||||
|
|
||||||
|
private final ReadableChannel channel; |
||||||
|
|
||||||
|
private final Set<CorruptObject> corruptObjects = new HashSet<>(); |
||||||
|
|
||||||
|
private long expectedObjectCount = -1L; |
||||||
|
|
||||||
|
private long offset; |
||||||
|
|
||||||
|
private int blockSize; |
||||||
|
|
||||||
|
/** |
||||||
|
* @param db |
||||||
|
* the object database which stores repository's data. |
||||||
|
* @param channel |
||||||
|
* readable channel of the pack file. |
||||||
|
*/ |
||||||
|
public FsckPackParser(ObjectDatabase db, ReadableChannel channel) { |
||||||
|
super(db, Channels.newInputStream(channel)); |
||||||
|
this.channel = channel; |
||||||
|
setCheckObjectCollisions(false); |
||||||
|
this.crc = new CRC32(); |
||||||
|
this.blockSize = channel.blockSize() > 0 ? channel.blockSize() : 65536; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
protected void onPackHeader(long objCnt) throws IOException { |
||||||
|
if (expectedObjectCount >= 0) { |
||||||
|
// Some DFS pack files don't contain the correct object count, e.g.
|
||||||
|
// INSERT/RECEIVE packs don't always contain the correct object
|
||||||
|
// count in their headers. Overwrite the expected object count
|
||||||
|
// after parsing the pack header.
|
||||||
|
setExpectedObjectCount(expectedObjectCount); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
protected void onBeginWholeObject(long streamPosition, int type, |
||||||
|
long inflatedSize) throws IOException { |
||||||
|
crc.reset(); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
protected void onObjectHeader(Source src, byte[] raw, int pos, int len) |
||||||
|
throws IOException { |
||||||
|
crc.update(raw, pos, len); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
protected void onObjectData(Source src, byte[] raw, int pos, int len) |
||||||
|
throws IOException { |
||||||
|
crc.update(raw, pos, len); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
protected void onEndWholeObject(PackedObjectInfo info) throws IOException { |
||||||
|
info.setCRC((int) crc.getValue()); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
protected void onBeginOfsDelta(long deltaStreamPosition, |
||||||
|
long baseStreamPosition, long inflatedSize) throws IOException { |
||||||
|
crc.reset(); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
protected void onBeginRefDelta(long deltaStreamPosition, AnyObjectId baseId, |
||||||
|
long inflatedSize) throws IOException { |
||||||
|
crc.reset(); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
protected UnresolvedDelta onEndDelta() throws IOException { |
||||||
|
UnresolvedDelta delta = new UnresolvedDelta(); |
||||||
|
delta.setCRC((int) crc.getValue()); |
||||||
|
return delta; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
protected void onInflatedObjectData(PackedObjectInfo obj, int typeCode, |
||||||
|
byte[] data) throws IOException { |
||||||
|
// FsckPackParser ignores this event.
|
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
protected void verifySafeObject(final AnyObjectId id, final int type, |
||||||
|
final byte[] data) { |
||||||
|
try { |
||||||
|
super.verifySafeObject(id, type, data); |
||||||
|
} catch (CorruptObjectException e) { |
||||||
|
// catch the exception and continue parse the pack file
|
||||||
|
CorruptObject o = new CorruptObject(id.toObjectId(), type); |
||||||
|
if (e.getErrorType() != null) { |
||||||
|
o.setErrorType(e.getErrorType()); |
||||||
|
} |
||||||
|
corruptObjects.add(o); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
protected void onPackFooter(byte[] hash) throws IOException { |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
protected boolean onAppendBase(int typeCode, byte[] data, |
||||||
|
PackedObjectInfo info) throws IOException { |
||||||
|
// Do nothing.
|
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
protected void onEndThinPack() throws IOException { |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
protected ObjectTypeAndSize seekDatabase(PackedObjectInfo obj, |
||||||
|
ObjectTypeAndSize info) throws IOException { |
||||||
|
crc.reset(); |
||||||
|
offset = obj.getOffset(); |
||||||
|
return readObjectHeader(info); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
protected ObjectTypeAndSize seekDatabase(UnresolvedDelta delta, |
||||||
|
ObjectTypeAndSize info) throws IOException { |
||||||
|
crc.reset(); |
||||||
|
offset = delta.getOffset(); |
||||||
|
return readObjectHeader(info); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
protected int readDatabase(byte[] dst, int pos, int cnt) |
||||||
|
throws IOException { |
||||||
|
// read from input instead of database.
|
||||||
|
int n = read(offset, dst, pos, cnt); |
||||||
|
if (n > 0) { |
||||||
|
offset += n; |
||||||
|
} |
||||||
|
return n; |
||||||
|
} |
||||||
|
|
||||||
|
int read(long channelPosition, byte[] dst, int pos, int cnt) |
||||||
|
throws IOException { |
||||||
|
long block = channelPosition / blockSize; |
||||||
|
byte[] bytes = readFromChannel(block); |
||||||
|
if (bytes == null) { |
||||||
|
return -1; |
||||||
|
} |
||||||
|
int offset = (int) (channelPosition - block * blockSize); |
||||||
|
int bytesToCopy = Math.min(cnt, bytes.length - offset); |
||||||
|
if (bytesToCopy < 1) { |
||||||
|
return -1; |
||||||
|
} |
||||||
|
System.arraycopy(bytes, offset, dst, pos, bytesToCopy); |
||||||
|
return bytesToCopy; |
||||||
|
} |
||||||
|
|
||||||
|
private byte[] readFromChannel(long block) throws IOException { |
||||||
|
channel.position(block * blockSize); |
||||||
|
ByteBuffer buf = ByteBuffer.allocate(blockSize); |
||||||
|
int totalBytesRead = 0; |
||||||
|
while (totalBytesRead < blockSize) { |
||||||
|
int bytesRead = channel.read(buf); |
||||||
|
if (bytesRead == -1) { |
||||||
|
if (totalBytesRead == 0) { |
||||||
|
return null; |
||||||
|
} |
||||||
|
return Arrays.copyOf(buf.array(), totalBytesRead); |
||||||
|
} |
||||||
|
totalBytesRead += bytesRead; |
||||||
|
} |
||||||
|
return buf.array(); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
protected boolean checkCRC(int oldCRC) { |
||||||
|
return oldCRC == (int) crc.getValue(); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
protected void onStoreStream(byte[] raw, int pos, int len) |
||||||
|
throws IOException { |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* @return corrupt objects that reported by {@link ObjectChecker}. |
||||||
|
*/ |
||||||
|
public Set<CorruptObject> getCorruptObjects() { |
||||||
|
return corruptObjects; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Verify the existing index file with all objects from the pack. |
||||||
|
* |
||||||
|
* @param entries |
||||||
|
* all the entries that are expected in the index file |
||||||
|
* @param idx |
||||||
|
* index file associate with the pack |
||||||
|
* @throws CorruptPackIndexException |
||||||
|
* when the index file is corrupt. |
||||||
|
*/ |
||||||
|
public void verifyIndex(List<PackedObjectInfo> entries, PackIndex idx) |
||||||
|
throws CorruptPackIndexException { |
||||||
|
Set<String> all = new HashSet<>(); |
||||||
|
for (PackedObjectInfo entry : entries) { |
||||||
|
all.add(entry.getName()); |
||||||
|
long offset = idx.findOffset(entry); |
||||||
|
if (offset == -1) { |
||||||
|
throw new CorruptPackIndexException( |
||||||
|
MessageFormat.format(JGitText.get().missingObject, |
||||||
|
entry.getType(), entry.getName()), |
||||||
|
ErrorType.MISSING_OBJ); |
||||||
|
} else if (offset != entry.getOffset()) { |
||||||
|
throw new CorruptPackIndexException(MessageFormat |
||||||
|
.format(JGitText.get().mismatchOffset, entry.getName()), |
||||||
|
ErrorType.MISMATCH_OFFSET); |
||||||
|
} |
||||||
|
|
||||||
|
try { |
||||||
|
if (idx.hasCRC32Support() |
||||||
|
&& (int) idx.findCRC32(entry) != entry.getCRC()) { |
||||||
|
throw new CorruptPackIndexException( |
||||||
|
MessageFormat.format(JGitText.get().mismatchCRC, |
||||||
|
entry.getName()), |
||||||
|
ErrorType.MISMATCH_CRC); |
||||||
|
} |
||||||
|
} catch (MissingObjectException e) { |
||||||
|
throw new CorruptPackIndexException(MessageFormat |
||||||
|
.format(JGitText.get().missingCRC, entry.getName()), |
||||||
|
ErrorType.MISSING_CRC); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
for (MutableEntry entry : idx) { |
||||||
|
if (!all.contains(entry.name())) { |
||||||
|
throw new CorruptPackIndexException(MessageFormat.format( |
||||||
|
JGitText.get().unknownObjectInIndex, entry.name()), |
||||||
|
ErrorType.UNKNOWN_OBJ); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Set the object count for overwriting the expected object count from pack |
||||||
|
* header. |
||||||
|
* |
||||||
|
* @param expectedObjectCount |
||||||
|
* the actual expected object count. |
||||||
|
*/ |
||||||
|
public void overwriteObjectCount(long expectedObjectCount) { |
||||||
|
this.expectedObjectCount = expectedObjectCount; |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,4 @@ |
|||||||
|
/** |
||||||
|
* Git fsck support. |
||||||
|
*/ |
||||||
|
package org.eclipse.jgit.internal.fsck; |
||||||
@ -0,0 +1,134 @@ |
|||||||
|
/* |
||||||
|
* Copyright (C) 2017, Google Inc. |
||||||
|
* and other copyright owners as documented in the project's IP log. |
||||||
|
* |
||||||
|
* This program and the accompanying materials are made available |
||||||
|
* under the terms of the Eclipse Distribution License v1.0 which |
||||||
|
* accompanies this distribution, is reproduced below, and is |
||||||
|
* available at http://www.eclipse.org/org/documents/edl-v10.php
|
||||||
|
* |
||||||
|
* All rights reserved. |
||||||
|
* |
||||||
|
* Redistribution and use in source and binary forms, with or |
||||||
|
* without modification, are permitted provided that the following |
||||||
|
* conditions are met: |
||||||
|
* |
||||||
|
* - Redistributions of source code must retain the above copyright |
||||||
|
* notice, this list of conditions and the following disclaimer. |
||||||
|
* |
||||||
|
* - Redistributions in binary form must reproduce the above |
||||||
|
* copyright notice, this list of conditions and the following |
||||||
|
* disclaimer in the documentation and/or other materials provided |
||||||
|
* with the distribution. |
||||||
|
* |
||||||
|
* - Neither the name of the Eclipse Foundation, Inc. nor the |
||||||
|
* names of its contributors may be used to endorse or promote |
||||||
|
* products derived from this software without specific prior |
||||||
|
* written permission. |
||||||
|
* |
||||||
|
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND |
||||||
|
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, |
||||||
|
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
||||||
|
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
||||||
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR |
||||||
|
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
||||||
|
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
||||||
|
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
||||||
|
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER |
||||||
|
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, |
||||||
|
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
||||||
|
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF |
||||||
|
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.eclipse.jgit.internal.storage.dfs; |
||||||
|
|
||||||
|
import java.io.IOException; |
||||||
|
import java.util.List; |
||||||
|
|
||||||
|
import org.eclipse.jgit.errors.CorruptPackIndexException; |
||||||
|
import org.eclipse.jgit.errors.MissingObjectException; |
||||||
|
import org.eclipse.jgit.internal.fsck.FsckError; |
||||||
|
import org.eclipse.jgit.internal.fsck.FsckError.CorruptIndex; |
||||||
|
import org.eclipse.jgit.internal.fsck.FsckPackParser; |
||||||
|
import org.eclipse.jgit.internal.storage.pack.PackExt; |
||||||
|
import org.eclipse.jgit.lib.ObjectChecker; |
||||||
|
import org.eclipse.jgit.lib.ProgressMonitor; |
||||||
|
import org.eclipse.jgit.transport.PackedObjectInfo; |
||||||
|
|
||||||
|
/** Verify the validity and connectivity of a DFS repository. */ |
||||||
|
public class DfsFsck { |
||||||
|
private final DfsRepository repo; |
||||||
|
|
||||||
|
private final DfsObjDatabase objdb; |
||||||
|
|
||||||
|
private final DfsReader ctx; |
||||||
|
|
||||||
|
private ObjectChecker objChecker = new ObjectChecker(); |
||||||
|
|
||||||
|
/** |
||||||
|
* Initialize DFS fsck. |
||||||
|
* |
||||||
|
* @param repository |
||||||
|
* the dfs repository to check. |
||||||
|
*/ |
||||||
|
public DfsFsck(DfsRepository repository) { |
||||||
|
repo = repository; |
||||||
|
objdb = repo.getObjectDatabase(); |
||||||
|
ctx = objdb.newReader(); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* Verify the integrity and connectivity of all objects in the object |
||||||
|
* database. |
||||||
|
* |
||||||
|
* @param pm |
||||||
|
* callback to provide progress feedback during the check. |
||||||
|
* @return all errors about the repository. |
||||||
|
* @throws IOException |
||||||
|
* if encounters IO errors during the process. |
||||||
|
*/ |
||||||
|
public FsckError check(ProgressMonitor pm) throws IOException { |
||||||
|
FsckError errors = new FsckError(); |
||||||
|
try { |
||||||
|
for (DfsPackFile pack : objdb.getPacks()) { |
||||||
|
DfsPackDescription packDesc = pack.getPackDescription(); |
||||||
|
try (ReadableChannel channel = repo.getObjectDatabase() |
||||||
|
.openFile(packDesc, PackExt.PACK)) { |
||||||
|
List<PackedObjectInfo> objectsInPack; |
||||||
|
FsckPackParser parser = new FsckPackParser( |
||||||
|
repo.getObjectDatabase(), channel); |
||||||
|
parser.setObjectChecker(objChecker); |
||||||
|
parser.overwriteObjectCount(packDesc.getObjectCount()); |
||||||
|
parser.parse(pm); |
||||||
|
errors.getCorruptObjects() |
||||||
|
.addAll(parser.getCorruptObjects()); |
||||||
|
objectsInPack = parser.getSortedObjectList(null); |
||||||
|
parser.verifyIndex(objectsInPack, pack.getPackIndex(ctx)); |
||||||
|
} catch (MissingObjectException e) { |
||||||
|
errors.getMissingObjects().add(e.getObjectId()); |
||||||
|
} catch (CorruptPackIndexException e) { |
||||||
|
errors.getCorruptIndices().add(new CorruptIndex( |
||||||
|
pack.getPackDescription() |
||||||
|
.getFileName(PackExt.INDEX), |
||||||
|
e.getErrorType())); |
||||||
|
} |
||||||
|
} |
||||||
|
} finally { |
||||||
|
ctx.close(); |
||||||
|
} |
||||||
|
return errors; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Use a customized object checker instead of the default one. Caller can |
||||||
|
* specify a skip list to ignore some errors. |
||||||
|
* |
||||||
|
* @param objChecker |
||||||
|
* A customized object checker. |
||||||
|
*/ |
||||||
|
public void setObjectChecker(ObjectChecker objChecker) { |
||||||
|
this.objChecker = objChecker; |
||||||
|
} |
||||||
|
} |
||||||
Loading…
Reference in new issue