From fd74cf2f78c8e5c24792aa6b7b5df69d64018d09 Mon Sep 17 00:00:00 2001 From: Masaya Suzuki Date: Wed, 9 Aug 2017 23:30:03 -0700 Subject: [PATCH] Add BlobObjectChecker Some repositories can have a policy that do not accept certain blobs. To check if the incoming pack file contains such blobs, ObjectChecker can be used. However, this ObjectChecker is not called by PackParser if the blob is stored as a whole. This is because the object can be so large that it doesn't fit in memory. This change introduces BlobObjectChecker. This interface takes chunks of a blob instead of the entire object. ObjectChecker can optionally return a BlobObjectChecker. This won't change existing ObjectChecker implementation; existing implementation continues to receive deltified blob objects only. Change-Id: Ic33a92c2de42bd7a89786a4da26b7a648b25218d Signed-off-by: Masaya Suzuki --- .../eclipse/jgit/lib/ObjectCheckerTest.java | 98 +++++++++++++++++++ .../eclipse/jgit/lib/BlobObjectChecker.java | 92 +++++++++++++++++ .../org/eclipse/jgit/lib/ObjectChecker.java | 22 ++++- .../eclipse/jgit/transport/PackParser.java | 10 ++ 4 files changed, 221 insertions(+), 1 deletion(-) create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/lib/BlobObjectChecker.java diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/ObjectCheckerTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/ObjectCheckerTest.java index c8729d944..7f783536a 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/ObjectCheckerTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/ObjectCheckerTest.java @@ -63,6 +63,7 @@ import static org.eclipse.jgit.lib.ObjectChecker.ErrorType.HAS_DOTGIT; import static org.eclipse.jgit.lib.ObjectChecker.ErrorType.NULL_SHA1; import static org.eclipse.jgit.lib.ObjectChecker.ErrorType.TREE_NOT_SORTED; import static org.eclipse.jgit.lib.ObjectChecker.ErrorType.ZERO_PADDED_FILEMODE; +import static org.eclipse.jgit.util.RawParseUtils.decode; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertSame; import static org.junit.Assert.fail; @@ -73,11 +74,16 @@ import java.text.MessageFormat; import org.eclipse.jgit.errors.CorruptObjectException; import org.eclipse.jgit.internal.JGitText; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; public class ObjectCheckerTest { private ObjectChecker checker; + @Rule + public final ExpectedException thrown = ExpectedException.none(); + @Before public void setUp() throws Exception { checker = new ObjectChecker(); @@ -101,6 +107,98 @@ public class ObjectCheckerTest { checker.check(OBJ_BLOB, new byte[1]); } + @Test + public void testCheckBlobNotCorrupt() throws CorruptObjectException { + checker = new ObjectChecker() { + @Override + public void checkBlob(byte[] raw) throws CorruptObjectException { + String in = decode(raw); + if (in.contains("secret_key")) { + throw new CorruptObjectException("don't add a secret key"); + } + } + }; + checker.check(OBJ_BLOB, encodeASCII("key = \"public_key\"")); + } + + @Test + public void testCheckBlobCorrupt() throws CorruptObjectException { + checker = new ObjectChecker() { + @Override + public void checkBlob(byte[] raw) throws CorruptObjectException { + String in = decode(raw); + if (in.contains("secret_key")) { + throw new CorruptObjectException("don't add a secret key"); + } + } + }; + thrown.expect(CorruptObjectException.class); + checker.check(OBJ_BLOB, encodeASCII("key = \"secret_key\"")); + } + + @Test + public void testCheckBlobWithBlobObjectCheckerNotCorrupt() + throws CorruptObjectException { + checker = new ObjectChecker() { + @Override + public BlobObjectChecker newBlobObjectChecker() { + return new BlobObjectChecker() { + private boolean containSecretKey; + + @Override + public void update(byte[] in, int offset, int len) { + String str = decode(in, offset, offset + len); + if (str.contains("secret_key")) { + containSecretKey = true; + } + } + + @Override + public void endBlob(AnyObjectId id) + throws CorruptObjectException { + if (containSecretKey) { + throw new CorruptObjectException( + "don't add a secret key"); + } + } + }; + } + }; + checker.check(OBJ_BLOB, encodeASCII("key = \"public_key\"")); + } + + @Test + public void testCheckBlobWithBlobObjectCheckerCorrupt() + throws CorruptObjectException { + checker = new ObjectChecker() { + @Override + public BlobObjectChecker newBlobObjectChecker() { + return new BlobObjectChecker() { + private boolean containSecretKey; + + @Override + public void update(byte[] in, int offset, int len) { + String str = decode(in, offset, offset + len); + if (str.contains("secret_key")) { + containSecretKey = true; + } + } + + @Override + public void endBlob(AnyObjectId id) + throws CorruptObjectException { + if (containSecretKey) { + throw new CorruptObjectException( + "don't add a secret key"); + } + } + }; + } + }; + thrown.expect(CorruptObjectException.class); + checker.check(OBJ_BLOB, encodeASCII("key = \"secret_key\"")); + } + @Test public void testValidCommitNoParent() throws CorruptObjectException { StringBuilder b = new StringBuilder(); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/BlobObjectChecker.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/BlobObjectChecker.java new file mode 100644 index 000000000..0fe63ae5b --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/BlobObjectChecker.java @@ -0,0 +1,92 @@ +/* + * Copyright (C) 2017, Google Inc. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.eclipse.jgit.lib; + +import org.eclipse.jgit.errors.CorruptObjectException; + +/** + * Verifies that a blob object is a valid object. + *

+ * Unlike trees, commits and tags, there's no validity of blobs. Implementers + * can optionally implement this blob checker to reject certain blobs. + * + * @since 4.9 + */ +public interface BlobObjectChecker { + /** No-op implementation of {@link BlobObjectChecker}. */ + public static final BlobObjectChecker NULL_CHECKER = + new BlobObjectChecker() { + @Override + public void update(byte[] in, int p, int len) { + // Empty implementation. + } + + @Override + public void endBlob(AnyObjectId id) { + // Empty implementation. + } + }; + + /** + * Check a new fragment of the blob. + * + * @param in + * input array of bytes. + * @param offset + * offset to start at from {@code in}. + * @param len + * length of the fragment to check. + */ + void update(byte[] in, int offset, int len); + + /** + * Finalize the blob checking. + * + * @param id + * identity of the object being checked. + * @throws CorruptObjectException + * if any error was detected. + */ + void endBlob(AnyObjectId id) throws CorruptObjectException; +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectChecker.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectChecker.java index 9d3aee150..19c5c7eb4 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectChecker.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectChecker.java @@ -359,7 +359,13 @@ public class ObjectChecker { checkTree(id, raw); break; case OBJ_BLOB: - checkBlob(raw); + BlobObjectChecker checker = newBlobObjectChecker(); + if (checker == null) { + checkBlob(raw); + } else { + checker.update(raw, 0, raw.length); + checker.endBlob(id); + } break; default: report(UNKNOWN_TYPE, id, MessageFormat.format( @@ -1066,9 +1072,23 @@ public class ObjectChecker { return '1' <= b && b <= '9'; } + /** + * Create a new {@link BlobObjectChecker}. + * + * @return new BlobObjectChecker or null if it's not provided. + * @since 4.9 + */ + @Nullable + public BlobObjectChecker newBlobObjectChecker() { + return null; + } + /** * Check a blob for errors. * + *

This may not be called from PackParser in some cases. Use {@link + * #newBlobObjectChecker} instead. + * * @param raw * the blob data. The array is never modified. * @throws CorruptObjectException diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/PackParser.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PackParser.java index 2f6b271d8..833d2114c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/PackParser.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PackParser.java @@ -66,6 +66,7 @@ import org.eclipse.jgit.internal.storage.file.PackLock; import org.eclipse.jgit.internal.storage.pack.BinaryDelta; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.BatchingProgressMonitor; +import org.eclipse.jgit.lib.BlobObjectChecker; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.InflaterCache; import org.eclipse.jgit.lib.MutableObjectId; @@ -1043,16 +1044,25 @@ public abstract class PackParser { if (type == Constants.OBJ_BLOB) { byte[] readBuffer = buffer(); InputStream inf = inflate(Source.INPUT, sz); + BlobObjectChecker checker = null; + if (objCheck != null) { + checker = objCheck.newBlobObjectChecker(); + } + if (checker == null) { + checker = BlobObjectChecker.NULL_CHECKER; + } long cnt = 0; while (cnt < sz) { int r = inf.read(readBuffer); if (r <= 0) break; objectDigest.update(readBuffer, 0, r); + checker.update(readBuffer, 0, r); cnt += r; } inf.close(); objectDigest.digest(tempObjectId); + checker.endBlob(tempObjectId); data = null; } else { data = inflateAndReturn(Source.INPUT, sz);