From 1fbe688f51cbc1d951dee5443ae0b9d801904c93 Mon Sep 17 00:00:00 2001 From: Sasa Zivkov Date: Tue, 22 Nov 2011 17:43:00 +0100 Subject: [PATCH] maxObjectSizeLimit for receive-pack. ReceivePack (and PackParser) can be configured with the maxObjectSizeLimit in order to prevent users from pushing too large objects to Git. The limit check is applied to all object types although it is most likely that a BLOB will exceed the limit. In all cases the size of the object header is excluded from the object size which is checked against the limit as this is the size of which a BLOB object would take in the working tree when checked out as a file. When an object exceeds the maxObjectSizeLimit the receive-pack will abort immediately. Delta objects (both offset and ref delta) are also checked against the limit. However, for delta objects we will first check the size of the inflated delta block against the maxObjectSizeLimit and abort immediately if it exceeds the limit. In this case we even do not know the exact size of the resolved delta object but we assume it will be larger than the given maxObjectSizeLimit as delta is generally only chosen if the delta can copy more data from the base object than the delta needs to insert or needs to represent the copy ranges. Aborting early, in this case, avoids unnecessary inflating of the (huge) delta block. Unfortunately, it is too expensive (especially for a large delta) to compute SHA-1 of an object that causes the receive-pack to abort. This would decrease the value of this feature whose main purpose is to protect server resources from users pushing huge objects. Therefore we don't report the SHA-1 in the error message. Change-Id: I177ef24553faacda444ed5895e40ac8925ca0d1e Signed-off-by: Sasa Zivkov Signed-off-by: Matthias Sohn --- .../jgit/transport/PackParserTest.java | 95 +++++++++++++++++++ .../org/eclipse/jgit/JGitText.properties | 2 + .../src/org/eclipse/jgit/JGitText.java | 2 + .../errors/TooLargeObjectInPackException.java | 82 ++++++++++++++++ .../eclipse/jgit/transport/PackParser.java | 46 ++++++++- .../eclipse/jgit/transport/ReceivePack.java | 17 ++++ 6 files changed, 242 insertions(+), 2 deletions(-) create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/errors/TooLargeObjectInPackException.java diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/PackParserTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/PackParserTest.java index 01e69cb44..0de58bdf5 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/PackParserTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/PackParserTest.java @@ -47,6 +47,7 @@ package org.eclipse.jgit.transport; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -60,6 +61,7 @@ import java.text.MessageFormat; import java.util.zip.Deflater; import org.eclipse.jgit.JGitText; +import org.eclipse.jgit.errors.TooLargeObjectInPackException; import org.eclipse.jgit.junit.JGitTestUtil; import org.eclipse.jgit.junit.TestRepository; import org.eclipse.jgit.lib.Constants; @@ -209,6 +211,99 @@ public class PackParserTest extends RepositoryTestCase { } } + @Test + public void testMaxObjectSizeFullBlob() throws Exception { + TestRepository d = new TestRepository(db); + final byte[] data = Constants.encode("0123456789"); + d.blob(data); + + TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(1024); + + packHeader(pack, 1); + pack.write((Constants.OBJ_BLOB) << 4 | 10); + deflate(pack, data); + digest(pack); + + PackParser p = index(new ByteArrayInputStream(pack.toByteArray())); + p.setMaxObjectSizeLimit(11); + p.parse(NullProgressMonitor.INSTANCE); + + p = index(new ByteArrayInputStream(pack.toByteArray())); + p.setMaxObjectSizeLimit(10); + p.parse(NullProgressMonitor.INSTANCE); + + p = index(new ByteArrayInputStream(pack.toByteArray())); + p.setMaxObjectSizeLimit(9); + try { + p.parse(NullProgressMonitor.INSTANCE); + fail("PackParser should have failed"); + } catch (TooLargeObjectInPackException e) { + assertTrue(e.getMessage().contains("10")); // obj size + assertTrue(e.getMessage().contains("9")); // max obj size + } + } + + @Test + public void testMaxObjectSizeDeltaBlock() throws Exception { + TestRepository d = new TestRepository(db); + RevBlob a = d.blob("a"); + + TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(1024); + + packHeader(pack, 1); + pack.write((Constants.OBJ_REF_DELTA) << 4 | 14); + a.copyRawTo(pack); + deflate(pack, new byte[] { 1, 11, 11, 'a', '0', '1', '2', '3', '4', + '5', '6', '7', '8', '9' }); + digest(pack); + + PackParser p = index(new ByteArrayInputStream(pack.toByteArray())); + p.setAllowThin(true); + p.setMaxObjectSizeLimit(14); + p.parse(NullProgressMonitor.INSTANCE); + + p = index(new ByteArrayInputStream(pack.toByteArray())); + p.setAllowThin(true); + p.setMaxObjectSizeLimit(13); + try { + p.parse(NullProgressMonitor.INSTANCE); + fail("PackParser should have failed"); + } catch (TooLargeObjectInPackException e) { + assertTrue(e.getMessage().contains("13")); // max obj size + assertFalse(e.getMessage().contains("14")); // no delta size + } + } + + @Test + public void testMaxObjectSizeDeltaResultSize() throws Exception { + TestRepository d = new TestRepository(db); + RevBlob a = d.blob("0123456789"); + + TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(1024); + + packHeader(pack, 1); + pack.write((Constants.OBJ_REF_DELTA) << 4 | 4); + a.copyRawTo(pack); + deflate(pack, new byte[] { 10, 11, 1, 'a' }); + digest(pack); + + PackParser p = index(new ByteArrayInputStream(pack.toByteArray())); + p.setAllowThin(true); + p.setMaxObjectSizeLimit(11); + p.parse(NullProgressMonitor.INSTANCE); + + p = index(new ByteArrayInputStream(pack.toByteArray())); + p.setAllowThin(true); + p.setMaxObjectSizeLimit(10); + try { + p.parse(NullProgressMonitor.INSTANCE); + fail("PackParser should have failed"); + } catch (TooLargeObjectInPackException e) { + assertTrue(e.getMessage().contains("11")); // result obj size + assertTrue(e.getMessage().contains("10")); // max obj size + } + } + private void packHeader(TemporaryBuffer.Heap tinyPack, int cnt) throws IOException { final byte[] hdr = new byte[8]; diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/JGitText.properties index 9fe603ff8..a5fea35ab 100644 --- a/org.eclipse.jgit/resources/org/eclipse/jgit/JGitText.properties +++ b/org.eclipse.jgit/resources/org/eclipse/jgit/JGitText.properties @@ -359,6 +359,8 @@ pushNotPermitted=push not permitted rawLogMessageDoesNotParseAsLogEntry=Raw log message does not parse as log entry readTimedOut=Read timed out readingObjectsFromLocalRepositoryFailed=reading objects from local repository failed: {0} +receivePackObjectTooLarge1=Object too large, rejecting the pack. Max object size limit is {0} bytes. +receivePackObjectTooLarge2=Object too large ({0} bytes), rejecting the pack. Max object size limit is {1} bytes. receivingObjects=Receiving objects refAlreadyExists=Ref {0} already exists refNotResolved=Ref {0} can not be resolved diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/JGitText.java index e64534753..fcc6cafd5 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/JGitText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/JGitText.java @@ -419,6 +419,8 @@ public class JGitText extends TranslationBundle { /***/ public String rawLogMessageDoesNotParseAsLogEntry; /***/ public String readTimedOut; /***/ public String readingObjectsFromLocalRepositoryFailed; + /***/ public String receivePackObjectTooLarge1; + /***/ public String receivePackObjectTooLarge2; /***/ public String receivingObjects; /***/ public String refAlreadyExists; /***/ public String refNotResolved; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/errors/TooLargeObjectInPackException.java b/org.eclipse.jgit/src/org/eclipse/jgit/errors/TooLargeObjectInPackException.java new file mode 100644 index 000000000..dbe480ab9 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/errors/TooLargeObjectInPackException.java @@ -0,0 +1,82 @@ +/* + * Copyright (C) 2011, Sasa Zivkov + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.eclipse.jgit.errors; + +import java.io.IOException; +import java.text.MessageFormat; + +import org.eclipse.jgit.JGitText; + +/** Thrown when PackParser finds an object larger than a predefined limit */ +public class TooLargeObjectInPackException extends IOException { + private static final long serialVersionUID = 1L; + + /** + * Construct a too large object in pack exception when the exact size of the + * too large object is not available. This will be used when we find out + * that a delta sequence is already larger than the maxObjectSizeLimit but + * don't want to inflate the delta just to find out the exact size of the + * resulting object. + * + * @param maxObjectSizeLimit + * the maximum object size limit + */ + public TooLargeObjectInPackException(long maxObjectSizeLimit) { + super(MessageFormat.format(JGitText.get().receivePackObjectTooLarge1, + maxObjectSizeLimit)); + } + + /** + * Construct a too large object in pack exception when the exact size of the + * too large object is known. + * + * @param objectSize + * @param maxObjectSizeLimit + */ + public TooLargeObjectInPackException(long objectSize, + long maxObjectSizeLimit) { + super(MessageFormat.format(JGitText.get().receivePackObjectTooLarge2, + objectSize, maxObjectSizeLimit)); + } +} \ No newline at end of file diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/PackParser.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PackParser.java index 1b30e859e..c62959565 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/PackParser.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PackParser.java @@ -61,6 +61,7 @@ import java.util.zip.Inflater; import org.eclipse.jgit.JGitText; import org.eclipse.jgit.errors.CorruptObjectException; import org.eclipse.jgit.errors.MissingObjectException; +import org.eclipse.jgit.errors.TooLargeObjectInPackException; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.BatchingProgressMonitor; import org.eclipse.jgit.lib.Constants; @@ -180,6 +181,9 @@ public abstract class PackParser { /** Message to protect the pack data from garbage collection. */ private String lockMessage; + /** Git object size limit */ + private long maxObjectSizeLimit; + /** * Initialize a pack parser. * @@ -365,6 +369,19 @@ public abstract class PackParser { lockMessage = msg; } + /** + * Set the maximum allowed Git object size. + *

+ * If an object is larger than the given size the pack-parsing will throw an + * exception aborting the parsing. + * + * @param limit + * the Git object size limit. If zero then there is not limit. + */ + public void setMaxObjectSizeLimit(long limit) { + maxObjectSizeLimit = limit; + } + /** * Get the number of objects in the stream. *

@@ -584,8 +601,11 @@ public abstract class PackParser { JGitText.get().unknownObjectType, info.type)); } - visit.data = BinaryDelta.apply(visit.parent.data, // - inflateAndReturn(Source.DATABASE, info.size)); + byte[] delta = inflateAndReturn(Source.DATABASE, info.size); + checkIfTooLarge(type, BinaryDelta.getResultSize(delta)); + + visit.data = BinaryDelta.apply(visit.parent.data, delta); + delta = null; if (!checkCRC(visit.delta.crc)) throw new IOException(MessageFormat.format( @@ -613,6 +633,26 @@ public abstract class PackParser { } while (visit != null); } + private final void checkIfTooLarge(int typeCode, long size) + throws IOException { + if (0 < maxObjectSizeLimit && maxObjectSizeLimit < size) + switch (typeCode) { + case Constants.OBJ_COMMIT: + case Constants.OBJ_TREE: + case Constants.OBJ_BLOB: + case Constants.OBJ_TAG: + throw new TooLargeObjectInPackException(size, maxObjectSizeLimit); + + case Constants.OBJ_OFS_DELTA: + case Constants.OBJ_REF_DELTA: + throw new TooLargeObjectInPackException(maxObjectSizeLimit); + + default: + throw new IOException(MessageFormat.format( + JGitText.get().unknownObjectType, typeCode)); + } + } + /** * Read the header of the current object. *

@@ -856,6 +896,8 @@ public abstract class PackParser { shift += 7; } + checkIfTooLarge(typeCode, sz); + switch (typeCode) { case Constants.OBJ_COMMIT: case Constants.OBJ_TREE: diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/ReceivePack.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/ReceivePack.java index c6fe4d18c..04f8946b2 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/ReceivePack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/ReceivePack.java @@ -193,6 +193,9 @@ public class ReceivePack { private boolean checkReferencedIsReachable; + /** Git object size limit */ + private long maxObjectSizeLimit; + /** * Create a new pack receive for an open repository. * @@ -493,6 +496,19 @@ public class ReceivePack { timeout = seconds; } + /** + * Set the maximum allowed Git object size. + *

+ * If an object is larger than the given size the pack-parsing will throw an + * exception aborting the receive-pack operation. + * + * @param limit + * the Git object size limit. If zero then there is not limit. + */ + public void setMaxObjectSizeLimit(final long limit) { + maxObjectSizeLimit = limit; + } + /** @return all of the command received by the current request. */ public List getAllCommands() { return Collections.unmodifiableList(commands); @@ -829,6 +845,7 @@ public class ReceivePack { parser.setCheckEofAfterPackFooter(!biDirectionalPipe); parser.setObjectChecking(isCheckReceivedObjects()); parser.setLockMessage(lockMsg); + parser.setMaxObjectSizeLimit(maxObjectSizeLimit); packLock = parser.parse(receiving, resolving); ins.flush(); } finally {