Reject mixed case .git on Mac OS in ObjectChecker

Most Mac OS X systems use a case insensitive HFS+ volume. Like
Windows ".git" and ".GIT" are the same path and can confuse a Git
program into expecting a repository where one does not exist.

Change-Id: Iec6ce9e6c2872f8b0850cc6aec023fa0fcb05ae4

org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/ObjectCheckerTest.java

@@ -1294,7 +1294,7 @@ public class ObjectCheckerTest {
 	}
 
 	@Test
-	public void testInvalidTreeNameIsMixedCaseGit() {
+	public void testInvalidTreeNameIsMixedCaseGitWindows() {
 		StringBuilder b = new StringBuilder();
 		entry(b, "100644 .GiT");
 		byte[] data = Constants.encodeASCII(b.toString());
@@ -1307,6 +1307,20 @@ public class ObjectCheckerTest {
 		}
 	}
 
+	@Test
+	public void testInvalidTreeNameIsMixedCaseGitMacOS() {
+		StringBuilder b = new StringBuilder();
+		entry(b, "100644 .GiT");
+		byte[] data = Constants.encodeASCII(b.toString());
+		try {
+			checker.setSafeForMacOS(true);
+			checker.checkTree(data);
+			fail("incorrectly accepted an invalid tree");
+		} catch (CorruptObjectException e) {
+			assertEquals("invalid name '.GiT'", e.getMessage());
+		}
+	}
+
 	@Test
 	public void testInvalidTreeTruncatedInName() {
 		final StringBuilder b = new StringBuilder();

org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectChecker.java

@@ -100,6 +100,7 @@ public class ObjectChecker {
 
 	private boolean allowZeroMode;
 	private boolean windows;
+	private boolean macosx;
 
 	/**
 	 * Enable accepting leading zero mode in tree entries.
@@ -132,6 +133,21 @@ public class ObjectChecker {
 		return this;
 	}
 
+	/**
+	 * Restrict trees to only names legal on Mac OS X platforms.
+	 * <p>
+	 * Rejects any mixed case forms of reserved names ({@code .git})
+	 * for users working on HFS+ in case-insensitive (default) mode.
+	 *
+	 * @param mac true if Mac OS X name checking should be performed.
+	 * @return {@code this}.
+	 * @since 3.4
+	 */
+	public ObjectChecker setSafeForMacOS(boolean mac) {
+		macosx = mac;
+		return this;
+	}
+
 	/**
 	 * Check an object for parsing errors.
 	 *
@@ -491,7 +507,7 @@ public class ObjectChecker {
 	}
 
 	private boolean isDotGit(byte[] buf, int p) {
-		if (windows)
+		if (windows || macosx)
 			return toLower(buf[p]) == 'g'
 					&& toLower(buf[p + 1]) == 'i'
 					&& toLower(buf[p + 2]) == 't';