Merge branch 'stable-4.7' into stable-4.8

* stable-4.7:
  Run auto GC in the background

Change-Id: I5e25765f65d833f13cbe99696ef33055d7f5c4cf
Signed-off-by: Matthias Sohn <matthias.sohn@sap.com>

org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/LocalDiskRepositoryTestCase.java

@@ -62,6 +62,7 @@ import java.util.TreeSet;
 import org.eclipse.jgit.dircache.DirCache;
 import org.eclipse.jgit.dircache.DirCacheEntry;
 import org.eclipse.jgit.internal.storage.file.FileRepository;
+import org.eclipse.jgit.lib.ConfigConstants;
 import org.eclipse.jgit.lib.Constants;
 import org.eclipse.jgit.lib.ObjectId;
 import org.eclipse.jgit.lib.PersonIdent;
@@ -121,6 +122,12 @@ public abstract class LocalDiskRepositoryTestCase {
 		mockSystemReader = new MockSystemReader();
 		mockSystemReader.userGitConfig = new FileBasedConfig(new File(tmp,
 				"usergitconfig"), FS.DETECTED);
+		// We have to set autoDetach to false for tests, because tests expect to be able
+		// to clean up by recursively removing the repository, and background GC might be
+		// in the middle of writing or deleting files, which would disrupt this.
+		mockSystemReader.userGitConfig.setBoolean(ConfigConstants.CONFIG_GC_SECTION,
+				null, ConfigConstants.CONFIG_KEY_AUTODETACH, false);
+		mockSystemReader.userGitConfig.save();
 		ceilTestDirectories(getCeilings());
 		SystemReader.setInstance(mockSystemReader);
 

org.eclipse.jgit.pgm.test/tst/org/eclipse/jgit/pgm/ConfigTest.java

@@ -74,6 +74,7 @@ public class ConfigTest extends CLIRepositoryTestCase {
 
 		String[] output = execute("git config --list");
 		List<String> expect = new ArrayList<>();
+		expect.add("gc.autoDetach=false");
 		expect.add("core.filemode=" + !isWindows);
 		expect.add("core.logallrefupdates=true");
 		if (isMac)

org.eclipse.jgit/.settings/.api_filters

@@ -1,5 +1,21 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
 <component id="org.eclipse.jgit" version="2">
+    <resource path="src/org/eclipse/jgit/lib/ConfigConstants.java" type="org.eclipse.jgit.lib.ConfigConstants">
+        <filter id="1141899266">
+            <message_arguments>
+                <message_argument value="4.7"/>
+                <message_argument value="4.8"/>
+                <message_argument value="CONFIG_KEY_AUTODETACH"/>
+            </message_arguments>
+        </filter>
+        <filter id="1141899266">
+            <message_arguments>
+                <message_argument value="4.7"/>
+                <message_argument value="4.8"/>
+                <message_argument value="CONFIG_KEY_LOGEXPIRY"/>
+            </message_arguments>
+        </filter>
+    </resource>
     <resource path="src/org/eclipse/jgit/merge/MergeStrategy.java" type="org.eclipse.jgit.merge.MergeStrategy">
         <filter comment="OSGi semantic versioning allows breaking implementors of an API in a minor version" id="336695337">
             <message_arguments>

org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties

@@ -299,6 +299,8 @@ flagNotFromThis={0} not from this.
 flagsAlreadyCreated={0} flags already created.
 funnyRefname=funny refname
 gcFailed=Garbage collection failed.
+gcLogExists=A previous GC run reported an error: ''{0}''.  Automatic gc will fail until ''{1}'' is removed.
+gcTooManyUnpruned=Too many loose, unpruneable objects after garbage collection.  Consider adjusting gc.auto or gc.pruneExpire.
 gitmodulesNotFound=.gitmodules not found in tree.
 headRequiredToStash=HEAD required to stash local changes
 hoursAgo={0} hours ago

org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java

@@ -358,6 +358,8 @@ public class JGitText extends TranslationBundle {
 	/***/ public String flagsAlreadyCreated;
 	/***/ public String funnyRefname;
 	/***/ public String gcFailed;
+	/***/ public String gcLogExists;
+	/***/ public String gcTooManyUnpruned;
 	/***/ public String gitmodulesNotFound;
 	/***/ public String headRequiredToStash;
 	/***/ public String hoursAgo;

org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java

@@ -637,12 +637,18 @@ public class FileRepository extends Repository {
 
 	}
 
+	private boolean shouldAutoDetach() {
+		return getConfig().getBoolean(ConfigConstants.CONFIG_GC_SECTION,
+				ConfigConstants.CONFIG_KEY_AUTODETACH, true);
+	}
+
 	@Override
 	public void autoGC(ProgressMonitor monitor) {
 		GC gc = new GC(this);
 		gc.setPackConfig(new PackConfig(this));
 		gc.setProgressMonitor(monitor);
 		gc.setAuto(true);
+		gc.setBackground(shouldAutoDetach());
 		try {
 			gc.gc();
 		} catch (ParseException | IOException e) {

org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java

@@ -50,6 +50,8 @@ import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.OutputStream;
+import java.io.PrintWriter;
+import java.io.StringWriter;
 import java.nio.channels.Channels;
 import java.nio.channels.FileChannel;
 import java.nio.file.DirectoryStream;
@@ -73,11 +75,17 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.TreeMap;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import org.eclipse.jgit.annotations.NonNull;
+import org.eclipse.jgit.api.errors.JGitInternalException;
 import org.eclipse.jgit.dircache.DirCacheIterator;
 import org.eclipse.jgit.errors.CancelledException;
 import org.eclipse.jgit.errors.CorruptObjectException;
@@ -143,6 +151,8 @@ public class GC {
 
 	private static final int DEFAULT_AUTOLIMIT = 6700;
 
+	private static ExecutorService executor = Executors.newFixedThreadPool(1);
+
 	private final FileRepository repo;
 
 	private ProgressMonitor pm;
@@ -177,6 +187,11 @@ public class GC {
 	 */
 	private boolean automatic;
 
+	/**
+	 * Whether to run gc in a background thread
+	 */
+	private boolean background;
+
 	/**
 	 * Creates a new garbage collector with default values. An expirationTime of
 	 * two weeks and <code>null</code> as progress monitor will be used.
@@ -202,13 +217,73 @@ public class GC {
 	 * first check whether any housekeeping is required; if not, it exits
 	 * without performing any work.
 	 *
+	 * If {@link #setBackground(boolean)} was set to {@code true}
+	 * {@code collectGarbage} will start the gc in the background, and then
+	 * return immediately. In this case, errors will not be reported except in
+	 * gc.log.
+	 *
 	 * @return the collection of {@link PackFile}'s which are newly created
 	 * @throws IOException
 	 * @throws ParseException
 	 *             If the configuration parameter "gc.pruneexpire" couldn't be
 	 *             parsed
 	 */
+	// TODO(ms): in 5.0 change signature and return Future<Collection<PackFile>>
 	public Collection<PackFile> gc() throws IOException, ParseException {
+		final GcLog gcLog = background ? new GcLog(repo) : null;
+		if (gcLog != null && !gcLog.lock(background)) {
+			// there is already a background gc running
+			return Collections.emptyList();
+		}
+
+		Callable<Collection<PackFile>> gcTask = () -> {
+			try {
+				Collection<PackFile> newPacks = doGc();
+				if (automatic && tooManyLooseObjects() && gcLog != null) {
+					String message = JGitText.get().gcTooManyUnpruned;
+					gcLog.write(message);
+					gcLog.commit();
+				}
+				return newPacks;
+			} catch (IOException | ParseException e) {
+				if (background) {
+					if (gcLog == null) {
+						// Lacking a log, there's no way to report this.
+						return Collections.emptyList();
+					}
+					try {
+						gcLog.write(e.getMessage());
+						StringWriter sw = new StringWriter();
+						e.printStackTrace(new PrintWriter(sw));
+						gcLog.write(sw.toString());
+						gcLog.commit();
+					} catch (IOException e2) {
+						e2.addSuppressed(e);
+						LOG.error(e2.getMessage(), e2);
+					}
+				} else {
+					throw new JGitInternalException(e.getMessage(), e);
+				}
+			} finally {
+				if (gcLog != null) {
+					gcLog.unlock();
+				}
+			}
+			return Collections.emptyList();
+		};
+		Future<Collection<PackFile>> result = executor.submit(gcTask);
+		if (background) {
+			// TODO(ms): in 5.0 change signature and return the Future
+			return Collections.emptyList();
+		}
+		try {
+			return result.get();
+		} catch (InterruptedException | ExecutionException e) {
+			throw new IOException(e);
+		}
+	}
+
+	private Collection<PackFile> doGc() throws IOException, ParseException {
 		if (automatic && !needGc()) {
 			return Collections.emptyList();
 		}
@@ -1365,6 +1440,14 @@ public class GC {
 		this.automatic = auto;
 	}
 
+	/**
+	 * @param background
+	 *            whether to run the gc in a background thread.
+	 */
+	void setBackground(boolean background) {
+		this.background = background;
+	}
+
 	private boolean needGc() {
 		if (tooManyPacks()) {
 			addRepackAllOption();
@@ -1403,8 +1486,7 @@ public class GC {
 	 * @return {@code true} if number of loose objects > gc.auto (default 6700)
 	 */
 	boolean tooManyLooseObjects() {
-		int auto = repo.getConfig().getInt(ConfigConstants.CONFIG_GC_SECTION,
+		int auto = getLooseObjectLimit();
-				ConfigConstants.CONFIG_KEY_AUTO, DEFAULT_AUTOLIMIT);
 		if (auto <= 0) {
 			return false;
 		}
@@ -1436,4 +1518,9 @@ public class GC {
 		}
 		return false;
 	}
+
+	private int getLooseObjectLimit() {
+		return repo.getConfig().getInt(ConfigConstants.CONFIG_GC_SECTION,
+				ConfigConstants.CONFIG_KEY_AUTO, DEFAULT_AUTOLIMIT);
+	}
 }

org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GcLog.java

@@ -0,0 +1,191 @@
+/*
+ * Copyright (C) 2017 Two Sigma Open Source
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.eclipse.jgit.internal.storage.file;
+
+import org.eclipse.jgit.api.errors.JGitInternalException;
+import org.eclipse.jgit.internal.JGitText;
+import org.eclipse.jgit.lib.ConfigConstants;
+import org.eclipse.jgit.util.GitDateParser;
+import org.eclipse.jgit.util.SystemReader;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.attribute.FileTime;
+import java.text.MessageFormat;
+import java.text.ParseException;
+import java.time.Instant;
+
+/**
+ * This class manages the gc.log file for a {@link FileRepository}.
+ */
+class GcLog {
+	private final FileRepository repo;
+
+	private final File logFile;
+
+	private final LockFile lock;
+
+	private Instant gcLogExpire;
+
+	private static final String LOG_EXPIRY_DEFAULT = "1.day.ago"; //$NON-NLS-1$
+
+	private boolean nonEmpty = false;
+
+	/**
+	 * Construct a GcLog object for a {@link FileRepository}
+	 *
+	 * @param repo
+	 *            the repository
+	 */
+	GcLog(FileRepository repo) {
+		this.repo = repo;
+		logFile = new File(repo.getDirectory(), "gc.log"); //$NON-NLS-1$
+		lock = new LockFile(logFile);
+	}
+
+	private Instant getLogExpiry() throws ParseException {
+		if (gcLogExpire == null) {
+			String logExpiryStr = repo.getConfig().getString(
+					ConfigConstants.CONFIG_GC_SECTION, null,
+					ConfigConstants.CONFIG_KEY_LOGEXPIRY);
+			if (logExpiryStr == null) {
+				logExpiryStr = LOG_EXPIRY_DEFAULT;
+			}
+			gcLogExpire = GitDateParser.parse(logExpiryStr, null,
+					SystemReader.getInstance().getLocale()).toInstant();
+		}
+		return gcLogExpire;
+	}
+
+	private boolean autoGcBlockedByOldLockFile(boolean background) {
+		try {
+			FileTime lastModified = Files.getLastModifiedTime(logFile.toPath());
+			if (lastModified.toInstant().compareTo(getLogExpiry()) > 0) {
+				// There is an existing log file, which is too recent to ignore
+				if (!background) {
+					try (BufferedReader reader = Files
+							.newBufferedReader(logFile.toPath())) {
+						char[] buf = new char[1000];
+						int len = reader.read(buf, 0, 1000);
+						String oldError = new String(buf, 0, len);
+
+						throw new JGitInternalException(MessageFormat.format(
+								JGitText.get().gcLogExists, oldError, logFile));
+					}
+				}
+				return true;
+			}
+		} catch (NoSuchFileException e) {
+			// No existing log file, OK.
+		} catch (IOException | ParseException e) {
+			throw new JGitInternalException(e.getMessage(), e);
+		}
+		return false;
+	}
+
+	/**
+	 * Lock the GC log file for updates
+	 *
+	 * @param background
+	 *            If true, and if gc.log already exists, unlock and return false
+	 * @return {@code true} if we hold the lock
+	 */
+	boolean lock(boolean background) {
+		try {
+			if (!lock.lock()) {
+				return false;
+			}
+		} catch (IOException e) {
+			throw new JGitInternalException(e.getMessage(), e);
+		}
+		if (autoGcBlockedByOldLockFile(background)) {
+			lock.unlock();
+			return false;
+		}
+		return true;
+	}
+
+	/**
+	 * Unlock (roll back) the GC log lock
+	 */
+	void unlock() {
+		lock.unlock();
+	}
+
+	/**
+	 * Commit changes to the gc log, if there have been any writes. Otherwise,
+	 * just unlock and delete the existing file (if any)
+	 *
+	 * @return true if committing (or unlocking/deleting) succeeds.
+	 */
+	boolean commit() {
+		if (nonEmpty) {
+			return lock.commit();
+		} else {
+			logFile.delete();
+			lock.unlock();
+			return true;
+		}
+	}
+
+	/**
+	 * Write to the pending gc log. Content will be committed upon a call to
+	 * commit()
+	 *
+	 * @param content
+	 *            The content to write
+	 * @throws IOException
+	 */
+	void write(String content) throws IOException {
+		if (content.length() > 0) {
+			nonEmpty = true;
+		}
+		lock.write(content.getBytes(UTF_8));
+	}
+}

org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java

@@ -290,6 +290,20 @@ public class ConfigConstants {
 	 */
 	public static final String CONFIG_KEY_PRUNEPACKEXPIRE = "prunepackexpire";
 
+	/**
+	 * The "logexpiry" key
+	 *
+	 * @since 4.7
+	 */
+	public static final String CONFIG_KEY_LOGEXPIRY = "logExpiry";
+
+	/**
+	 * The "autodetach" key
+	 *
+	 * @since 4.7
+	 */
+	public static final String CONFIG_KEY_AUTODETACH = "autoDetach";
+
 	/**
 	 * The "aggressiveDepth" key
 	 * @since 3.6