Add symlink support to JGit

The change includes comparing symbolic links between disk and index,
adding symbolic links to the index, creating/modifying links on
checkout. The behavior is controlled by the core.symlinks setting, just
as C Git does. When a new repository is created core.symlinks will be
set depending on the capabilities of the operating system and Java
runtime.

If core.symlinks is set to true, the assumption is that symlinks are
supported, which may result in runtime errors if this turns out not to
be the case.

Measuring the cost of jgit status on a repository with ~70000 files,
of which ~30000 are tracked reveals a penalty of about 10% for using
the Java7 (really NIO2) support module.

Bug: 354367
Change-Id: I12f0fdd9d26212324a586896ef7eb1f6ff89c39c
Signed-off-by: Matthias Sohn <matthias.sohn@sap.com>

org.eclipse.jgit.pgm.test/tst/org/eclipse/jgit/pgm/ConfigTest.java

@@ -46,6 +46,8 @@ import static org.junit.Assert.assertArrayEquals;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import org.eclipse.jgit.api.Git;
 import org.eclipse.jgit.lib.CLIRepositoryTestCase;
@@ -75,8 +77,28 @@ public class ConfigTest extends CLIRepositoryTestCase {
 		if (isMac)
 			expect.add("core.precomposeunicode=true");
 		expect.add("core.repositoryformatversion=0");
+		if (SystemReader.getInstance().isWindows() && osVersion() < 6
+				|| javaVersion() < 1.7) {
+			expect.add("core.symlinks=false");
+		}
 		expect.add(""); // ends with LF (last line empty)
 		assertArrayEquals("expected default configuration", expect.toArray(),
 				output);
 	}
+
+	private static float javaVersion() {
+		String versionString = System.getProperty("java.version");
+		Matcher matcher = Pattern.compile("(\\d+\\.\\d+).*").matcher(
+				versionString);
+		matcher.matches();
+		return Float.parseFloat(matcher.group(1));
+	}
+
+	private static float osVersion() {
+		String versionString = System.getProperty("os.version");
+		Matcher matcher = Pattern.compile("(\\d+\\.\\d+).*").matcher(
+				versionString);
+		matcher.matches();
+		return Float.parseFloat(matcher.group(1));
+	}
 }

org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Blame.java

@@ -181,7 +181,7 @@ class Blame extends TextBuiltin {
 						generator.push(null, dc.getEntry(entry).getObjectId());
 
 					File inTree = new File(db.getWorkTree(), file);
-					if (inTree.isFile())
+					if (db.getFS().isFile(inTree))
 						generator.push(null, new RawText(inTree));
 				}
 			}

org.eclipse.jgit.test/tst/org/eclipse/jgit/treewalk/FileTreeIteratorTest.java

@@ -63,6 +63,7 @@ import org.eclipse.jgit.errors.CorruptObjectException;
 import org.eclipse.jgit.errors.IncorrectObjectTypeException;
 import org.eclipse.jgit.errors.MissingObjectException;
 import org.eclipse.jgit.junit.RepositoryTestCase;
+import org.eclipse.jgit.lib.ConfigConstants;
 import org.eclipse.jgit.lib.Constants;
 import org.eclipse.jgit.lib.FileMode;
 import org.eclipse.jgit.lib.ObjectId;
@@ -242,9 +243,11 @@ public class FileTreeIteratorTest extends RepositoryTestCase {
 	}
 
 	@Test
-	public void testIsModifiedSymlink() throws Exception {
+	public void testIsModifiedSymlinkAsFile() throws Exception {
 		File f = writeTrashFile("symlink", "content");
 		Git git = new Git(db);
+		db.getConfig().setString(ConfigConstants.CONFIG_CORE_SECTION, null,
+				ConfigConstants.CONFIG_KEY_SYMLINKS, "false");
 		git.add().addFilepattern("symlink").call();
 		git.commit().setMessage("commit").call();
 

org.eclipse.jgit/src/org/eclipse/jgit/api/BlameCommand.java

@@ -215,7 +215,7 @@ public class BlameCommand extends GitCommand<BlameResult> {
 						gen.push(null, dc.getEntry(entry).getObjectId());
 
 					File inTree = new File(repo.getWorkTree(), path);
-					if (inTree.isFile())
+					if (repo.getFS().isFile(inTree))
 						gen.push(null, new RawText(inTree));
 				}
 			}

org.eclipse.jgit/src/org/eclipse/jgit/api/CleanCommand.java

@@ -53,6 +53,7 @@ import org.eclipse.jgit.api.errors.GitAPIException;
 import org.eclipse.jgit.api.errors.JGitInternalException;
 import org.eclipse.jgit.errors.NoWorkTreeException;
 import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.util.FS;
 import org.eclipse.jgit.util.FileUtils;
 
 /**
@@ -100,13 +101,13 @@ public class CleanCommand extends GitCommand<Set<String>> {
 			Set<String> untrackedAndIgnoredDirs = new TreeSet<String>(
 					status.getUntrackedFolders());
 
+			FS fs = getRepository().getFS();
 			for (String p : status.getIgnoredNotInIndex()) {
 				File f = new File(repo.getWorkTree(), p);
-				if (f.isFile()) {
+				if (fs.isFile(f) || fs.isSymLink(f))
 					untrackedAndIgnoredFiles.add(p);
-				} else if (f.isDirectory()) {
+				else if (fs.isDirectory(f))
 					untrackedAndIgnoredDirs.add(p);
-				}
 			}
 
 			Set<String> filtered = filterFolders(untrackedAndIgnoredFiles,

org.eclipse.jgit/src/org/eclipse/jgit/api/RebaseCommand.java

@@ -731,7 +731,7 @@ public class RebaseCommand extends GitCommand<RebaseResult> {
 				List<String> fileList = dco.getToBeDeleted();
 				for (String filePath : fileList) {
 					File fileToDelete = new File(repo.getWorkTree(), filePath);
-					if (fileToDelete.exists())
+					if (repo.getFS().exists(fileToDelete))
 						FileUtils.delete(fileToDelete, FileUtils.RECURSIVE
 								| FileUtils.RETRY);
 				}

org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java

@@ -60,6 +60,7 @@ import org.eclipse.jgit.errors.MissingObjectException;
 import org.eclipse.jgit.internal.JGitText;
 import org.eclipse.jgit.lib.Constants;
 import org.eclipse.jgit.lib.CoreConfig.AutoCRLF;
+import org.eclipse.jgit.lib.CoreConfig.SymLinks;
 import org.eclipse.jgit.lib.FileMode;
 import org.eclipse.jgit.lib.ObjectId;
 import org.eclipse.jgit.lib.ObjectLoader;
@@ -399,7 +400,6 @@ public class DirCacheCheckout {
 			MissingObjectException, IncorrectObjectTypeException,
 			CheckoutConflictException, IndexWriteException {
 		toBeDeleted.clear();
-
 		ObjectReader objectReader = repo.getObjectDatabase().newReader();
 		try {
 			if (headCommitTree != null)
@@ -425,13 +425,13 @@ public class DirCacheCheckout {
 			for (int i = removed.size() - 1; i >= 0; i--) {
 				String r = removed.get(i);
 				file = new File(repo.getWorkTree(), r);
-				if (!file.delete() && file.exists()) {
+				if (!file.delete() && repo.getFS().exists(file)) {
 					// The list of stuff to delete comes from the index
 					// which will only contain a directory if it is
 					// a submodule, in which case we shall not attempt
 					// to delete it. A submodule is not empty, so it
 					// is safe to check this after a failed delete.
-					if (!file.isDirectory())
+					if (!repo.getFS().isDirectory(file))
 						toBeDeleted.add(r);
 				} else {
 					if (last != null && !isSamePrefix(r, last))
@@ -583,9 +583,8 @@ public class DirCacheCheckout {
 		// represents the state for the merge iterator, the second last the
 		// state for the index iterator and the third last represents the state
 		// for the head iterator. The hexadecimal constant "F" stands for
-		// "file",
-		// an "D" stands for "directory" (tree), and a "0" stands for
-		// non-existing
+		// "file", a "D" stands for "directory" (tree), and a "0" stands for
+		// non-existing. Symbolic links and git links are treated as File here.
 		//
 		// Examples:
 		// ffMask == 0xFFD -> Head=File, Index=File, Merge=Tree
@@ -1117,35 +1116,45 @@ public class DirCacheCheckout {
 		ObjectLoader ol = or.open(entry.getObjectId());
 		File parentDir = f.getParentFile();
 		parentDir.mkdirs();
-		File tmpFile = File.createTempFile("._" + f.getName(), null, parentDir); //$NON-NLS-1$
-		WorkingTreeOptions opt = repo.getConfig().get(WorkingTreeOptions.KEY);
-		FileOutputStream rawChannel = new FileOutputStream(tmpFile);
-		OutputStream channel;
-		if (opt.getAutoCRLF() == AutoCRLF.TRUE)
-			channel = new AutoCRLFOutputStream(rawChannel);
-		else
-			channel = rawChannel;
-		try {
-			ol.copyTo(channel);
-		} finally {
-			channel.close();
-		}
 		FS fs = repo.getFS();
-		if (opt.isFileMode() && fs.supportsExecute()) {
-			if (FileMode.EXECUTABLE_FILE.equals(entry.getRawMode())) {
-				if (!fs.canExecute(tmpFile))
-					fs.setExecute(tmpFile, true);
-			} else {
-				if (fs.canExecute(tmpFile))
-					fs.setExecute(tmpFile, false);
+		WorkingTreeOptions opt = repo.getConfig().get(WorkingTreeOptions.KEY);
+		if (entry.getFileMode() == FileMode.SYMLINK
+				&& opt.getSymLinks() == SymLinks.TRUE) {
+			byte[] bytes = ol.getBytes();
+			String target = RawParseUtils.decode(bytes);
+			fs.createSymLink(f, target);
+			entry.setLength(bytes.length);
+			entry.setLastModified(fs.lastModified(f));
+		} else {
+			File tmpFile = File.createTempFile(
+					"._" + f.getName(), null, parentDir); //$NON-NLS-1$
+			FileOutputStream rawChannel = new FileOutputStream(tmpFile);
+			OutputStream channel;
+			if (opt.getAutoCRLF() == AutoCRLF.TRUE)
+				channel = new AutoCRLFOutputStream(rawChannel);
+			else
+				channel = rawChannel;
+			try {
+				ol.copyTo(channel);
+			} finally {
+				channel.close();
+			}
+			if (opt.isFileMode() && fs.supportsExecute()) {
+				if (FileMode.EXECUTABLE_FILE.equals(entry.getRawMode())) {
+					if (!fs.canExecute(tmpFile))
+						fs.setExecute(tmpFile, true);
+				} else {
+					if (fs.canExecute(tmpFile))
+						fs.setExecute(tmpFile, false);
+				}
+			}
+			try {
+				FileUtils.rename(tmpFile, f);
+			} catch (IOException e) {
+				throw new IOException(MessageFormat.format(
+						JGitText.get().couldNotWriteFile, tmpFile.getPath(),
+						f.getPath()));
 			}
-		}
-		try {
-			FileUtils.rename(tmpFile, f);
-		} catch (IOException e) {
-			throw new IOException(MessageFormat.format(
-					JGitText.get().couldNotWriteFile, tmpFile.getPath(),
-					f.getPath()));
 		}
 		entry.setLastModified(f.lastModified());
 		if (opt.getAutoCRLF() != AutoCRLF.FALSE)

org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java

@@ -64,6 +64,7 @@ import org.eclipse.jgit.internal.storage.file.ObjectDirectory.AlternateRepositor
 import org.eclipse.jgit.lib.BaseRepositoryBuilder;
 import org.eclipse.jgit.lib.ConfigConstants;
 import org.eclipse.jgit.lib.Constants;
+import org.eclipse.jgit.lib.CoreConfig.SymLinks;
 import org.eclipse.jgit.lib.ObjectId;
 import org.eclipse.jgit.lib.Ref;
 import org.eclipse.jgit.lib.RefDatabase;
@@ -293,6 +294,21 @@ public class FileRepository extends Repository {
 			fileMode = false;
 		}
 
+		SymLinks symLinks = SymLinks.FALSE;
+		if (getFS().supportsSymlinks()) {
+			File tmp = new File(getDirectory(), "tmplink"); //$NON-NLS-1$
+			try {
+				getFS().createSymLink(tmp, "target"); //$NON-NLS-1$
+				symLinks = null;
+				FileUtils.delete(tmp);
+			} catch (IOException e) {
+				// Normally a java.nio.file.FileSystemException
+			}
+		}
+		if (symLinks != null)
+			cfg.setString(ConfigConstants.CONFIG_CORE_SECTION, null,
+					ConfigConstants.CONFIG_KEY_SYMLINKS, symLinks.name()
+							.toLowerCase());
 		cfg.setInt(ConfigConstants.CONFIG_CORE_SECTION, null,
 				ConfigConstants.CONFIG_KEY_REPO_FORMAT_VERSION, 0);
 		cfg.setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null,

org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java

@@ -123,6 +123,13 @@ public class ConfigConstants {
 	/** The "deltaBaseCacheLimit" key */
 	public static final String CONFIG_KEY_DELTA_BASE_CACHE_LIMIT = "deltaBaseCacheLimit";
 
+	/**
+	 * The "symlinks" key
+	 *
+	 * @since 3.3
+	 */
+	public static final String CONFIG_KEY_SYMLINKS = "symlinks";
+
 	/** The "streamFileThreshold" key */
 	public static final String CONFIG_KEY_STREAM_FILE_TRESHOLD = "streamFileThreshold";
 

org.eclipse.jgit/src/org/eclipse/jgit/lib/CoreConfig.java

@@ -101,6 +101,18 @@ public class CoreConfig {
 
 	private final String excludesfile;
 
+	/**
+	 * Options for symlink handling
+	 *
+	 * @since 3.3
+	 */
+	public static enum SymLinks {
+		/** Checkout symbolic links as plain files */
+		FALSE,
+		/** Checkout symbolic links as links */
+		TRUE
+	}
+
 	private CoreConfig(final Config rc) {
 		compression = rc.getInt(ConfigConstants.CONFIG_CORE_SECTION,
 				ConfigConstants.CONFIG_KEY_COMPRESSION, DEFAULT_COMPRESSION);

org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java

@@ -86,6 +86,7 @@ import org.eclipse.jgit.treewalk.AbstractTreeIterator;
 import org.eclipse.jgit.treewalk.CanonicalTreeParser;
 import org.eclipse.jgit.treewalk.NameConflictTreeWalk;
 import org.eclipse.jgit.treewalk.WorkingTreeIterator;
+import org.eclipse.jgit.util.FS;
 import org.eclipse.jgit.util.FileUtils;
 
 /**
@@ -255,11 +256,11 @@ public class ResolveMerger extends ThreeWayMerger {
 	}
 
 	private void createDir(File f) throws IOException {
-		if (!f.isDirectory() && !f.mkdirs()) {
+		if (!db.getFS().isDirectory(f) && !f.mkdirs()) {
 			File p = f;
-			while (p != null && !p.exists())
+			while (p != null && !db.getFS().exists(p))
 				p = p.getParentFile();
-			if (p == null || p.isDirectory())
+			if (p == null || db.getFS().isDirectory(p))
 				throw new IOException(JGitText.get().cannotCreateDirectory);
 			FileUtils.delete(p);
 			if (!f.mkdirs())
@@ -719,9 +720,10 @@ public class ResolveMerger extends ThreeWayMerger {
 				// support write operations
 				throw new UnsupportedOperationException();
 
+			FS fs = db.getFS();
 			of = new File(workTree, tw.getPathString());
 			File parentFolder = of.getParentFile();
-			if (!parentFolder.exists())
+			if (!fs.exists(parentFolder))
 				parentFolder.mkdirs();
 			fos = new FileOutputStream(of);
 			try {

org.eclipse.jgit/src/org/eclipse/jgit/treewalk/FileTreeIterator.java

@@ -46,6 +46,7 @@
 
 package org.eclipse.jgit.treewalk;
 
+import java.io.ByteArrayInputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
@@ -156,6 +157,8 @@ public class FileTreeIterator extends WorkingTreeIterator {
 
 		private long lastModified;
 
+		private FS fs;
+
 		/**
 		 * Create a new file entry.
 		 *
@@ -166,16 +169,26 @@ public class FileTreeIterator extends WorkingTreeIterator {
 		 */
 		public FileEntry(final File f, FS fs) {
 			file = f;
+			this.fs = fs;
 
-			if (f.isDirectory()) {
-				if (fs.exists(new File(f, Constants.DOT_GIT)))
-					mode = FileMode.GITLINK;
+			@SuppressWarnings("hiding")
+			FileMode mode = null;
+			try {
+				if (fs.isSymLink(f)) {
+					mode = FileMode.SYMLINK;
+				} else if (fs.isDirectory(f)) {
+					if (fs.exists(new File(f, Constants.DOT_GIT)))
+						mode = FileMode.GITLINK;
+					else
+						mode = FileMode.TREE;
+				} else if (fs.canExecute(file))
+					mode = FileMode.EXECUTABLE_FILE;
 				else
-					mode = FileMode.TREE;
-			} else if (fs.canExecute(file))
-				mode = FileMode.EXECUTABLE_FILE;
-			else
-				mode = FileMode.REGULAR_FILE;
+					mode = FileMode.REGULAR_FILE;
+			} catch (IOException e) {
+				mode = FileMode.MISSING;
+			}
+			this.mode = mode;
 		}
 
 		@Override
@@ -190,21 +203,35 @@ public class FileTreeIterator extends WorkingTreeIterator {
 
 		@Override
 		public long getLength() {
-			if (length < 0)
-				length = file.length();
+			if (length < 0) {
+				try {
+					length = fs.length(file);
+				} catch (IOException e) {
+					length = 0;
+				}
+			}
 			return length;
 		}
 
 		@Override
 		public long getLastModified() {
-			if (lastModified == 0)
-				lastModified = file.lastModified();
+			if (lastModified == 0) {
+				try {
+					lastModified = fs.lastModified(file);
+				} catch (IOException e) {
+					lastModified = 0;
+				}
+			}
 			return lastModified;
 		}
 
 		@Override
 		public InputStream openInputStream() throws IOException {
-			return new FileInputStream(file);
+			if (fs.isSymLink(file))
+				return new ByteArrayInputStream(fs.readSymLink(file).getBytes(
+						Constants.CHARACTER_ENCODING));
+			else
+				return new FileInputStream(file);
 		}
 
 		/**

org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java

@@ -74,6 +74,7 @@ import org.eclipse.jgit.internal.JGitText;
 import org.eclipse.jgit.lib.Constants;
 import org.eclipse.jgit.lib.CoreConfig;
 import org.eclipse.jgit.lib.CoreConfig.CheckStat;
+import org.eclipse.jgit.lib.CoreConfig.SymLinks;
 import org.eclipse.jgit.lib.FileMode;
 import org.eclipse.jgit.lib.ObjectId;
 import org.eclipse.jgit.lib.ObjectLoader;
@@ -202,6 +203,15 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator {
 		ignoreNode = new RootIgnoreNode(entry, repo);
 	}
 
+	/**
+	 * @return the repository this iterator works with
+	 *
+	 * @since 3.3
+	 */
+	public Repository getRepository() {
+		return repository;
+	}
+
 	/**
 	 * Define the matching {@link DirCacheIterator}, to optimize ObjectIds.
 	 *
@@ -252,14 +262,10 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator {
 			}
 		}
 		switch (mode & FileMode.TYPE_MASK) {
+		case FileMode.TYPE_SYMLINK:
 		case FileMode.TYPE_FILE:
 			contentIdFromPtr = ptr;
 			return contentId = idBufferBlob(entries[ptr]);
-		case FileMode.TYPE_SYMLINK:
-			// Java does not support symbolic links, so we should not
-			// have reached this particular part of the walk code.
-			//
-			return zeroid;
 		case FileMode.TYPE_GITLINK:
 			contentIdFromPtr = ptr;
 			return contentId = idSubmodule(entries[ptr]);
@@ -723,8 +729,9 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator {
 			return false;
 
 		// Do not rely on filemode differences in case of symbolic links
-		if (FileMode.SYMLINK.equals(rawMode))
-			return false;
+		if (getOptions().getSymLinks() == SymLinks.FALSE)
+			if (FileMode.SYMLINK.equals(rawMode))
+				return false;
 
 		// Ignore the executable file bits if WorkingTreeOptions tell me to
 		// do so. Ignoring is done by setting the bits representing a

org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeOptions.java

@@ -48,6 +48,7 @@ import org.eclipse.jgit.lib.ConfigConstants;
 import org.eclipse.jgit.lib.Config.SectionParser;
 import org.eclipse.jgit.lib.CoreConfig.AutoCRLF;
 import org.eclipse.jgit.lib.CoreConfig.CheckStat;
+import org.eclipse.jgit.lib.CoreConfig.SymLinks;
 
 /** Options used by the {@link WorkingTreeIterator}. */
 public class WorkingTreeOptions {
@@ -64,6 +65,8 @@ public class WorkingTreeOptions {
 
 	private final CheckStat checkStat;
 
+	private final SymLinks symlinks;
+
 	private WorkingTreeOptions(final Config rc) {
 		fileMode = rc.getBoolean(ConfigConstants.CONFIG_CORE_SECTION,
 				ConfigConstants.CONFIG_KEY_FILEMODE, true);
@@ -71,6 +74,8 @@ public class WorkingTreeOptions {
 				ConfigConstants.CONFIG_KEY_AUTOCRLF, AutoCRLF.FALSE);
 		checkStat = rc.getEnum(ConfigConstants.CONFIG_CORE_SECTION, null,
 				ConfigConstants.CONFIG_KEY_CHECKSTAT, CheckStat.DEFAULT);
+		symlinks = rc.getEnum(ConfigConstants.CONFIG_CORE_SECTION, null,
+				ConfigConstants.CONFIG_KEY_SYMLINKS, SymLinks.TRUE);
 	}
 
 	/** @return true if the execute bit on working files should be trusted. */
@@ -90,4 +95,12 @@ public class WorkingTreeOptions {
 	public CheckStat getCheckStat() {
 		return checkStat;
 	}
+
+	/**
+	 * @return how we handle symbolic links
+	 * @since 3.3
+	 */
+	public SymLinks getSymLinks() {
+		return symlinks;
+	}
 }