Merge branch 'stable-4.6' into stable-4.7

* stable-4.6:
  Update Oxygen Orbit p2 repository to R20170516192513
  Fix exception handling for opening bitmap index files

Change-Id: I669fe48ce0034f9ea1977d38ee39099497422c1c
Signed-off-by: Matthias Sohn <matthias.sohn@sap.com>

org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.5.target

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
 <?pde?>
 <!-- generated with https://github.com/mbarbero/fr.obeo.releng.targetplatform -->
-<target name="jgit-4.5" sequenceNumber="1491140572">
+<target name="jgit-4.5" sequenceNumber="1502747250">
   <locations>
     <location includeMode="slicer" includeAllPlatforms="false" includeSource="true" includeConfigurePhase="true" type="InstallableUnit">
       <unit id="org.eclipse.jetty.client" version="9.3.17.v20170317"/>
@@ -98,7 +98,7 @@
       <unit id="org.slf4j.api.source" version="1.7.2.v20121108-1250"/>
       <unit id="org.slf4j.impl.log4j12" version="1.7.2.v20131105-2200"/>
       <unit id="org.slf4j.impl.log4j12.source" version="1.7.2.v20131105-2200"/>
-      <repository location="http://download.eclipse.org/tools/orbit/downloads/drops/S20170306214312/repository"/>
+      <repository location="http://download.eclipse.org/tools/orbit/R-builds/R20170516192513/repository"/>
     </location>
     <location includeMode="slicer" includeAllPlatforms="false" includeSource="true" includeConfigurePhase="true" type="InstallableUnit">
       <unit id="org.eclipse.osgi" version="0.0.0"/>

org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.5.tpd

@@ -2,7 +2,7 @@ target "jgit-4.5" with source configurePhase
 
 include "projects/jetty-9.3.17.tpd"
 include "orbit/R20160221192158-Mars.tpd"
-include "orbit/S20170306214312-Oxygen.tpd"
+include "orbit/R20170516192513-Oxygen.tpd"
 
 location "http://download.eclipse.org/releases/mars/" {
 	org.eclipse.osgi lazy

org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.6.target

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
 <?pde?>
 <!-- generated with https://github.com/mbarbero/fr.obeo.releng.targetplatform -->
-<target name="jgit-4.6" sequenceNumber="1491140561">
+<target name="jgit-4.6" sequenceNumber="1502747233">
   <locations>
     <location includeMode="slicer" includeAllPlatforms="false" includeSource="true" includeConfigurePhase="true" type="InstallableUnit">
       <unit id="org.eclipse.jetty.client" version="9.3.17.v20170317"/>
@@ -60,7 +60,7 @@
       <unit id="org.slf4j.api.source" version="1.7.2.v20121108-1250"/>
       <unit id="org.slf4j.impl.log4j12" version="1.7.2.v20131105-2200"/>
       <unit id="org.slf4j.impl.log4j12.source" version="1.7.2.v20131105-2200"/>
-      <repository location="http://download.eclipse.org/tools/orbit/downloads/drops/S20170306214312/repository"/>
+      <repository location="http://download.eclipse.org/tools/orbit/R-builds/R20170516192513/repository"/>
     </location>
     <location includeMode="slicer" includeAllPlatforms="false" includeSource="true" includeConfigurePhase="true" type="InstallableUnit">
       <unit id="org.eclipse.osgi" version="0.0.0"/>

org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.6.tpd

@@ -1,7 +1,7 @@
 target "jgit-4.6" with source configurePhase
 
 include "projects/jetty-9.3.17.tpd"
-include "orbit/S20170306214312-Oxygen.tpd"
+include "orbit/R20170516192513-Oxygen.tpd"
 
 location "http://download.eclipse.org/releases/neon/" {
 	org.eclipse.osgi lazy

org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.7.target

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
 <?pde?>
 <!-- generated with https://github.com/mbarbero/fr.obeo.releng.targetplatform -->
-<target name="jgit-4.7" sequenceNumber="1491140544">
+<target name="jgit-4.7" sequenceNumber="1502747215">
   <locations>
     <location includeMode="slicer" includeAllPlatforms="false" includeSource="true" includeConfigurePhase="true" type="InstallableUnit">
       <unit id="org.eclipse.jetty.client" version="9.3.17.v20170317"/>
@@ -60,7 +60,7 @@
       <unit id="org.slf4j.api.source" version="1.7.2.v20121108-1250"/>
       <unit id="org.slf4j.impl.log4j12" version="1.7.2.v20131105-2200"/>
       <unit id="org.slf4j.impl.log4j12.source" version="1.7.2.v20131105-2200"/>
-      <repository location="http://download.eclipse.org/tools/orbit/downloads/drops/S20170306214312/repository"/>
+      <repository location="http://download.eclipse.org/tools/orbit/R-builds/R20170516192513/repository"/>
     </location>
     <location includeMode="slicer" includeAllPlatforms="false" includeSource="true" includeConfigurePhase="true" type="InstallableUnit">
       <unit id="org.eclipse.osgi" version="0.0.0"/>

org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.7.tpd

@@ -1,7 +1,7 @@
 target "jgit-4.7" with source configurePhase
 
 include "projects/jetty-9.3.17.tpd"
-include "orbit/S20170306214312-Oxygen.tpd"
+include "orbit/R20170516192513-Oxygen.tpd"
 
 location "http://download.eclipse.org/releases/oxygen/" {
 	org.eclipse.osgi lazy

org.eclipse.jgit.packaging/org.eclipse.jgit.target/orbit/R20170516192513-Oxygen.tpd

@@ -1,7 +1,7 @@
-target "S20170306214312-Oxygen" with source configurePhase
+target "R20170516192513-Oxygen" with source configurePhase
 // see http://download.eclipse.org/tools/orbit/downloads/
 
-location "http://download.eclipse.org/tools/orbit/downloads/drops/S20170306214312/repository" {
+location "http://download.eclipse.org/tools/orbit/R-builds/R20170516192513/repository" {
 	org.apache.ant [1.9.6.v201510161327,1.9.6.v201510161327]
 	org.apache.ant.source [1.9.6.v201510161327,1.9.6.v201510161327]
 	org.apache.commons.compress [1.6.0.v201310281400,1.6.0.v201310281400]

org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcConcurrentTest.java

@@ -45,8 +45,12 @@ package org.eclipse.jgit.internal.storage.file;
 
 import static java.lang.Integer.valueOf;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
 
 import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.concurrent.BrokenBarrierException;
 import java.util.concurrent.Callable;
 import java.util.concurrent.CyclicBarrier;
@@ -56,8 +60,14 @@ import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 
 import org.eclipse.jgit.internal.JGitText;
+import org.eclipse.jgit.internal.storage.pack.PackWriter;
+import org.eclipse.jgit.junit.TestRepository;
 import org.eclipse.jgit.lib.EmptyProgressMonitor;
+import org.eclipse.jgit.lib.NullProgressMonitor;
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.Sets;
 import org.eclipse.jgit.revwalk.RevBlob;
+import org.eclipse.jgit.revwalk.RevCommit;
 import org.junit.Test;
 
 public class GcConcurrentTest extends GcTestCase {
@@ -118,4 +128,97 @@ public class GcConcurrentTest extends GcTestCase {
 			pool.awaitTermination(Long.MAX_VALUE, TimeUnit.SECONDS);
 		}
 	}
+
+	@Test
+	public void repackAndGetStats() throws Exception {
+		TestRepository<FileRepository>.BranchBuilder test = tr.branch("test");
+		test.commit().add("a", "a").create();
+		GC gc1 = new GC(tr.getRepository());
+		gc1.setPackExpireAgeMillis(0);
+		gc1.gc();
+		test.commit().add("b", "b").create();
+
+		// Create a new Repository instance and trigger a gc
+		// from that instance. Reusing the existing repo instance
+		// tr.getRepository() would not show the problem.
+		FileRepository r2 = new FileRepository(
+				tr.getRepository().getDirectory());
+		GC gc2 = new GC(r2);
+		gc2.setPackExpireAgeMillis(0);
+		gc2.gc();
+
+		new GC(tr.getRepository()).getStatistics();
+	}
+
+	@Test
+	public void repackAndUploadPack() throws Exception {
+		TestRepository<FileRepository>.BranchBuilder test = tr.branch("test");
+		// RevCommit a = test.commit().add("a", "a").create();
+		test.commit().add("a", "a").create();
+
+		GC gc1 = new GC(tr.getRepository());
+		gc1.setPackExpireAgeMillis(0);
+		gc1.gc();
+
+		RevCommit b = test.commit().add("b", "b").create();
+
+		FileRepository r2 = new FileRepository(
+				tr.getRepository().getDirectory());
+		GC gc2 = new GC(r2);
+		gc2.setPackExpireAgeMillis(0);
+		gc2.gc();
+
+		// Simulate parts of an UploadPack. This is the situation on
+		// server side (e.g. gerrit) when when clients are
+		// cloning/fetching while the server side repo's
+		// are gc'ed by an external process (e.g. scheduled
+		// native git gc)
+		try (PackWriter pw = new PackWriter(tr.getRepository())) {
+			pw.setUseBitmaps(true);
+			pw.preparePack(NullProgressMonitor.INSTANCE, Sets.of(b),
+					Collections.<ObjectId> emptySet());
+			new GC(tr.getRepository()).getStatistics();
+		}
+	}
+
+	PackFile getSinglePack(FileRepository r) {
+		Collection<PackFile> packs = r.getObjectDatabase().getPacks();
+		assertEquals(1, packs.size());
+		return packs.iterator().next();
+	}
+
+	@Test
+	public void repackAndCheckBitmapUsage() throws Exception {
+		// create a test repository with one commit and pack all objects. After
+		// packing create loose objects to trigger creation of a new packfile on
+		// the next gc
+		TestRepository<FileRepository>.BranchBuilder test = tr.branch("test");
+		test.commit().add("a", "a").create();
+		FileRepository repository = tr.getRepository();
+		GC gc1 = new GC(repository);
+		gc1.setPackExpireAgeMillis(0);
+		gc1.gc();
+		String oldPackName = getSinglePack(repository).getPackName();
+		RevCommit b = test.commit().add("b", "b").create();
+
+		// start the garbage collection on a new repository instance,
+		FileRepository repository2 = new FileRepository(repository.getDirectory());
+		GC gc2 = new GC(repository2);
+		gc2.setPackExpireAgeMillis(0);
+		gc2.gc();
+		String newPackName = getSinglePack(repository2).getPackName();
+		// make sure gc() has caused creation of a new packfile
+		assertNotEquals(oldPackName, newPackName);
+
+		// Even when asking again for the set of packfiles outdated data
+		// will be returned. As long as the repository can work on cached data
+		// it will do so and not detect that a new packfile exists.
+		assertNotEquals(getSinglePack(repository).getPackName(), newPackName);
+
+		// Only when accessing object content it is required to rescan the pack
+		// directory and the new packfile will be detected.
+		repository.getObjectDatabase().open(b).getSize();
+		assertEquals(getSinglePack(repository).getPackName(), newPackName);
+		assertNotNull(getSinglePack(repository).getBitmapIndex());
+	}
 }

org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java

@@ -1108,8 +1108,17 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> {
 		if (invalid || invalidBitmap)
 			return null;
 		if (bitmapIdx == null && hasExt(BITMAP_INDEX)) {
-			final PackBitmapIndex idx = PackBitmapIndex.open(
+			final PackBitmapIndex idx;
-					extFile(BITMAP_INDEX), idx(), getReverseIdx());
+			try {
+				idx = PackBitmapIndex.open(extFile(BITMAP_INDEX), idx(),
+						getReverseIdx());
+			} catch (FileNotFoundException e) {
+				// Once upon a time this bitmap file existed. Now it
+				// has been removed. Most likely an external gc  has
+				// removed this packfile and the bitmap
+				 invalidBitmap = true;
+				 return null;
+			}
 
 			// At this point, idx() will have set packChecksum.
 			if (Arrays.equals(packChecksum, idx.packChecksum))