Browse Source

FetchCommand: Add basic support for recursing into submodules

Extend FetchCommand to expose a new method, setRecurseSubmodules(mode),
which allows to set the mode to ON, OFF or ON_DEMAND.

After fetching a repository, its submodules are recursively fetched:

- When the mode is YES, submodules are always fetched.

- When the mode is NO, submodules are not fetched.

- When the mode is ON_DEMAND, submodules are only fetched when the
  parent repository receives an update of the submodule and the new
  revision is not already in the submodule.

The mode is determined in the following order of precedence:

- Value specified in the API call using setRecurseSubmodules.

- Value specified in the repository's config under the key
  submodule.name.fetchRecurseSubmodules

- Defaults to ON_DEMAND if neither of the previous is set.

Extend FetchResult to recursively include results for submodules, as
a map of the submodule path to an instance of FetchResult.

Test setup is based on testCloneRepositoryWithNestedSubmodules.

Change-Id: Ibc841683763307cb76e78e142e0da5b11b1add2a
Signed-off-by: David Pursehouse <david.pursehouse@gmail.com>
stable-4.7
David Pursehouse 8 years ago
parent
commit
503d59044f
  1. 273
      org.eclipse.jgit.test/tst/org/eclipse/jgit/api/FetchCommandRecurseSubmodulesTest.java
  2. 98
      org.eclipse.jgit/src/org/eclipse/jgit/api/FetchCommand.java
  3. 6
      org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java
  4. 86
      org.eclipse.jgit/src/org/eclipse/jgit/lib/SubmoduleConfig.java
  5. 30
      org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchResult.java

273
org.eclipse.jgit.test/tst/org/eclipse/jgit/api/FetchCommandRecurseSubmodulesTest.java

@ -0,0 +1,273 @@
/*
* Copyright (C) 2017 David Pursehouse <david.pursehouse@gmail.com>
* and other copyright owners as documented in the project's IP log.
*
* This program and the accompanying materials are made available
* under the terms of the Eclipse Distribution License v1.0 which
* accompanies this distribution, is reproduced below, and is
* available at http://www.eclipse.org/org/documents/edl-v10.php
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
*
* - Neither the name of the Eclipse Foundation, Inc. nor the
* names of its contributors may be used to endorse or promote
* products derived from this software without specific prior
* written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.eclipse.jgit.api;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import java.io.File;
import org.eclipse.jgit.api.ResetCommand.ResetType;
import org.eclipse.jgit.junit.JGitTestUtil;
import org.eclipse.jgit.junit.RepositoryTestCase;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.lib.SubmoduleConfig.FetchRecurseSubmodulesMode;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.submodule.SubmoduleStatus;
import org.eclipse.jgit.submodule.SubmoduleStatusType;
import org.eclipse.jgit.submodule.SubmoduleWalk;
import org.eclipse.jgit.transport.FetchResult;
import org.eclipse.jgit.transport.RefSpec;
import org.junit.Before;
import org.junit.Test;
public class FetchCommandRecurseSubmodulesTest extends RepositoryTestCase {
private Git git;
private Git git2;
private Git sub1Git;
private Git sub2Git;
private RevCommit commit1;
private RevCommit commit2;
private ObjectId submodule1Head;
private ObjectId submodule2Head;
private final RefSpec REFSPEC = new RefSpec("refs/heads/master");
private final String REMOTE = "origin";
@Before
public void setUpSubmodules()
throws Exception {
git = new Git(db);
// Create submodule 1
File submodule1 = createTempDirectory(
"testCloneRepositoryWithNestedSubmodules1");
sub1Git = Git.init().setDirectory(submodule1).call();
assertNotNull(sub1Git);
Repository sub1 = sub1Git.getRepository();
assertNotNull(sub1);
addRepoToClose(sub1);
String file = "file.txt";
String path = "sub";
write(new File(sub1.getWorkTree(), file), "content");
sub1Git.add().addFilepattern(file).call();
RevCommit commit = sub1Git.commit().setMessage("create file").call();
assertNotNull(commit);
// Create submodule 2
File submodule2 = createTempDirectory(
"testCloneRepositoryWithNestedSubmodules2");
sub2Git = Git.init().setDirectory(submodule2).call();
assertNotNull(sub2Git);
Repository sub2 = sub2Git.getRepository();
assertNotNull(sub2);
addRepoToClose(sub2);
write(new File(sub2.getWorkTree(), file), "content");
sub2Git.add().addFilepattern(file).call();
RevCommit sub2Head = sub2Git.commit().setMessage("create file").call();
assertNotNull(sub2Head);
// Add submodule 2 to submodule 1
Repository r2 = sub1Git.submoduleAdd().setPath(path)
.setURI(sub2.getDirectory().toURI().toString()).call();
assertNotNull(r2);
addRepoToClose(r2);
RevCommit sub1Head = sub1Git.commit().setAll(true)
.setMessage("Adding submodule").call();
assertNotNull(sub1Head);
// Add submodule 1 to default repository
Repository r1 = git.submoduleAdd().setPath(path)
.setURI(sub1.getDirectory().toURI().toString()).call();
assertNotNull(r1);
addRepoToClose(r1);
assertNotNull(git.commit().setAll(true).setMessage("Adding submodule")
.call());
// Clone default repository and include submodules
File directory = createTempDirectory(
"testCloneRepositoryWithNestedSubmodules");
CloneCommand clone = Git.cloneRepository();
clone.setDirectory(directory);
clone.setCloneSubmodules(true);
clone.setURI(git.getRepository().getDirectory().toURI().toString());
git2 = clone.call();
addRepoToClose(git2.getRepository());
assertNotNull(git2);
// Record current FETCH_HEAD of submodules
try (SubmoduleWalk walk = SubmoduleWalk
.forIndex(git2.getRepository())) {
assertTrue(walk.next());
Repository r = walk.getRepository();
submodule1Head = r.resolve(Constants.FETCH_HEAD);
try (SubmoduleWalk walk2 = SubmoduleWalk.forIndex(r)) {
assertTrue(walk2.next());
submodule2Head = walk2.getRepository()
.resolve(Constants.FETCH_HEAD);
}
}
// Commit in submodule 1
JGitTestUtil.writeTrashFile(r1, "f1.txt", "test");
sub1Git.add().addFilepattern("f1.txt").call();
commit1 = sub1Git.commit().setMessage("new commit").call();
// Commit in submodule 2
JGitTestUtil.writeTrashFile(r2, "f2.txt", "test");
sub2Git.add().addFilepattern("f2.txt").call();
commit2 = sub2Git.commit().setMessage("new commit").call();
}
@Test
public void shouldNotFetchSubmodulesWhenNo() throws Exception {
FetchResult result = fetch(FetchRecurseSubmodulesMode.NO);
assertTrue(result.submoduleResults().isEmpty());
assertSubmoduleFetchHeads(submodule1Head, submodule2Head);
}
@Test
public void shouldFetchSubmodulesWhenYes() throws Exception {
FetchResult result = fetch(FetchRecurseSubmodulesMode.YES);
assertTrue(result.submoduleResults().containsKey("sub"));
FetchResult subResult = result.submoduleResults().get("sub");
assertTrue(subResult.submoduleResults().containsKey("sub"));
assertSubmoduleFetchHeads(commit1, commit2);
}
@Test
public void shouldFetchSubmodulesWhenOnDemandAndRevisionChanged()
throws Exception {
// Fetch the submodule in the original git and reset it to
// the commit that was created
try (SubmoduleWalk w = SubmoduleWalk.forIndex(git.getRepository())) {
assertTrue(w.next());
try (Git g = new Git(w.getRepository())) {
g.fetch().setRemote(REMOTE).setRefSpecs(REFSPEC).call();
g.reset().setMode(ResetType.HARD).setRef(commit1.name()).call();
}
}
// Submodule index Id should be same as before, but head Id should be
// updated to the new commit, and status should be "checked out".
SubmoduleStatus subStatus = git.submoduleStatus().call().get("sub");
assertEquals(submodule1Head, subStatus.getIndexId());
assertEquals(commit1, subStatus.getHeadId());
assertEquals(SubmoduleStatusType.REV_CHECKED_OUT, subStatus.getType());
// Add and commit the submodule status
git.add().addFilepattern("sub").call();
RevCommit update = git.commit().setMessage("update sub").call();
// Both submodule index and head should now be at the new commit, and
// the status should be "initialized".
subStatus = git.submoduleStatus().call().get("sub");
assertEquals(commit1, subStatus.getIndexId());
assertEquals(commit1, subStatus.getHeadId());
assertEquals(SubmoduleStatusType.INITIALIZED, subStatus.getType());
FetchResult result = fetch(FetchRecurseSubmodulesMode.ON_DEMAND);
// The first submodule should have been updated
assertTrue(result.submoduleResults().containsKey("sub"));
FetchResult subResult = result.submoduleResults().get("sub");
// The second submodule should not get updated
assertTrue(subResult.submoduleResults().isEmpty());
assertSubmoduleFetchHeads(commit1, submodule2Head);
// After fetch the parent repo's fetch head should be the commit
// that updated the submodule.
assertEquals(update,
git2.getRepository().resolve(Constants.FETCH_HEAD));
}
@Test
public void shouldNotFetchSubmodulesWhenOnDemandAndRevisionNotChanged()
throws Exception {
FetchResult result = fetch(FetchRecurseSubmodulesMode.ON_DEMAND);
assertTrue(result.submoduleResults().isEmpty());
assertSubmoduleFetchHeads(submodule1Head, submodule2Head);
}
private FetchResult fetch(FetchRecurseSubmodulesMode mode)
throws Exception {
FetchResult result = git2.fetch().setRemote(REMOTE).setRefSpecs(REFSPEC)
.setRecurseSubmodules(mode).call();
assertNotNull(result);
return result;
}
private void assertSubmoduleFetchHeads(ObjectId expectedHead1,
ObjectId expectedHead2) throws Exception {
try (SubmoduleWalk walk = SubmoduleWalk
.forIndex(git2.getRepository())) {
assertTrue(walk.next());
Repository r = walk.getRepository();
ObjectId newHead1 = r.resolve(Constants.FETCH_HEAD);
ObjectId newHead2;
try (SubmoduleWalk walk2 = SubmoduleWalk.forIndex(r)) {
assertTrue(walk2.next());
newHead2 = walk2.getRepository().resolve(Constants.FETCH_HEAD);
}
assertEquals(expectedHead1, newHead1);
assertEquals(expectedHead2, newHead2);
}
}
}

98
org.eclipse.jgit/src/org/eclipse/jgit/api/FetchCommand.java

@ -42,14 +42,17 @@
*/
package org.eclipse.jgit.api;
import java.io.IOException;
import java.net.URISyntaxException;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.List;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.api.errors.InvalidConfigurationException;
import org.eclipse.jgit.api.errors.InvalidRemoteException;
import org.eclipse.jgit.api.errors.JGitInternalException;
import org.eclipse.jgit.errors.ConfigInvalidException;
import org.eclipse.jgit.errors.NoRemoteRepositoryException;
import org.eclipse.jgit.errors.NotSupportedException;
import org.eclipse.jgit.errors.TransportException;
@ -60,6 +63,9 @@ import org.eclipse.jgit.lib.NullProgressMonitor;
import org.eclipse.jgit.lib.ProgressMonitor;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.lib.StoredConfig;
import org.eclipse.jgit.lib.SubmoduleConfig.FetchRecurseSubmodulesMode;
import org.eclipse.jgit.revwalk.RevWalk;
import org.eclipse.jgit.submodule.SubmoduleWalk;
import org.eclipse.jgit.transport.FetchResult;
import org.eclipse.jgit.transport.RefSpec;
import org.eclipse.jgit.transport.TagOpt;
@ -74,7 +80,6 @@ import org.eclipse.jgit.transport.Transport;
* >Git documentation about Fetch</a>
*/
public class FetchCommand extends TransportCommand<FetchCommand, FetchResult> {
private String remote = Constants.DEFAULT_REMOTE_NAME;
private List<RefSpec> refSpecs;
@ -91,6 +96,8 @@ public class FetchCommand extends TransportCommand<FetchCommand, FetchResult> {
private TagOpt tagOption;
private FetchRecurseSubmodulesMode submoduleRecurseMode = null;
/**
* @param repo
*/
@ -99,6 +106,76 @@ public class FetchCommand extends TransportCommand<FetchCommand, FetchResult> {
refSpecs = new ArrayList<>(3);
}
private FetchRecurseSubmodulesMode getRecurseMode(Repository repository,
String path) {
// Use the caller-specified mode, if set
if (submoduleRecurseMode != null) {
return submoduleRecurseMode;
}
// Fall back to submodule config, if set
FetchRecurseSubmodulesMode mode = repository.getConfig().getEnum(
FetchRecurseSubmodulesMode.values(),
ConfigConstants.CONFIG_SUBMODULE_SECTION, path,
ConfigConstants.CONFIG_KEY_FETCH_RECURSE_SUBMODULES, null);
if (mode != null) {
return mode;
}
// Default to on-demand mode
return FetchRecurseSubmodulesMode.ON_DEMAND;
}
private boolean isRecurseSubmodules() {
return submoduleRecurseMode != null
&& submoduleRecurseMode != FetchRecurseSubmodulesMode.NO;
}
private void fetchSubmodules(FetchResult results)
throws org.eclipse.jgit.api.errors.TransportException,
GitAPIException, InvalidConfigurationException {
try (SubmoduleWalk walk = new SubmoduleWalk(repo);
RevWalk revWalk = new RevWalk(repo)) {
// Walk over submodules in the parent repository's FETCH_HEAD.
walk.setTree(revWalk.parseTree(repo.resolve(Constants.FETCH_HEAD)));
while (walk.next()) {
Repository submoduleRepo = walk.getRepository();
// Skip submodules that don't exist locally (have not been
// cloned), are not registered in the .gitmodules file, or
// not registered in the parent repository's config.
if (submoduleRepo == null || walk.getModulesPath() == null
|| walk.getConfigUrl() == null) {
continue;
}
FetchRecurseSubmodulesMode recurseMode = getRecurseMode(
submoduleRepo, walk.getPath());
// When the fetch mode is "yes" we always fetch. When the mode
// is "on demand", we only fetch if the submodule's revision was
// updated to an object that is not currently present in the
// submodule.
if ((recurseMode == FetchRecurseSubmodulesMode.ON_DEMAND
&& !submoduleRepo.hasObject(walk.getObjectId()))
|| recurseMode == FetchRecurseSubmodulesMode.YES) {
FetchCommand f = new FetchCommand(submoduleRepo)
.setProgressMonitor(monitor).setTagOpt(tagOption)
.setCheckFetchedObjects(checkFetchedObjects)
.setRemoveDeletedRefs(isRemoveDeletedRefs())
.setThin(thin).setRefSpecs(refSpecs)
.setDryRun(dryRun)
.setRecurseSubmodules(recurseMode);
results.addSubmodule(walk.getPath(), f.call());
}
}
} catch (IOException e) {
throw new JGitInternalException(e.getMessage(), e);
} catch (ConfigInvalidException e) {
throw new InvalidConfigurationException(e.getMessage(), e);
}
}
/**
* Executes the {@code fetch} command with all the options and parameters
* collected by the setter methods of this class. Each instance of this
@ -127,6 +204,11 @@ public class FetchCommand extends TransportCommand<FetchCommand, FetchResult> {
configure(transport);
FetchResult result = transport.fetch(monitor, refSpecs);
if (!repo.isBare() && (!result.getTrackingRefUpdates().isEmpty()
|| isRecurseSubmodules())) {
fetchSubmodules(result);
}
return result;
} catch (NoRemoteRepositoryException e) {
throw new InvalidRemoteException(MessageFormat.format(
@ -145,6 +227,20 @@ public class FetchCommand extends TransportCommand<FetchCommand, FetchResult> {
}
/**
* Set the mode to be used for recursing into submodules.
*
* @param recurse
* @return {@code this}
* @since 4.7
*/
public FetchCommand setRecurseSubmodules(
FetchRecurseSubmodulesMode recurse) {
checkCallable();
submoduleRecurseMode = recurse;
return this;
}
/**
* The remote (uri or name) used for the fetch operation. If no remote is
* set, the default value of <code>Constants.DEFAULT_REMOTE_NAME</code> will

6
org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java

@ -374,4 +374,10 @@ public class ConfigConstants {
* @since 4.6
*/
public static final String CONFIG_KEY_USEJGITBUILTIN = "useJGitBuiltin";
/**
* The "fetchRecurseSubmodules" key
* @since 4.7
*/
public static final String CONFIG_KEY_FETCH_RECURSE_SUBMODULES = "fetchRecurseSubmodules";
}

86
org.eclipse.jgit/src/org/eclipse/jgit/lib/SubmoduleConfig.java

@ -0,0 +1,86 @@
/*
* Copyright (C) 2017, David Pursehouse <david.pursehouse@gmail.com>
* and other copyright owners as documented in the project's IP log.
*
* This program and the accompanying materials are made available
* under the terms of the Eclipse Distribution License v1.0 which
* accompanies this distribution, is reproduced below, and is
* available at http://www.eclipse.org/org/documents/edl-v10.php
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
*
* - Neither the name of the Eclipse Foundation, Inc. nor the
* names of its contributors may be used to endorse or promote
* products derived from this software without specific prior
* written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.eclipse.jgit.lib;
/**
* Submodule section of a Git configuration file.
*
* @since 4.7
*/
public class SubmoduleConfig {
/**
* Config values for submodule.[name].fetchRecurseSubmodules.
*/
public enum FetchRecurseSubmodulesMode implements Config.ConfigEnum {
/** Unconditionally recurse into all populated submodules. */
YES("true"), //$NON-NLS-1$
/**
* Only recurse into a populated submodule when the superproject
* retrieves a commit that updates the submodule's reference to a commit
* that isn't already in the local submodule clone.
*/
ON_DEMAND("on-demand"), //$NON-NLS-1$
/** Completely disable recursion. */
NO("false"); //$NON-NLS-1$
private final String configValue;
private FetchRecurseSubmodulesMode(String configValue) {
this.configValue = configValue;
}
@Override
public String toConfigValue() {
return configValue;
}
@Override
public boolean matchConfigValue(String s) {
return configValue.equals(s);
}
}
}

30
org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchResult.java

@ -48,7 +48,10 @@ package org.eclipse.jgit.transport;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* Final status after a successful fetch from a remote repository.
@ -58,12 +61,39 @@ import java.util.List;
public class FetchResult extends OperationResult {
private final List<FetchHeadRecord> forMerge;
private final Map<String, FetchResult> submodules;
FetchResult() {
forMerge = new ArrayList<>();
submodules = new HashMap<>();
}
void add(final FetchHeadRecord r) {
if (!r.notForMerge)
forMerge.add(r);
}
/**
* Add fetch results for a submodule.
*
* @param path
* the submodule path
* @param result
* the fetch result
* @since 4.7
*/
public void addSubmodule(String path, FetchResult result) {
submodules.put(path, result);
}
/**
* Get fetch results for submodules.
*
* @return Fetch results for submodules as a map of submodule paths to fetch
* results.
* @since 4.7
*/
public Map<String, FetchResult> submoduleResults() {
return Collections.unmodifiableMap(submodules);
}
}

Loading…
Cancel
Save