Browse Source

Merge changes from topic 'ketch'

* changes:
  daemon: Add --ketch=LEADER flag
  Ketch: Intercept push and route it through Ketch
  Ketch: Simple map of running leader instances
  Ketch: Basic replication system
stable-4.3
Shawn Pearce 9 years ago committed by Gerrit Code Review @ Eclipse.org
parent
commit
8d0551dd64
  1. 1
      org.eclipse.jgit.pgm/META-INF/MANIFEST.MF
  2. 47
      org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Daemon.java
  3. 1
      org.eclipse.jgit/META-INF/MANIFEST.MF
  4. 13
      org.eclipse.jgit/resources/org/eclipse/jgit/internal/ketch/KetchText.properties
  5. 141
      org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ElectionRound.java
  6. 82
      org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchConstants.java
  7. 624
      org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchLeader.java
  8. 126
      org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchLeaderCache.java
  9. 155
      org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchPreReceive.java
  10. 755
      org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchReplica.java
  11. 301
      org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchSystem.java
  12. 70
      org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchText.java
  13. 183
      org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LagCheck.java
  14. 164
      org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LeaderSnapshot.java
  15. 229
      org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LocalReplica.java
  16. 119
      org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LogIndex.java
  17. 391
      org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/Proposal.java
  18. 299
      org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ProposalRound.java
  19. 316
      org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/RemoteGitReplica.java
  20. 233
      org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ReplicaConfig.java
  21. 96
      org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ReplicaFetchRequest.java
  22. 168
      org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ReplicaPushRequest.java
  23. 109
      org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ReplicaSnapshot.java
  24. 116
      org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/Round.java
  25. 270
      org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/StageBuilder.java
  26. 4
      org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/package-info.java
  27. 10
      org.eclipse.jgit/src/org/eclipse/jgit/transport/Daemon.java

1
org.eclipse.jgit.pgm/META-INF/MANIFEST.MF

@ -19,6 +19,7 @@ Import-Package: org.apache.commons.compress.archivers;version="[1.3,2.0)",
org.eclipse.jgit.dircache;version="[4.2.0,4.3.0)", org.eclipse.jgit.dircache;version="[4.2.0,4.3.0)",
org.eclipse.jgit.errors;version="[4.2.0,4.3.0)", org.eclipse.jgit.errors;version="[4.2.0,4.3.0)",
org.eclipse.jgit.gitrepo;version="[4.2.0,4.3.0)", org.eclipse.jgit.gitrepo;version="[4.2.0,4.3.0)",
org.eclipse.jgit.internal.ketch;version="[4.2.0,4.3.0)",
org.eclipse.jgit.internal.storage.file;version="[4.2.0,4.3.0)", org.eclipse.jgit.internal.storage.file;version="[4.2.0,4.3.0)",
org.eclipse.jgit.internal.storage.pack;version="[4.2.0,4.3.0)", org.eclipse.jgit.internal.storage.pack;version="[4.2.0,4.3.0)",
org.eclipse.jgit.internal.storage.reftree;version="[4.2.0,4.3.0)", org.eclipse.jgit.internal.storage.reftree;version="[4.2.0,4.3.0)",

47
org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Daemon.java

@ -45,18 +45,29 @@ package org.eclipse.jgit.pgm;
import java.io.File; import java.io.File;
import java.net.InetSocketAddress; import java.net.InetSocketAddress;
import java.net.URISyntaxException;
import java.text.MessageFormat; import java.text.MessageFormat;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import org.eclipse.jgit.internal.ketch.KetchLeader;
import org.eclipse.jgit.internal.ketch.KetchLeaderCache;
import org.eclipse.jgit.internal.ketch.KetchPreReceive;
import org.eclipse.jgit.internal.ketch.KetchSystem;
import org.eclipse.jgit.internal.ketch.KetchText;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.pgm.internal.CLIText; import org.eclipse.jgit.pgm.internal.CLIText;
import org.eclipse.jgit.storage.file.FileBasedConfig; import org.eclipse.jgit.storage.file.FileBasedConfig;
import org.eclipse.jgit.storage.file.WindowCacheConfig; import org.eclipse.jgit.storage.file.WindowCacheConfig;
import org.eclipse.jgit.storage.pack.PackConfig; import org.eclipse.jgit.storage.pack.PackConfig;
import org.eclipse.jgit.transport.DaemonClient; import org.eclipse.jgit.transport.DaemonClient;
import org.eclipse.jgit.transport.DaemonService; import org.eclipse.jgit.transport.DaemonService;
import org.eclipse.jgit.transport.ReceivePack;
import org.eclipse.jgit.transport.resolver.FileResolver; import org.eclipse.jgit.transport.resolver.FileResolver;
import org.eclipse.jgit.transport.resolver.ReceivePackFactory;
import org.eclipse.jgit.transport.resolver.ServiceNotAuthorizedException;
import org.eclipse.jgit.transport.resolver.ServiceNotEnabledException;
import org.eclipse.jgit.util.FS; import org.eclipse.jgit.util.FS;
import org.kohsuke.args4j.Argument; import org.kohsuke.args4j.Argument;
import org.kohsuke.args4j.Option; import org.kohsuke.args4j.Option;
@ -90,6 +101,13 @@ class Daemon extends TextBuiltin {
@Option(name = "--export-all", usage = "usage_exportWithoutGitDaemonExportOk") @Option(name = "--export-all", usage = "usage_exportWithoutGitDaemonExportOk")
boolean exportAll; boolean exportAll;
@Option(name = "--ketch")
KetchServerType ketchServerType;
enum KetchServerType {
LEADER;
}
@Argument(required = true, metaVar = "metaVar_directory", usage = "usage_directoriesToExport") @Argument(required = true, metaVar = "metaVar_directory", usage = "usage_directoriesToExport")
final List<File> directory = new ArrayList<File>(); final List<File> directory = new ArrayList<File>();
@ -146,7 +164,9 @@ class Daemon extends TextBuiltin {
service(d, n).setOverridable(true); service(d, n).setOverridable(true);
for (final String n : forbidOverride) for (final String n : forbidOverride)
service(d, n).setOverridable(false); service(d, n).setOverridable(false);
if (ketchServerType == KetchServerType.LEADER) {
startKetchLeader(d);
}
d.start(); d.start();
outw.println(MessageFormat.format(CLIText.get().listeningOn, d.getAddress())); outw.println(MessageFormat.format(CLIText.get().listeningOn, d.getAddress()));
} }
@ -159,4 +179,29 @@ class Daemon extends TextBuiltin {
throw die(MessageFormat.format(CLIText.get().serviceNotSupported, n)); throw die(MessageFormat.format(CLIText.get().serviceNotSupported, n));
return svc; return svc;
} }
private void startKetchLeader(org.eclipse.jgit.transport.Daemon daemon) {
KetchSystem system = new KetchSystem();
final KetchLeaderCache leaders = new KetchLeaderCache(system);
final ReceivePackFactory<DaemonClient> factory;
factory = daemon.getReceivePackFactory();
daemon.setReceivePackFactory(new ReceivePackFactory<DaemonClient>() {
@Override
public ReceivePack create(DaemonClient req, Repository repo)
throws ServiceNotEnabledException,
ServiceNotAuthorizedException {
ReceivePack rp = factory.create(req, repo);
KetchLeader leader;
try {
leader = leaders.get(repo);
} catch (URISyntaxException err) {
throw new ServiceNotEnabledException(
KetchText.get().invalidFollowerUri, err);
}
rp.setPreReceiveHook(new KetchPreReceive(leader));
return rp;
}
});
}
} }

1
org.eclipse.jgit/META-INF/MANIFEST.MF

@ -59,6 +59,7 @@ Export-Package: org.eclipse.jgit.annotations;version="4.2.0",
org.eclipse.jgit.ignore;version="4.2.0", org.eclipse.jgit.ignore;version="4.2.0",
org.eclipse.jgit.ignore.internal;version="4.2.0";x-friends:="org.eclipse.jgit.test", org.eclipse.jgit.ignore.internal;version="4.2.0";x-friends:="org.eclipse.jgit.test",
org.eclipse.jgit.internal;version="4.2.0";x-friends:="org.eclipse.jgit.test,org.eclipse.jgit.http.test", org.eclipse.jgit.internal;version="4.2.0";x-friends:="org.eclipse.jgit.test,org.eclipse.jgit.http.test",
org.eclipse.jgit.internal.ketch;version="4.2.0";x-friends:="org.eclipse.jgit.junit,org.eclipse.jgit.test,org.eclipse.jgit.pgm",
org.eclipse.jgit.internal.storage.dfs;version="4.2.0";x-friends:="org.eclipse.jgit.test,org.eclipse.jgit.http.server", org.eclipse.jgit.internal.storage.dfs;version="4.2.0";x-friends:="org.eclipse.jgit.test,org.eclipse.jgit.http.server",
org.eclipse.jgit.internal.storage.file;version="4.2.0"; org.eclipse.jgit.internal.storage.file;version="4.2.0";
x-friends:="org.eclipse.jgit.test, x-friends:="org.eclipse.jgit.test,

13
org.eclipse.jgit/resources/org/eclipse/jgit/internal/ketch/KetchText.properties

@ -0,0 +1,13 @@
accepted=accepted.
cannotFetchFromLocalReplica=cannot fetch from LocalReplica
failed=failed!
invalidFollowerUri=invalid follower URI
leaderFailedToStore=leader failed to store
localReplicaRequired=LocalReplica instance is required
mismatchedTxnNamespace=mismatched txnNamespace; expected {0} found {1}
outsideTxnNamespace=ref {0} is outside of txnNamespace {1}
proposingUpdates=Proposing updates
queuedProposalFailedToApply=queued proposal failed to apply
starting=starting!
unsupportedVoterCount=unsupported voter count {0}, expected one of {1}
waitingForQueue=Waiting for queue

141
org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ElectionRound.java

@ -0,0 +1,141 @@
/*
* Copyright (C) 2016, Google Inc.
* and other copyright owners as documented in the project's IP log.
*
* This program and the accompanying materials are made available
* under the terms of the Eclipse Distribution License v1.0 which
* accompanies this distribution, is reproduced below, and is
* available at http://www.eclipse.org/org/documents/edl-v10.php
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
*
* - Neither the name of the Eclipse Foundation, Inc. nor the
* names of its contributors may be used to endorse or promote
* products derived from this software without specific prior
* written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.eclipse.jgit.internal.ketch;
import static org.eclipse.jgit.internal.ketch.KetchConstants.TERM;
import java.io.IOException;
import java.util.List;
import org.eclipse.jgit.lib.CommitBuilder;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectInserter;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.lib.TreeFormatter;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.RevWalk;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* The initial {@link Round} for a leaderless repository, used to establish a
* leader.
*/
class ElectionRound extends Round {
private static final Logger log = LoggerFactory.getLogger(ElectionRound.class);
private long term;
ElectionRound(KetchLeader leader, LogIndex head) {
super(leader, head);
}
@Override
void start() throws IOException {
ObjectId id;
try (Repository git = leader.openRepository();
ObjectInserter inserter = git.newObjectInserter()) {
id = bumpTerm(git, inserter);
inserter.flush();
}
runAsync(id);
}
@Override
void success() {
// Do nothing upon election, KetchLeader will copy the term.
}
long getTerm() {
return term;
}
private ObjectId bumpTerm(Repository git, ObjectInserter inserter)
throws IOException {
CommitBuilder b = new CommitBuilder();
if (!ObjectId.zeroId().equals(acceptedOldIndex)) {
try (RevWalk rw = new RevWalk(git)) {
RevCommit c = rw.parseCommit(acceptedOldIndex);
b.setTreeId(c.getTree());
b.setParentId(acceptedOldIndex);
term = parseTerm(c.getFooterLines(TERM)) + 1;
}
} else {
term = 1;
b.setTreeId(inserter.insert(new TreeFormatter()));
}
StringBuilder msg = new StringBuilder();
msg.append(KetchConstants.TERM.getName())
.append(": ") //$NON-NLS-1$
.append(term);
String tag = leader.getSystem().newLeaderTag();
if (tag != null && !tag.isEmpty()) {
msg.append(' ').append(tag);
}
b.setAuthor(leader.getSystem().newCommitter());
b.setCommitter(b.getAuthor());
b.setMessage(msg.toString());
if (log.isDebugEnabled()) {
log.debug("Trying to elect myself " + b.getMessage()); //$NON-NLS-1$
}
return inserter.insert(b);
}
private static long parseTerm(List<String> footer) {
if (footer.isEmpty()) {
return 0;
}
String s = footer.get(0);
int p = s.indexOf(' ');
if (p > 0) {
s = s.substring(0, p);
}
return Long.parseLong(s, 10);
}
}

82
org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchConstants.java

@ -0,0 +1,82 @@
/*
* Copyright (C) 2016, Google Inc.
* and other copyright owners as documented in the project's IP log.
*
* This program and the accompanying materials are made available
* under the terms of the Eclipse Distribution License v1.0 which
* accompanies this distribution, is reproduced below, and is
* available at http://www.eclipse.org/org/documents/edl-v10.php
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
*
* - Neither the name of the Eclipse Foundation, Inc. nor the
* names of its contributors may be used to endorse or promote
* products derived from this software without specific prior
* written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.eclipse.jgit.internal.ketch;
import org.eclipse.jgit.revwalk.FooterKey;
/** Frequently used constants in a Ketch system. */
public class KetchConstants {
/**
* Default reference namespace holding {@link #ACCEPTED} and
* {@link #COMMITTED} references and the {@link #STAGE} sub-namespace.
*/
public static final String DEFAULT_TXN_NAMESPACE = "refs/txn/"; //$NON-NLS-1$
/** Reference name holding the RefTree accepted by a follower. */
public static final String ACCEPTED = "accepted"; //$NON-NLS-1$
/** Reference name holding the RefTree known to be committed. */
public static final String COMMITTED = "committed"; //$NON-NLS-1$
/** Reference subdirectory holding proposed heads. */
public static final String STAGE = "stage/"; //$NON-NLS-1$
/** Footer containing the current term. */
public static final FooterKey TERM = new FooterKey("Term"); //$NON-NLS-1$
/** Section for Ketch configuration ({@code ketch}). */
public static final String CONFIG_SECTION_KETCH = "ketch"; //$NON-NLS-1$
/** Behavior for a replica ({@code remote.$name.ketch-type}) */
public static final String CONFIG_KEY_TYPE = "ketch-type"; //$NON-NLS-1$
/** Behavior for a replica ({@code remote.$name.ketch-commit}) */
public static final String CONFIG_KEY_COMMIT = "ketch-commit"; //$NON-NLS-1$
/** Behavior for a replica ({@code remote.$name.ketch-speed}) */
public static final String CONFIG_KEY_SPEED = "ketch-speed"; //$NON-NLS-1$
private KetchConstants() {
}
}

624
org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchLeader.java

@ -0,0 +1,624 @@
/*
* Copyright (C) 2016, Google Inc.
* and other copyright owners as documented in the project's IP log.
*
* This program and the accompanying materials are made available
* under the terms of the Eclipse Distribution License v1.0 which
* accompanies this distribution, is reproduced below, and is
* available at http://www.eclipse.org/org/documents/edl-v10.php
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
*
* - Neither the name of the Eclipse Foundation, Inc. nor the
* names of its contributors may be used to endorse or promote
* products derived from this software without specific prior
* written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.eclipse.jgit.internal.ketch;
import static org.eclipse.jgit.internal.ketch.KetchLeader.State.CANDIDATE;
import static org.eclipse.jgit.internal.ketch.KetchLeader.State.LEADER;
import static org.eclipse.jgit.internal.ketch.KetchLeader.State.SHUTDOWN;
import static org.eclipse.jgit.internal.ketch.KetchReplica.Participation.FOLLOWER_ONLY;
import static org.eclipse.jgit.internal.ketch.Proposal.State.QUEUED;
import java.io.IOException;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import org.eclipse.jgit.internal.storage.reftree.RefTree;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.RevWalk;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* A leader managing consensus across remote followers.
* <p>
* A leader instance starts up in {@link State#CANDIDATE} and tries to begin a
* new term by sending an {@link ElectionRound} to all replicas. Its term starts
* if a majority of replicas have accepted this leader instance for the term.
* <p>
* Once elected by a majority the instance enters {@link State#LEADER} and runs
* proposals offered to {@link #queueProposal(Proposal)}. This continues until
* the leader is timed out for inactivity, or is deposed by a competing leader
* gaining its own majority.
* <p>
* Once timed out or deposed this {@code KetchLeader} instance should be
* discarded, and a new instance takes over.
* <p>
* Each leader instance coordinates a group of {@link KetchReplica}s. Replica
* instances are owned by the leader instance and must be discarded when the
* leader is discarded.
* <p>
* In Ketch all push requests are issued through the leader. The steps are as
* follows (see {@link KetchPreReceive} for an example):
* <ul>
* <li>Create a {@link Proposal} with the
* {@link org.eclipse.jgit.transport.ReceiveCommand}s that represent the push.
* <li>Invoke {@link #queueProposal(Proposal)} on the leader instance.
* <li>Wait for consensus with {@link Proposal#await()}.
* <li>To examine the status of the push, check {@link Proposal#getCommands()},
* looking at
* {@link org.eclipse.jgit.internal.storage.reftree.Command#getResult()}.
* </ul>
* <p>
* The leader gains consensus by first pushing the needed objects and a
* {@link RefTree} representing the desired target repository state to the
* {@code refs/txn/accepted} branch on each of the replicas. Once a majority has
* succeeded, the leader commits the state by either pushing the
* {@code refs/txn/accepted} value to {@code refs/txn/committed} (for
* Ketch-aware replicas) or by pushing updates to {@code refs/heads/master},
* etc. for stock Git replicas.
* <p>
* Internally, the actual transport to replicas is performed on background
* threads via the {@link KetchSystem}'s executor service. For performance, the
* {@link KetchLeader}, {@link KetchReplica} and {@link Proposal} objects share
* some state, and may invoke each other's methods on different threads. This
* access is protected by the leader's {@link #lock} object. Care must be taken
* to prevent concurrent access by correctly obtaining the leader's lock.
*/
public abstract class KetchLeader {
private static final Logger log = LoggerFactory.getLogger(KetchLeader.class);
/** Current state of the leader instance. */
public static enum State {
/** Newly created instance trying to elect itself leader. */
CANDIDATE,
/** Leader instance elected by a majority. */
LEADER,
/** Instance has been deposed by another with a more recent term. */
DEPOSED,
/** Leader has been gracefully shutdown, e.g. due to inactivity. */
SHUTDOWN;
}
private final KetchSystem system;
/** Leader's knowledge of replicas for this repository. */
private KetchReplica[] voters;
private KetchReplica[] followers;
private LocalReplica self;
/**
* Lock protecting all data within this leader instance.
* <p>
* This lock extends into the {@link KetchReplica} instances used by the
* leader. They share the same lock instance to simplify concurrency.
*/
final Lock lock;
private State state = CANDIDATE;
/** Term of this leader, once elected. */
private long term;
/**
* Pending proposals accepted into the queue in FIFO order.
* <p>
* These proposals were preflighted and do not contain any conflicts with
* each other and their expectations matched the leader's local view of the
* agreed upon {@code refs/txn/accepted} tree.
*/
private final List<Proposal> queued;
/**
* State of the repository's RefTree after applying all entries in
* {@link #queued}. New proposals must be consistent with this tree to be
* appended to the end of {@link #queued}.
* <p>
* Must be deep-copied with {@link RefTree#copy()} if
* {@link #roundHoldsReferenceToRefTree} is {@code true}.
*/
private RefTree refTree;
/**
* If {@code true} {@link #refTree} must be duplicated before queuing the
* next proposal. The {@link #refTree} was passed into the constructor of a
* {@link ProposalRound}, and that external reference to the {@link RefTree}
* object is held by the proposal until it materializes the tree object in
* the object store. This field is set {@code true} when the proposal begins
* execution and set {@code false} once tree objects are persisted in the
* local repository's object store or {@link #refTree} is replaced with a
* copy to isolate it from any running rounds.
* <p>
* If proposals arrive less frequently than the {@code RefTree} is written
* out to the repository the {@link #roundHoldsReferenceToRefTree} behavior
* avoids duplicating {@link #refTree}, reducing both time and memory used.
* However if proposals arrive more frequently {@link #refTree} must be
* duplicated to prevent newly queued proposals from corrupting the
* {@link #runningRound}.
*/
volatile boolean roundHoldsReferenceToRefTree;
/** End of the leader's log. */
private LogIndex headIndex;
/** Leader knows this (and all prior) states are committed. */
private LogIndex committedIndex;
/**
* Is the leader idle with no work pending? If {@code true} there is no work
* for the leader (normal state). This field is {@code false} when the
* leader thread is scheduled for execution, or while {@link #runningRound}
* defines a round in progress.
*/
private boolean idle;
/** Current round the leader is preparing and waiting for a vote on. */
private Round runningRound;
/**
* Construct a leader for a Ketch instance.
*
* @param system
* Ketch system configuration the leader must adhere to.
*/
protected KetchLeader(KetchSystem system) {
this.system = system;
this.lock = new ReentrantLock(true /* fair */);
this.queued = new ArrayList<>(4);
this.idle = true;
}
/** @return system configuration. */
KetchSystem getSystem() {
return system;
}
/**
* Configure the replicas used by this Ketch instance.
* <p>
* Replicas should be configured once at creation before any proposals are
* executed. Once elections happen, <b>reconfiguration is a complicated
* concept that is not currently supported</b>.
*
* @param replicas
* members participating with the same repository.
*/
public void setReplicas(Collection<KetchReplica> replicas) {
List<KetchReplica> v = new ArrayList<>(5);
List<KetchReplica> f = new ArrayList<>(5);
for (KetchReplica r : replicas) {
switch (r.getParticipation()) {
case FULL:
v.add(r);
break;
case FOLLOWER_ONLY:
f.add(r);
break;
}
}
Collection<Integer> validVoters = validVoterCounts();
if (!validVoters.contains(Integer.valueOf(v.size()))) {
throw new IllegalArgumentException(MessageFormat.format(
KetchText.get().unsupportedVoterCount,
Integer.valueOf(v.size()),
validVoters));
}
LocalReplica me = findLocal(v);
if (me == null) {
throw new IllegalArgumentException(
KetchText.get().localReplicaRequired);
}
lock.lock();
try {
voters = v.toArray(new KetchReplica[v.size()]);
followers = f.toArray(new KetchReplica[f.size()]);
self = me;
} finally {
lock.unlock();
}
}
private static Collection<Integer> validVoterCounts() {
@SuppressWarnings("boxing")
Integer[] valid = {
// An odd number of voting replicas is required.
1, 3, 5, 7, 9 };
return Arrays.asList(valid);
}
private static LocalReplica findLocal(Collection<KetchReplica> voters) {
for (KetchReplica r : voters) {
if (r instanceof LocalReplica) {
return (LocalReplica) r;
}
}
return null;
}
/**
* Get an instance of the repository for use by a leader thread.
* <p>
* The caller will close the repository.
*
* @return opened repository for use by the leader thread.
* @throws IOException
* cannot reopen the repository for the leader.
*/
protected abstract Repository openRepository() throws IOException;
/**
* Queue a reference update proposal for consensus.
* <p>
* This method does not wait for consensus to be reached. The proposal is
* checked to look for risks of conflicts, and then submitted into the queue
* for distribution as soon as possible.
* <p>
* Callers must use {@link Proposal#await()} to see if the proposal is done.
*
* @param proposal
* the proposed reference updates to queue for consideration.
* Once execution is complete the individual reference result
* fields will be populated with the outcome.
* @throws InterruptedException
* current thread was interrupted. The proposal may have been
* aborted if it was not yet queued for execution.
* @throws IOException
* unrecoverable error preventing proposals from being attempted
* by this leader.
*/
public void queueProposal(Proposal proposal)
throws InterruptedException, IOException {
try {
lock.lockInterruptibly();
} catch (InterruptedException e) {
proposal.abort();
throw e;
}
try {
if (refTree == null) {
initialize();
for (Proposal p : queued) {
refTree.apply(p.getCommands());
}
} else if (roundHoldsReferenceToRefTree) {
refTree = refTree.copy();
roundHoldsReferenceToRefTree = false;
}
if (!refTree.apply(proposal.getCommands())) {
// A conflict exists so abort the proposal.
proposal.abort();
return;
}
queued.add(proposal);
proposal.notifyState(QUEUED);
if (idle) {
scheduleLeader();
}
} finally {
lock.unlock();
}
}
private void initialize() throws IOException {
try (Repository git = openRepository(); RevWalk rw = new RevWalk(git)) {
self.initialize(git);
ObjectId accepted = self.getTxnAccepted();
if (!ObjectId.zeroId().equals(accepted)) {
RevCommit c = rw.parseCommit(accepted);
headIndex = LogIndex.unknown(accepted);
refTree = RefTree.read(rw.getObjectReader(), c.getTree());
} else {
headIndex = LogIndex.unknown(ObjectId.zeroId());
refTree = RefTree.newEmptyTree();
}
}
}
private void scheduleLeader() {
idle = false;
system.getExecutor().execute(new Runnable() {
@Override
public void run() {
runLeader();
}
});
}
private void runLeader() {
Round round;
lock.lock();
try {
switch (state) {
case CANDIDATE:
round = new ElectionRound(this, headIndex);
break;
case LEADER:
round = newProposalRound();
break;
case DEPOSED:
case SHUTDOWN:
default:
log.warn("Leader cannot run {}", state); //$NON-NLS-1$
// TODO(sop): Redirect proposals.
return;
}
} finally {
lock.unlock();
}
try {
round.start();
} catch (IOException e) {
// TODO(sop) Depose leader if it cannot use its repository.
log.error(KetchText.get().leaderFailedToStore, e);
lock.lock();
try {
nextRound();
} finally {
lock.unlock();
}
}
}
private ProposalRound newProposalRound() {
List<Proposal> todo = new ArrayList<>(queued);
queued.clear();
roundHoldsReferenceToRefTree = true;
return new ProposalRound(this, headIndex, todo, refTree);
}
/** @return term of this leader's reign. */
long getTerm() {
return term;
}
/** @return end of the leader's log. */
LogIndex getHead() {
return headIndex;
}
/**
* @return state leader knows it has committed across a quorum of replicas.
*/
LogIndex getCommitted() {
return committedIndex;
}
boolean isIdle() {
return idle;
}
void runAsync(Round round) {
lock.lock();
try {
// End of the log is this round. Once transport begins it is
// reasonable to assume at least one replica will eventually get
// this, and there is reasonable probability it commits.
headIndex = round.acceptedNewIndex;
runningRound = round;
for (KetchReplica replica : voters) {
replica.pushTxnAcceptedAsync(round);
}
for (KetchReplica replica : followers) {
replica.pushTxnAcceptedAsync(round);
}
} finally {
lock.unlock();
}
}
/**
* Asynchronous signal from a replica after completion.
* <p>
* Must be called while {@link #lock} is held by the replica.
*
* @param replica
* replica posting a completion event.
*/
void onReplicaUpdate(KetchReplica replica) {
if (log.isDebugEnabled()) {
log.debug("Replica {} finished:\n{}", //$NON-NLS-1$
replica.describeForLog(), snapshot());
}
if (replica.getParticipation() == FOLLOWER_ONLY) {
// Followers cannot vote, so votes haven't changed.
return;
} else if (runningRound == null) {
// No round running, no need to tally votes.
return;
}
assert headIndex.equals(runningRound.acceptedNewIndex);
int matching = 0;
for (KetchReplica r : voters) {
if (r.hasAccepted(headIndex)) {
matching++;
}
}
int quorum = voters.length / 2 + 1;
boolean success = matching >= quorum;
if (!success) {
return;
}
switch (state) {
case CANDIDATE:
term = ((ElectionRound) runningRound).getTerm();
state = LEADER;
if (log.isDebugEnabled()) {
log.debug("Won election, running term " + term); //$NON-NLS-1$
}
//$FALL-THROUGH$
case LEADER:
committedIndex = headIndex;
if (log.isDebugEnabled()) {
log.debug("Committed {} in term {}", //$NON-NLS-1$
committedIndex.describeForLog(),
Long.valueOf(term));
}
nextRound();
commitAsync(replica);
notifySuccess(runningRound);
if (log.isDebugEnabled()) {
log.debug("Leader state:\n{}", snapshot()); //$NON-NLS-1$
}
break;
default:
log.debug("Leader ignoring replica while in {}", state); //$NON-NLS-1$
break;
}
}
private void notifySuccess(Round round) {
// Drop the leader lock while notifying Proposal listeners.
lock.unlock();
try {
round.success();
} finally {
lock.lock();
}
}
private void commitAsync(KetchReplica caller) {
for (KetchReplica r : voters) {
if (r == caller) {
continue;
}
if (r.shouldPushUnbatchedCommit(committedIndex, isIdle())) {
r.pushCommitAsync(committedIndex);
}
}
for (KetchReplica r : followers) {
if (r == caller) {
continue;
}
if (r.shouldPushUnbatchedCommit(committedIndex, isIdle())) {
r.pushCommitAsync(committedIndex);
}
}
}
/** Schedule the next round; invoked while {@link #lock} is held. */
void nextRound() {
runningRound = null;
if (queued.isEmpty()) {
idle = true;
} else {
// Caller holds lock. Reschedule leader on a new thread so
// the call stack can unwind and lock is not held unexpectedly
// during prepare for the next round.
scheduleLeader();
}
}
/** @return snapshot this leader. */
public LeaderSnapshot snapshot() {
lock.lock();
try {
LeaderSnapshot s = new LeaderSnapshot();
s.state = state;
s.term = term;
s.headIndex = headIndex;
s.committedIndex = committedIndex;
s.idle = isIdle();
for (KetchReplica r : voters) {
s.replicas.add(r.snapshot());
}
for (KetchReplica r : followers) {
s.replicas.add(r.snapshot());
}
return s;
} finally {
lock.unlock();
}
}
/** Gracefully shutdown this leader and cancel outstanding operations. */
public void shutdown() {
lock.lock();
try {
if (state != SHUTDOWN) {
state = SHUTDOWN;
for (KetchReplica r : voters) {
r.shutdown();
}
for (KetchReplica r : followers) {
r.shutdown();
}
}
} finally {
lock.unlock();
}
}
@Override
public String toString() {
return snapshot().toString();
}
}

126
org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchLeaderCache.java

@ -0,0 +1,126 @@
/*
* Copyright (C) 2016, Google Inc.
* and other copyright owners as documented in the project's IP log.
*
* This program and the accompanying materials are made available
* under the terms of the Eclipse Distribution License v1.0 which
* accompanies this distribution, is reproduced below, and is
* available at http://www.eclipse.org/org/documents/edl-v10.php
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
*
* - Neither the name of the Eclipse Foundation, Inc. nor the
* names of its contributors may be used to endorse or promote
* products derived from this software without specific prior
* written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.eclipse.jgit.internal.ketch;
import java.net.URISyntaxException;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import org.eclipse.jgit.internal.storage.dfs.DfsRepository;
import org.eclipse.jgit.lib.Repository;
/**
* A cache of live leader instances, keyed by repository.
* <p>
* Ketch only assigns a leader to a repository when needed. If
* {@link #get(Repository)} is called for a repository that does not have a
* leader, the leader is created and added to the cache.
*/
public class KetchLeaderCache {
private final KetchSystem system;
private final ConcurrentMap<String, KetchLeader> leaders;
private final Lock startLock;
/**
* Initialize a new leader cache.
*
* @param system
* system configuration for the leaders
*/
public KetchLeaderCache(KetchSystem system) {
this.system = system;
leaders = new ConcurrentHashMap<>();
startLock = new ReentrantLock(true /* fair */);
}
/**
* Lookup the leader instance for a given repository.
*
* @param repo
* repository to get the leader for.
* @return the leader instance for the repository.
* @throws URISyntaxException
* remote configuration contains an invalid URL.
*/
public KetchLeader get(Repository repo)
throws URISyntaxException {
String key = computeKey(repo);
KetchLeader leader = leaders.get(key);
if (leader != null) {
return leader;
}
return startLeader(key, repo);
}
private KetchLeader startLeader(String key, Repository repo)
throws URISyntaxException {
startLock.lock();
try {
KetchLeader leader = leaders.get(key);
if (leader != null) {
return leader;
}
leader = system.createLeader(repo);
leaders.put(key, leader);
return leader;
} finally {
startLock.unlock();
}
}
private static String computeKey(Repository repo) {
if (repo instanceof DfsRepository) {
DfsRepository dfs = (DfsRepository) repo;
return dfs.getDescription().getRepositoryName();
}
if (repo.getDirectory() != null) {
return repo.getDirectory().toURI().toString();
}
throw new IllegalArgumentException();
}
}

155
org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchPreReceive.java

@ -0,0 +1,155 @@
/*
* Copyright (C) 2016, Google Inc.
* and other copyright owners as documented in the project's IP log.
*
* This program and the accompanying materials are made available
* under the terms of the Eclipse Distribution License v1.0 which
* accompanies this distribution, is reproduced below, and is
* available at http://www.eclipse.org/org/documents/edl-v10.php
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
*
* - Neither the name of the Eclipse Foundation, Inc. nor the
* names of its contributors may be used to endorse or promote
* products derived from this software without specific prior
* written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.eclipse.jgit.internal.ketch;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.eclipse.jgit.internal.ketch.Proposal.State.EXECUTED;
import static org.eclipse.jgit.internal.ketch.Proposal.State.QUEUED;
import static org.eclipse.jgit.transport.ReceiveCommand.Result.NOT_ATTEMPTED;
import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_OTHER_REASON;
import java.io.IOException;
import java.util.Collection;
import org.eclipse.jgit.internal.JGitText;
import org.eclipse.jgit.transport.PreReceiveHook;
import org.eclipse.jgit.transport.ProgressSpinner;
import org.eclipse.jgit.transport.ReceiveCommand;
import org.eclipse.jgit.transport.ReceivePack;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* PreReceiveHook for handling push traffic in a Ketch system.
* <p>
* Install an instance on {@link ReceivePack} to capture the commands and other
* connection state and relay them through the {@link KetchLeader}, allowing the
* leader to gain consensus about the new reference state.
*/
public class KetchPreReceive implements PreReceiveHook {
private static final Logger log = LoggerFactory.getLogger(KetchPreReceive.class);
private final KetchLeader leader;
/**
* Construct a hook executing updates through a {@link KetchLeader}.
*
* @param leader
* leader for this repository.
*/
public KetchPreReceive(KetchLeader leader) {
this.leader = leader;
}
@Override
public void onPreReceive(ReceivePack rp, Collection<ReceiveCommand> cmds) {
cmds = ReceiveCommand.filter(cmds, NOT_ATTEMPTED);
if (cmds.isEmpty()) {
return;
}
try {
Proposal proposal = new Proposal(rp.getRevWalk(), cmds)
.setPushCertificate(rp.getPushCertificate())
.setAuthor(rp.getRefLogIdent())
.setMessage("push"); //$NON-NLS-1$
leader.queueProposal(proposal);
if (proposal.isDone()) {
// This failed fast, e.g. conflict or bad precondition.
return;
}
ProgressSpinner spinner = new ProgressSpinner(
rp.getMessageOutputStream());
if (proposal.getState() == QUEUED) {
waitForQueue(proposal, spinner);
}
if (!proposal.isDone()) {
waitForPropose(proposal, spinner);
}
} catch (IOException | InterruptedException e) {
String msg = JGitText.get().transactionAborted;
for (ReceiveCommand cmd : cmds) {
if (cmd.getResult() == NOT_ATTEMPTED) {
cmd.setResult(REJECTED_OTHER_REASON, msg);
}
}
log.error(msg, e);
}
}
private void waitForQueue(Proposal proposal, ProgressSpinner spinner)
throws InterruptedException {
spinner.beginTask(KetchText.get().waitingForQueue, 1, SECONDS);
while (!proposal.awaitStateChange(QUEUED, 250, MILLISECONDS)) {
spinner.update();
}
switch (proposal.getState()) {
case RUNNING:
default:
spinner.endTask(KetchText.get().starting);
break;
case EXECUTED:
spinner.endTask(KetchText.get().accepted);
break;
case ABORTED:
spinner.endTask(KetchText.get().failed);
break;
}
}
private void waitForPropose(Proposal proposal, ProgressSpinner spinner)
throws InterruptedException {
spinner.beginTask(KetchText.get().proposingUpdates, 2, SECONDS);
while (!proposal.await(250, MILLISECONDS)) {
spinner.update();
}
spinner.endTask(proposal.getState() == EXECUTED
? KetchText.get().accepted
: KetchText.get().failed);
}
}

755
org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchReplica.java

@ -0,0 +1,755 @@
/*
* Copyright (C) 2016, Google Inc.
* and other copyright owners as documented in the project's IP log.
*
* This program and the accompanying materials are made available
* under the terms of the Eclipse Distribution License v1.0 which
* accompanies this distribution, is reproduced below, and is
* available at http://www.eclipse.org/org/documents/edl-v10.php
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
*
* - Neither the name of the Eclipse Foundation, Inc. nor the
* names of its contributors may be used to endorse or promote
* products derived from this software without specific prior
* written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.eclipse.jgit.internal.ketch;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static org.eclipse.jgit.internal.ketch.KetchReplica.CommitSpeed.BATCHED;
import static org.eclipse.jgit.internal.ketch.KetchReplica.CommitSpeed.FAST;
import static org.eclipse.jgit.internal.ketch.KetchReplica.State.CURRENT;
import static org.eclipse.jgit.internal.ketch.KetchReplica.State.LAGGING;
import static org.eclipse.jgit.internal.ketch.KetchReplica.State.OFFLINE;
import static org.eclipse.jgit.internal.ketch.KetchReplica.State.UNKNOWN;
import static org.eclipse.jgit.lib.Constants.HEAD;
import static org.eclipse.jgit.lib.FileMode.TYPE_GITLINK;
import static org.eclipse.jgit.transport.ReceiveCommand.Result.NOT_ATTEMPTED;
import static org.eclipse.jgit.transport.ReceiveCommand.Result.OK;
import static org.eclipse.jgit.transport.ReceiveCommand.Type.CREATE;
import java.io.IOException;
import java.lang.ref.WeakReference;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.Future;
import org.eclipse.jgit.annotations.NonNull;
import org.eclipse.jgit.annotations.Nullable;
import org.eclipse.jgit.internal.storage.reftree.RefTree;
import org.eclipse.jgit.lib.AnyObjectId;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.revwalk.RevWalk;
import org.eclipse.jgit.transport.ReceiveCommand;
import org.eclipse.jgit.treewalk.TreeWalk;
import org.eclipse.jgit.util.SystemReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* A Ketch replica, either {@link LocalReplica} or {@link RemoteGitReplica}.
* <p>
* Replicas can be either a stock Git replica, or a Ketch-aware replica.
* <p>
* A stock Git replica has no special knowledge of Ketch and simply stores
* objects and references. Ketch communicates with the stock Git replica using
* the Git push wire protocol. The {@link KetchLeader} commits an agreed upon
* state by pushing all references to the Git replica, for example
* {@code "refs/heads/master"} is pushed during commit. Stock Git replicas use
* {@link CommitMethod#ALL_REFS} to record the final state.
* <p>
* Ketch-aware replicas understand the {@code RefTree} sent during the proposal
* and during commit are able to update their own reference space to match the
* state represented by the {@code RefTree}. Ketch-aware replicas typically use
* a {@link org.eclipse.jgit.internal.storage.reftree.RefTreeDatabase} and
* {@link CommitMethod#TXN_COMMITTED} to record the final state.
* <p>
* KetchReplica instances are tightly coupled with a single {@link KetchLeader}.
* Some state may be accessed by the leader thread and uses the leader's own
* {@link KetchLeader#lock} to protect shared data.
*/
public abstract class KetchReplica {
static final Logger log = LoggerFactory.getLogger(KetchReplica.class);
private static final byte[] PEEL = { ' ', '^' };
/** Participation of a replica in establishing consensus. */
public enum Participation {
/** Replica can vote. */
FULL,
/** Replica does not vote, but tracks leader. */
FOLLOWER_ONLY;
}
/** How this replica wants to receive Ketch commit operations. */
public enum CommitMethod {
/** All references are pushed to the peer as standard Git. */
ALL_REFS,
/** Only {@code refs/txn/committed} is written/updated. */
TXN_COMMITTED;
}
/** Delay before committing to a replica. */
public enum CommitSpeed {
/**
* Send the commit immediately, even if it could be batched with the
* next proposal.
*/
FAST,
/**
* If the next proposal is available, batch the commit with it,
* otherwise just send the commit. This generates less network use, but
* may provide slower consistency on the replica.
*/
BATCHED;
}
/** Current state of a replica. */
public enum State {
/** Leader has not yet contacted the replica. */
UNKNOWN,
/** Replica is behind the consensus. */
LAGGING,
/** Replica matches the consensus. */
CURRENT,
/** Replica has a different (or unknown) history. */
DIVERGENT,
/** Replica's history contains the leader's history. */
AHEAD,
/** Replica can not be contacted. */
OFFLINE;
}
private final KetchLeader leader;
private final String replicaName;
private final Participation participation;
private final CommitMethod commitMethod;
private final CommitSpeed commitSpeed;
private final long minRetryMillis;
private final long maxRetryMillis;
private final Map<ObjectId, List<ReceiveCommand>> staged;
private final Map<String, ReceiveCommand> running;
private final Map<String, ReceiveCommand> waiting;
private final List<ReplicaPushRequest> queued;
/**
* Value known for {@code "refs/txn/accepted"}.
* <p>
* Raft literature refers to this as {@code matchIndex}.
*/
private ObjectId txnAccepted;
/**
* Value known for {@code "refs/txn/committed"}.
* <p>
* Raft literature refers to this as {@code commitIndex}. In traditional
* Raft this is a state variable inside the follower implementation, but
* Ketch keeps it in the leader.
*/
private ObjectId txnCommitted;
/** What is happening with this replica. */
private State state = UNKNOWN;
private String error;
/** Scheduled retry due to communication failure. */
private Future<?> retryFuture;
private long lastRetryMillis;
private long retryAtMillis;
/**
* Configure a replica representation.
*
* @param leader
* instance this replica follows.
* @param name
* unique-ish name identifying this replica for debugging.
* @param cfg
* how Ketch should treat the replica.
*/
protected KetchReplica(KetchLeader leader, String name, ReplicaConfig cfg) {
this.leader = leader;
this.replicaName = name;
this.participation = cfg.getParticipation();
this.commitMethod = cfg.getCommitMethod();
this.commitSpeed = cfg.getCommitSpeed();
this.minRetryMillis = cfg.getMinRetry(MILLISECONDS);
this.maxRetryMillis = cfg.getMaxRetry(MILLISECONDS);
this.staged = new HashMap<>();
this.running = new HashMap<>();
this.waiting = new HashMap<>();
this.queued = new ArrayList<>(4);
}
/** @return system configuration. */
public KetchSystem getSystem() {
return getLeader().getSystem();
}
/** @return leader instance this replica follows. */
public KetchLeader getLeader() {
return leader;
}
/** @return unique-ish name for debugging. */
public String getName() {
return replicaName;
}
/** @return description of this replica for error/debug logging purposes. */
protected String describeForLog() {
return getName();
}
/** @return how the replica participates in this Ketch system. */
public Participation getParticipation() {
return participation;
}
/** @return how Ketch will commit to the repository. */
public CommitMethod getCommitMethod() {
return commitMethod;
}
/** @return when Ketch will commit to the repository. */
public CommitSpeed getCommitSpeed() {
return commitSpeed;
}
/**
* Called by leader to perform graceful shutdown.
* <p>
* Default implementation cancels any scheduled retry. Subclasses may add
* additional logic before or after calling {@code super.shutdown()}.
* <p>
* Called with {@link KetchLeader#lock} held by caller.
*/
protected void shutdown() {
Future<?> f = retryFuture;
if (f != null) {
retryFuture = null;
f.cancel(true);
}
}
ReplicaSnapshot snapshot() {
ReplicaSnapshot s = new ReplicaSnapshot(this);
s.accepted = txnAccepted;
s.committed = txnCommitted;
s.state = state;
s.error = error;
s.retryAtMillis = waitingForRetry() ? retryAtMillis : 0;
return s;
}
/**
* Update the leader's view of the replica after a poll.
* <p>
* Called with {@link KetchLeader#lock} held by caller.
*
* @param refs
* map of refs from the replica.
*/
void initialize(Map<String, Ref> refs) {
if (txnAccepted == null) {
txnAccepted = getId(refs.get(getSystem().getTxnAccepted()));
}
if (txnCommitted == null) {
txnCommitted = getId(refs.get(getSystem().getTxnCommitted()));
}
}
ObjectId getTxnAccepted() {
return txnAccepted;
}
boolean hasAccepted(LogIndex id) {
return equals(txnAccepted, id);
}
private static boolean equals(@Nullable ObjectId a, LogIndex b) {
return a != null && b != null && AnyObjectId.equals(a, b);
}
/**
* Schedule a proposal round with the replica.
* <p>
* Called with {@link KetchLeader#lock} held by caller.
*
* @param round
* current round being run by the leader.
*/
void pushTxnAcceptedAsync(Round round) {
List<ReceiveCommand> cmds = new ArrayList<>();
if (commitSpeed == BATCHED) {
LogIndex committedIndex = leader.getCommitted();
if (equals(txnAccepted, committedIndex)
&& !equals(txnCommitted, committedIndex)) {
prepareTxnCommitted(cmds, committedIndex);
}
}
// TODO(sop) Lagging replicas should build accept on the fly.
if (round.stageCommands != null) {
for (ReceiveCommand cmd : round.stageCommands) {
// TODO(sop): Do not send certain object graphs to replica.
cmds.add(copy(cmd));
}
}
cmds.add(new ReceiveCommand(
round.acceptedOldIndex, round.acceptedNewIndex,
getSystem().getTxnAccepted()));
pushAsync(new ReplicaPushRequest(this, cmds));
}
private static ReceiveCommand copy(ReceiveCommand c) {
return new ReceiveCommand(c.getOldId(), c.getNewId(), c.getRefName());
}
boolean shouldPushUnbatchedCommit(LogIndex committed, boolean leaderIdle) {
return (leaderIdle || commitSpeed == FAST) && hasAccepted(committed);
}
void pushCommitAsync(LogIndex committed) {
List<ReceiveCommand> cmds = new ArrayList<>();
prepareTxnCommitted(cmds, committed);
pushAsync(new ReplicaPushRequest(this, cmds));
}
private void prepareTxnCommitted(List<ReceiveCommand> cmds,
ObjectId committed) {
removeStaged(cmds, committed);
cmds.add(new ReceiveCommand(
txnCommitted, committed,
getSystem().getTxnCommitted()));
}
private void removeStaged(List<ReceiveCommand> cmds, ObjectId committed) {
List<ReceiveCommand> a = staged.remove(committed);
if (a != null) {
delete(cmds, a);
}
if (staged.isEmpty() || !(committed instanceof LogIndex)) {
return;
}
LogIndex committedIndex = (LogIndex) committed;
Iterator<Map.Entry<ObjectId, List<ReceiveCommand>>> itr = staged
.entrySet().iterator();
while (itr.hasNext()) {
Map.Entry<ObjectId, List<ReceiveCommand>> e = itr.next();
if (e.getKey() instanceof LogIndex) {
LogIndex stagedIndex = (LogIndex) e.getKey();
if (stagedIndex.isBefore(committedIndex)) {
delete(cmds, e.getValue());
itr.remove();
}
}
}
}
private static void delete(List<ReceiveCommand> cmds,
List<ReceiveCommand> createCmds) {
for (ReceiveCommand cmd : createCmds) {
ObjectId id = cmd.getNewId();
String name = cmd.getRefName();
cmds.add(new ReceiveCommand(id, ObjectId.zeroId(), name));
}
}
/**
* Determine the next push for this replica (if any) and start it.
* <p>
* If the replica has successfully accepted the committed state of the
* leader, this method will push all references to the replica using the
* configured {@link CommitMethod}.
* <p>
* If the replica is {@link State#LAGGING} this method will begin catch up
* by sending a more recent {@code refs/txn/accepted}.
* <p>
* Must be invoked with {@link KetchLeader#lock} held by caller.
*/
private void runNextPushRequest() {
LogIndex committed = leader.getCommitted();
if (!equals(txnCommitted, committed)
&& shouldPushUnbatchedCommit(committed, leader.isIdle())) {
pushCommitAsync(committed);
}
if (queued.isEmpty() || !running.isEmpty() || waitingForRetry()) {
return;
}
// Collapse all queued requests into a single request.
Map<String, ReceiveCommand> cmdMap = new HashMap<>();
for (ReplicaPushRequest req : queued) {
for (ReceiveCommand cmd : req.getCommands()) {
String name = cmd.getRefName();
ReceiveCommand old = cmdMap.remove(name);
if (old != null) {
cmd = new ReceiveCommand(
old.getOldId(), cmd.getNewId(),
name);
}
cmdMap.put(name, cmd);
}
}
queued.clear();
waiting.clear();
List<ReceiveCommand> next = new ArrayList<>(cmdMap.values());
for (ReceiveCommand cmd : next) {
running.put(cmd.getRefName(), cmd);
}
startPush(new ReplicaPushRequest(this, next));
}
private void pushAsync(ReplicaPushRequest req) {
if (defer(req)) {
// TODO(sop) Collapse during long retry outage.
for (ReceiveCommand cmd : req.getCommands()) {
waiting.put(cmd.getRefName(), cmd);
}
queued.add(req);
} else {
for (ReceiveCommand cmd : req.getCommands()) {
running.put(cmd.getRefName(), cmd);
}
startPush(req);
}
}
private boolean defer(ReplicaPushRequest req) {
if (waitingForRetry()) {
// Prior communication failure; everything is deferred.
return true;
}
for (ReceiveCommand nextCmd : req.getCommands()) {
ReceiveCommand priorCmd = waiting.get(nextCmd.getRefName());
if (priorCmd == null) {
priorCmd = running.get(nextCmd.getRefName());
}
if (priorCmd != null) {
// Another request pending on same ref; that must go first.
// Verify priorCmd.newId == nextCmd.oldId?
return true;
}
}
return false;
}
private boolean waitingForRetry() {
Future<?> f = retryFuture;
return f != null && !f.isDone();
}
private void retryLater(ReplicaPushRequest req) {
Collection<ReceiveCommand> cmds = req.getCommands();
for (ReceiveCommand cmd : cmds) {
cmd.setResult(NOT_ATTEMPTED, null);
if (!waiting.containsKey(cmd.getRefName())) {
waiting.put(cmd.getRefName(), cmd);
}
}
queued.add(0, new ReplicaPushRequest(this, cmds));
if (!waitingForRetry()) {
long delay = KetchSystem.delay(
lastRetryMillis,
minRetryMillis, maxRetryMillis);
if (log.isDebugEnabled()) {
log.debug("Retrying {} after {} ms", //$NON-NLS-1$
describeForLog(), Long.valueOf(delay));
}
lastRetryMillis = delay;
retryAtMillis = SystemReader.getInstance().getCurrentTime() + delay;
retryFuture = getSystem().getExecutor()
.schedule(new WeakRetryPush(this), delay, MILLISECONDS);
}
}
/** Weakly holds a retrying replica, allowing it to garbage collect. */
static class WeakRetryPush extends WeakReference<KetchReplica>
implements Callable<Void> {
WeakRetryPush(KetchReplica r) {
super(r);
}
@Override
public Void call() throws Exception {
KetchReplica r = get();
if (r != null) {
r.doRetryPush();
}
return null;
}
}
private void doRetryPush() {
leader.lock.lock();
try {
retryFuture = null;
runNextPushRequest();
} finally {
leader.lock.unlock();
}
}
/**
* Begin executing a single push.
* <p>
* This method must move processing onto another thread.
* Called with {@link KetchLeader#lock} held by caller.
*
* @param req
* the request to send to the replica.
*/
protected abstract void startPush(ReplicaPushRequest req);
/**
* Callback from {@link ReplicaPushRequest} upon success or failure.
* <p>
* Acquires the {@link KetchLeader#lock} and updates the leader's internal
* knowledge about this replica to reflect what has been learned during a
* push to the replica. In some cases of divergence this method may take
* some time to determine how the replica has diverged; to reduce contention
* this is evaluated before acquiring the leader lock.
*
* @param repo
* local repository instance used by the push thread.
* @param req
* push request just attempted.
*/
void afterPush(@Nullable Repository repo, ReplicaPushRequest req) {
ReceiveCommand acceptCmd = null;
ReceiveCommand commitCmd = null;
List<ReceiveCommand> stages = null;
for (ReceiveCommand cmd : req.getCommands()) {
String name = cmd.getRefName();
if (name.equals(getSystem().getTxnAccepted())) {
acceptCmd = cmd;
} else if (name.equals(getSystem().getTxnCommitted())) {
commitCmd = cmd;
} else if (cmd.getResult() == OK && cmd.getType() == CREATE
&& name.startsWith(getSystem().getTxnStage())) {
if (stages == null) {
stages = new ArrayList<>();
}
stages.add(cmd);
}
}
State newState = null;
ObjectId acceptId = readId(req, acceptCmd);
if (repo != null && acceptCmd != null && acceptCmd.getResult() != OK
&& req.getException() == null) {
try (LagCheck lag = new LagCheck(this, repo)) {
newState = lag.check(acceptId, acceptCmd);
acceptId = lag.getRemoteId();
}
}
leader.lock.lock();
try {
for (ReceiveCommand cmd : req.getCommands()) {
running.remove(cmd.getRefName());
}
Throwable err = req.getException();
if (err != null) {
state = OFFLINE;
error = err.toString();
retryLater(req);
leader.onReplicaUpdate(this);
return;
}
lastRetryMillis = 0;
error = null;
updateView(req, acceptId, commitCmd);
if (acceptCmd != null && acceptCmd.getResult() == OK) {
state = hasAccepted(leader.getHead()) ? CURRENT : LAGGING;
if (stages != null) {
staged.put(acceptCmd.getNewId(), stages);
}
} else if (newState != null) {
state = newState;
}
leader.onReplicaUpdate(this);
runNextPushRequest();
} finally {
leader.lock.unlock();
}
}
private void updateView(ReplicaPushRequest req, @Nullable ObjectId acceptId,
ReceiveCommand commitCmd) {
if (acceptId != null) {
txnAccepted = acceptId;
}
ObjectId committed = readId(req, commitCmd);
if (committed != null) {
txnCommitted = committed;
} else if (acceptId != null && txnCommitted == null) {
// Initialize during first conversation.
Map<String, Ref> adv = req.getRefs();
if (adv != null) {
Ref refs = adv.get(getSystem().getTxnCommitted());
txnCommitted = getId(refs);
}
}
}
@Nullable
private static ObjectId readId(ReplicaPushRequest req,
@Nullable ReceiveCommand cmd) {
if (cmd == null) {
// Ref was not in the command list, do not trust advertisement.
return null;
} else if (cmd.getResult() == OK) {
// Currently at newId.
return cmd.getNewId();
}
Map<String, Ref> refs = req.getRefs();
return refs != null ? getId(refs.get(cmd.getRefName())) : null;
}
/**
* Fetch objects from the remote using the calling thread.
* <p>
* Called without {@link KetchLeader#lock}.
*
* @param repo
* local repository to fetch objects into.
* @param req
* the request to fetch from a replica.
* @throws IOException
* communication with the replica was not possible.
*/
protected abstract void blockingFetch(Repository repo,
ReplicaFetchRequest req) throws IOException;
/**
* Build a list of commands to commit {@link CommitMethod#ALL_REFS}.
*
* @param git
* local leader repository to read committed state from.
* @param current
* all known references in the replica's repository. Typically
* this comes from a push advertisement.
* @param committed
* state being pushed to {@code refs/txn/committed}.
* @return commands to update during commit.
* @throws IOException
* cannot read the committed state.
*/
protected Collection<ReceiveCommand> prepareCommit(Repository git,
Map<String, Ref> current, ObjectId committed) throws IOException {
List<ReceiveCommand> delta = new ArrayList<>();
Map<String, Ref> remote = new HashMap<>(current);
try (RevWalk rw = new RevWalk(git);
TreeWalk tw = new TreeWalk(rw.getObjectReader())) {
tw.setRecursive(true);
tw.addTree(rw.parseCommit(committed).getTree());
while (tw.next()) {
if (tw.getRawMode(0) != TYPE_GITLINK
|| tw.isPathSuffix(PEEL, 2)) {
// Symbolic references cannot be pushed.
// Caching peeled values is handled remotely.
continue;
}
// TODO(sop) Do not send certain ref names to replica.
String name = RefTree.refName(tw.getPathString());
Ref oldRef = remote.remove(name);
ObjectId oldId = getId(oldRef);
ObjectId newId = tw.getObjectId(0);
if (!AnyObjectId.equals(oldId, newId)) {
delta.add(new ReceiveCommand(oldId, newId, name));
}
}
}
// Delete any extra references not in the committed state.
for (Ref ref : remote.values()) {
if (canDelete(ref)) {
delta.add(new ReceiveCommand(
ref.getObjectId(), ObjectId.zeroId(),
ref.getName()));
}
}
return delta;
}
boolean canDelete(Ref ref) {
String name = ref.getName();
if (HEAD.equals(name)) {
return false;
}
if (name.startsWith(getSystem().getTxnNamespace())) {
return false;
}
// TODO(sop) Do not delete precious names from replica.
return true;
}
@NonNull
static ObjectId getId(@Nullable Ref ref) {
if (ref != null) {
ObjectId id = ref.getObjectId();
if (id != null) {
return id;
}
}
return ObjectId.zeroId();
}
}

301
org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchSystem.java

@ -0,0 +1,301 @@
/*
* Copyright (C) 2016, Google Inc.
* and other copyright owners as documented in the project's IP log.
*
* This program and the accompanying materials are made available
* under the terms of the Eclipse Distribution License v1.0 which
* accompanies this distribution, is reproduced below, and is
* available at http://www.eclipse.org/org/documents/edl-v10.php
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
*
* - Neither the name of the Eclipse Foundation, Inc. nor the
* names of its contributors may be used to endorse or promote
* products derived from this software without specific prior
* written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.eclipse.jgit.internal.ketch;
import static org.eclipse.jgit.internal.ketch.KetchConstants.ACCEPTED;
import static org.eclipse.jgit.internal.ketch.KetchConstants.COMMITTED;
import static org.eclipse.jgit.internal.ketch.KetchConstants.CONFIG_KEY_TYPE;
import static org.eclipse.jgit.internal.ketch.KetchConstants.CONFIG_SECTION_KETCH;
import static org.eclipse.jgit.internal.ketch.KetchConstants.DEFAULT_TXN_NAMESPACE;
import static org.eclipse.jgit.internal.ketch.KetchConstants.STAGE;
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_NAME;
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_REMOTE;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.atomic.AtomicInteger;
import org.eclipse.jgit.annotations.Nullable;
import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.lib.PersonIdent;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.transport.RemoteConfig;
import org.eclipse.jgit.transport.URIish;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Ketch system-wide configuration.
* <p>
* This class provides useful defaults for testing and small proof of concepts.
* Full scale installations are expected to subclass and override methods to
* provide consistent configuration across all managed repositories.
* <p>
* Servers should configure their own {@link ScheduledExecutorService}.
*/
public class KetchSystem {
private static final Random RNG = new Random();
/** @return default executor, one thread per available processor. */
public static ScheduledExecutorService defaultExecutor() {
return DefaultExecutorHolder.I;
}
private final ScheduledExecutorService executor;
private final String txnNamespace;
private final String txnAccepted;
private final String txnCommitted;
private final String txnStage;
/** Create a default system with a thread pool of 1 thread per CPU. */
public KetchSystem() {
this(defaultExecutor(), DEFAULT_TXN_NAMESPACE);
}
/**
* Create a Ketch system with the provided executor service.
*
* @param executor
* thread pool to run background operations.
* @param txnNamespace
* reference namespace for the RefTree graph and associated
* transaction state. Must begin with {@code "refs/"} and end
* with {@code '/'}, for example {@code "refs/txn/"}.
*/
public KetchSystem(ScheduledExecutorService executor, String txnNamespace) {
this.executor = executor;
this.txnNamespace = txnNamespace;
this.txnAccepted = txnNamespace + ACCEPTED;
this.txnCommitted = txnNamespace + COMMITTED;
this.txnStage = txnNamespace + STAGE;
}
/** @return executor to perform background operations. */
public ScheduledExecutorService getExecutor() {
return executor;
}
/**
* Get the namespace used for the RefTree graph and transaction management.
*
* @return reference namespace such as {@code "refs/txn/"}.
*/
public String getTxnNamespace() {
return txnNamespace;
}
/** @return name of the accepted RefTree graph. */
public String getTxnAccepted() {
return txnAccepted;
}
/** @return name of the committed RefTree graph. */
public String getTxnCommitted() {
return txnCommitted;
}
/** @return prefix for staged objects, e.g. {@code "refs/txn/stage/"}. */
public String getTxnStage() {
return txnStage;
}
/** @return identity line for the committer header of a RefTreeGraph. */
public PersonIdent newCommitter() {
String name = "ketch"; //$NON-NLS-1$
String email = "ketch@system"; //$NON-NLS-1$
return new PersonIdent(name, email);
}
/**
* Construct a random tag to identify a candidate during leader election.
* <p>
* Multiple processes trying to elect themselves leaders at exactly the same
* time (rounded to seconds) using the same {@link #newCommitter()} identity
* strings, for the same term, may generate the same ObjectId for the
* election commit and falsely assume they have both won.
* <p>
* Candidates add this tag to their election ballot commit to disambiguate
* the election. The tag only needs to be unique for a given triplet of
* {@link #newCommitter()}, system time (rounded to seconds), and term. If
* every replica in the system uses a unique {@code newCommitter} (such as
* including the host name after the {@code "@"} in the email address) the
* tag could be the empty string.
* <p>
* The default implementation generates a few bytes of random data.
*
* @return unique tag; null or empty string if {@code newCommitter()} is
* sufficiently unique to identify the leader.
*/
@Nullable
public String newLeaderTag() {
int n = RNG.nextInt(1 << (6 * 4));
return String.format("%06x", Integer.valueOf(n)); //$NON-NLS-1$
}
/**
* Construct the KetchLeader instance of a repository.
*
* @param repo
* local repository stored by the leader.
* @return leader instance.
* @throws URISyntaxException
* a follower configuration contains an unsupported URI.
*/
public KetchLeader createLeader(final Repository repo)
throws URISyntaxException {
KetchLeader leader = new KetchLeader(this) {
@Override
protected Repository openRepository() {
repo.incrementOpen();
return repo;
}
};
leader.setReplicas(createReplicas(leader, repo));
return leader;
}
/**
* Get the collection of replicas for a repository.
* <p>
* The collection of replicas must include the local repository.
*
* @param leader
* the leader driving these replicas.
* @param repo
* repository to get the replicas of.
* @return collection of replicas for the specified repository.
* @throws URISyntaxException
* a configured URI is invalid.
*/
protected List<KetchReplica> createReplicas(KetchLeader leader,
Repository repo) throws URISyntaxException {
List<KetchReplica> replicas = new ArrayList<>();
Config cfg = repo.getConfig();
String localName = getLocalName(cfg);
for (String name : cfg.getSubsections(CONFIG_KEY_REMOTE)) {
if (!hasParticipation(cfg, name)) {
continue;
}
ReplicaConfig kc = ReplicaConfig.newFromConfig(cfg, name);
if (name.equals(localName)) {
replicas.add(new LocalReplica(leader, name, kc));
continue;
}
RemoteConfig rc = new RemoteConfig(cfg, name);
List<URIish> uris = rc.getPushURIs();
if (uris.isEmpty()) {
uris = rc.getURIs();
}
for (URIish uri : uris) {
String n = uris.size() == 1 ? name : uri.getHost();
replicas.add(new RemoteGitReplica(leader, n, uri, kc, rc));
}
}
return replicas;
}
private static boolean hasParticipation(Config cfg, String name) {
return cfg.getString(CONFIG_KEY_REMOTE, name, CONFIG_KEY_TYPE) != null;
}
private static String getLocalName(Config cfg) {
return cfg.getString(CONFIG_SECTION_KETCH, null, CONFIG_KEY_NAME);
}
static class DefaultExecutorHolder {
private static final Logger log = LoggerFactory.getLogger(KetchSystem.class);
static final ScheduledExecutorService I = create();
private static ScheduledExecutorService create() {
int cores = Runtime.getRuntime().availableProcessors();
int threads = Math.max(5, cores);
log.info("Using {} threads", Integer.valueOf(threads)); //$NON-NLS-1$
return Executors.newScheduledThreadPool(
threads,
new ThreadFactory() {
private final AtomicInteger threadCnt = new AtomicInteger();
@Override
public Thread newThread(Runnable r) {
int id = threadCnt.incrementAndGet();
Thread thr = new Thread(r);
thr.setName("KetchExecutor-" + id); //$NON-NLS-1$
return thr;
}
});
}
private DefaultExecutorHolder() {
}
}
/**
* Compute a delay in a {@code min..max} interval with random jitter.
*
* @param last
* amount of delay waited before the last attempt. This is used
* to seed the next delay interval. Should be 0 if there was no
* prior delay.
* @param min
* shortest amount of allowable delay between attempts.
* @param max
* longest amount of allowable delay between attempts.
* @return new amount of delay to wait before the next attempt.
*/
static long delay(long last, long min, long max) {
long r = Math.max(0, last * 3 - min);
if (r > 0) {
int c = (int) Math.min(r + 1, Integer.MAX_VALUE);
r = RNG.nextInt(c);
}
return Math.max(Math.min(min + r, max), min);
}
}

70
org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchText.java

@ -0,0 +1,70 @@
/*
* Copyright (C) 2016, Google Inc.
* and other copyright owners as documented in the project's IP log.
*
* This program and the accompanying materials are made available
* under the terms of the Eclipse Distribution License v1.0 which
* accompanies this distribution, is reproduced below, and is
* available at http://www.eclipse.org/org/documents/edl-v10.php
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
*
* - Neither the name of the Eclipse Foundation, Inc. nor the
* names of its contributors may be used to endorse or promote
* products derived from this software without specific prior
* written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.eclipse.jgit.internal.ketch;
import org.eclipse.jgit.nls.NLS;
import org.eclipse.jgit.nls.TranslationBundle;
/** Translation bundle for the Ketch implementation. */
public class KetchText extends TranslationBundle {
/** @return instance of this translation bundle. */
public static KetchText get() {
return NLS.getBundleFor(KetchText.class);
}
// @formatter:off
/***/ public String accepted;
/***/ public String cannotFetchFromLocalReplica;
/***/ public String failed;
/***/ public String invalidFollowerUri;
/***/ public String leaderFailedToStore;
/***/ public String localReplicaRequired;
/***/ public String mismatchedTxnNamespace;
/***/ public String outsideTxnNamespace;
/***/ public String proposingUpdates;
/***/ public String queuedProposalFailedToApply;
/***/ public String starting;
/***/ public String unsupportedVoterCount;
/***/ public String waitingForQueue;
}

183
org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LagCheck.java

@ -0,0 +1,183 @@
/*
* Copyright (C) 2016, Google Inc.
* and other copyright owners as documented in the project's IP log.
*
* This program and the accompanying materials are made available
* under the terms of the Eclipse Distribution License v1.0 which
* accompanies this distribution, is reproduced below, and is
* available at http://www.eclipse.org/org/documents/edl-v10.php
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
*
* - Neither the name of the Eclipse Foundation, Inc. nor the
* names of its contributors may be used to endorse or promote
* products derived from this software without specific prior
* written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.eclipse.jgit.internal.ketch;
import static org.eclipse.jgit.internal.ketch.KetchReplica.State.AHEAD;
import static org.eclipse.jgit.internal.ketch.KetchReplica.State.DIVERGENT;
import static org.eclipse.jgit.internal.ketch.KetchReplica.State.LAGGING;
import static org.eclipse.jgit.internal.ketch.KetchReplica.State.UNKNOWN;
import static org.eclipse.jgit.lib.Constants.OBJ_COMMIT;
import java.io.IOException;
import java.util.Collections;
import java.util.Map;
import org.eclipse.jgit.errors.MissingObjectException;
import org.eclipse.jgit.lib.AnyObjectId;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.RevWalk;
import org.eclipse.jgit.transport.ReceiveCommand;
/**
* A helper to check if a {@link KetchReplica} is ahead or behind the leader.
*/
class LagCheck implements AutoCloseable {
private final KetchReplica replica;
private final Repository repo;
private RevWalk rw;
private ObjectId remoteId;
LagCheck(KetchReplica replica, Repository repo) {
this.replica = replica;
this.repo = repo;
initRevWalk();
}
private void initRevWalk() {
if (rw != null) {
rw.close();
}
rw = new RevWalk(repo);
rw.setRetainBody(false);
}
public void close() {
if (rw != null) {
rw.close();
rw = null;
}
}
ObjectId getRemoteId() {
return remoteId;
}
KetchReplica.State check(ObjectId acceptId, ReceiveCommand acceptCmd) {
remoteId = acceptId;
if (remoteId == null) {
// Nothing advertised by the replica, value is unknown.
return UNKNOWN;
}
if (AnyObjectId.equals(remoteId, ObjectId.zeroId())) {
// Replica does not have the txnAccepted reference.
return LAGGING;
}
try {
RevCommit remote;
try {
remote = parseRemoteCommit(acceptCmd.getRefName());
} catch (RefGoneException gone) {
// Replica does not have the txnAccepted reference.
return LAGGING;
} catch (MissingObjectException notFound) {
// Local repository does not know this commit so it cannot
// be including the replica's log.
return DIVERGENT;
}
RevCommit head = rw.parseCommit(acceptCmd.getNewId());
if (rw.isMergedInto(remote, head)) {
return LAGGING;
}
// TODO(sop) Check term to see if my leader was deposed.
if (rw.isMergedInto(head, remote)) {
return AHEAD;
} else {
return DIVERGENT;
}
} catch (IOException err) {
KetchReplica.log.error(String.format(
"Cannot compare %s", //$NON-NLS-1$
acceptCmd.getRefName()), err);
return UNKNOWN;
}
}
private RevCommit parseRemoteCommit(String refName)
throws IOException, MissingObjectException, RefGoneException {
try {
return rw.parseCommit(remoteId);
} catch (MissingObjectException notLocal) {
// Fall through and try to acquire the object by fetching it.
}
ReplicaFetchRequest fetch = new ReplicaFetchRequest(
Collections.singleton(refName),
Collections.<ObjectId> emptySet());
try {
replica.blockingFetch(repo, fetch);
} catch (IOException fetchErr) {
KetchReplica.log.error(String.format(
"Cannot fetch %s (%s) from %s", //$NON-NLS-1$
remoteId.abbreviate(8).name(), refName,
replica.describeForLog()), fetchErr);
throw new MissingObjectException(remoteId, OBJ_COMMIT);
}
Map<String, Ref> adv = fetch.getRefs();
if (adv == null) {
throw new MissingObjectException(remoteId, OBJ_COMMIT);
}
Ref ref = adv.get(refName);
if (ref == null || ref.getObjectId() == null) {
throw new RefGoneException();
}
initRevWalk();
remoteId = ref.getObjectId();
return rw.parseCommit(remoteId);
}
private static class RefGoneException extends Exception {
private static final long serialVersionUID = 1L;
}
}

164
org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LeaderSnapshot.java

@ -0,0 +1,164 @@
/*
* Copyright (C) 2016, Google Inc.
* and other copyright owners as documented in the project's IP log.
*
* This program and the accompanying materials are made available
* under the terms of the Eclipse Distribution License v1.0 which
* accompanies this distribution, is reproduced below, and is
* available at http://www.eclipse.org/org/documents/edl-v10.php
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
*
* - Neither the name of the Eclipse Foundation, Inc. nor the
* names of its contributors may be used to endorse or promote
* products derived from this software without specific prior
* written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.eclipse.jgit.internal.ketch;
import static org.eclipse.jgit.internal.ketch.KetchReplica.State.OFFLINE;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import org.eclipse.jgit.annotations.Nullable;
import org.eclipse.jgit.lib.ObjectId;
/** A snapshot of a leader and its view of the world. */
public class LeaderSnapshot {
final List<ReplicaSnapshot> replicas = new ArrayList<>();
KetchLeader.State state;
long term;
LogIndex headIndex;
LogIndex committedIndex;
boolean idle;
LeaderSnapshot() {
}
/** @return unmodifiable view of configured replicas. */
public Collection<ReplicaSnapshot> getReplicas() {
return Collections.unmodifiableList(replicas);
}
/** @return current state of the leader. */
public KetchLeader.State getState() {
return state;
}
/**
* @return {@code true} if the leader is not running a round to reach
* consensus, and has no rounds queued.
*/
public boolean isIdle() {
return idle;
}
/**
* @return term of this leader. Valid only if {@link #getState()} is
* currently {@link KetchLeader.State#LEADER}.
*/
public long getTerm() {
return term;
}
/**
* @return end of the leader's log; null if leader hasn't started up enough
* to begin its own election.
*/
@Nullable
public LogIndex getHead() {
return headIndex;
}
/**
* @return state the leader knows is committed on a majority of participant
* replicas. Null until the leader instance has committed a log
* index within its own term.
*/
@Nullable
public LogIndex getCommitted() {
return committedIndex;
}
@Override
public String toString() {
StringBuilder s = new StringBuilder();
s.append(isIdle() ? "IDLE" : "RUNNING"); //$NON-NLS-1$ //$NON-NLS-2$
s.append(" state ").append(getState()); //$NON-NLS-1$
if (getTerm() > 0) {
s.append(" term ").append(getTerm()); //$NON-NLS-1$
}
s.append('\n');
s.append(String.format(
"%-10s %12s %12s\n", //$NON-NLS-1$
"Replica", "Accepted", "Committed")); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
s.append("------------------------------------\n"); //$NON-NLS-1$
debug(s, "(leader)", getHead(), getCommitted()); //$NON-NLS-1$
s.append('\n');
for (ReplicaSnapshot r : getReplicas()) {
debug(s, r);
s.append('\n');
}
s.append('\n');
return s.toString();
}
private static void debug(StringBuilder b, ReplicaSnapshot s) {
KetchReplica replica = s.getReplica();
debug(b, replica.getName(), s.getAccepted(), s.getCommitted());
b.append(String.format(" %-8s %s", //$NON-NLS-1$
replica.getParticipation(), s.getState()));
if (s.getState() == OFFLINE) {
String err = s.getErrorMessage();
if (err != null) {
b.append(" (").append(err).append(')'); //$NON-NLS-1$
}
}
}
private static void debug(StringBuilder s, String name,
ObjectId accepted, ObjectId committed) {
s.append(String.format(
"%-10s %-12s %-12s", //$NON-NLS-1$
name, str(accepted), str(committed)));
}
static String str(ObjectId c) {
if (c instanceof LogIndex) {
return ((LogIndex) c).describeForLog();
} else if (c != null) {
return c.abbreviate(8).name();
}
return "-"; //$NON-NLS-1$
}
}

229
org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LocalReplica.java

@ -0,0 +1,229 @@
/*
* Copyright (C) 2016, Google Inc.
* and other copyright owners as documented in the project's IP log.
*
* This program and the accompanying materials are made available
* under the terms of the Eclipse Distribution License v1.0 which
* accompanies this distribution, is reproduced below, and is
* available at http://www.eclipse.org/org/documents/edl-v10.php
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
*
* - Neither the name of the Eclipse Foundation, Inc. nor the
* names of its contributors may be used to endorse or promote
* products derived from this software without specific prior
* written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.eclipse.jgit.internal.ketch;
import static org.eclipse.jgit.internal.ketch.KetchReplica.CommitMethod.ALL_REFS;
import static org.eclipse.jgit.internal.ketch.KetchReplica.CommitMethod.TXN_COMMITTED;
import static org.eclipse.jgit.lib.RefDatabase.ALL;
import static org.eclipse.jgit.transport.ReceiveCommand.Result.OK;
import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_OTHER_REASON;
import java.io.IOException;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import org.eclipse.jgit.internal.storage.reftree.RefTreeDatabase;
import org.eclipse.jgit.lib.BatchRefUpdate;
import org.eclipse.jgit.lib.NullProgressMonitor;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.RefDatabase;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.revwalk.RevWalk;
import org.eclipse.jgit.transport.ReceiveCommand;
/** Ketch replica running on the same system as the {@link KetchLeader}. */
public class LocalReplica extends KetchReplica {
/**
* Configure a local replica.
*
* @param leader
* instance this replica follows.
* @param name
* unique-ish name identifying this replica for debugging.
* @param cfg
* how Ketch should treat the local system.
*/
public LocalReplica(KetchLeader leader, String name, ReplicaConfig cfg) {
super(leader, name, cfg);
}
@Override
protected String describeForLog() {
return String.format("%s (leader)", getName()); //$NON-NLS-1$
}
/**
* Initializes local replica by reading accepted and committed references.
* <p>
* Loads accepted and committed references from the reference database of
* the local replica and stores their current ObjectIds in memory.
*
* @param repo
* repository to initialize state from.
* @throws IOException
* cannot read repository state.
*/
void initialize(Repository repo) throws IOException {
RefDatabase refdb = repo.getRefDatabase();
if (refdb instanceof RefTreeDatabase) {
RefTreeDatabase treeDb = (RefTreeDatabase) refdb;
String txnNamespace = getSystem().getTxnNamespace();
if (!txnNamespace.equals(treeDb.getTxnNamespace())) {
throw new IOException(MessageFormat.format(
KetchText.get().mismatchedTxnNamespace,
txnNamespace, treeDb.getTxnNamespace()));
}
refdb = treeDb.getBootstrap();
}
initialize(refdb.exactRef(
getSystem().getTxnAccepted(),
getSystem().getTxnCommitted()));
}
@Override
protected void startPush(final ReplicaPushRequest req) {
getSystem().getExecutor().execute(new Runnable() {
@Override
public void run() {
try (Repository git = getLeader().openRepository()) {
try {
update(git, req);
req.done(git);
} catch (Throwable err) {
req.setException(git, err);
}
} catch (IOException err) {
req.setException(null, err);
}
}
});
}
@Override
protected void blockingFetch(Repository repo, ReplicaFetchRequest req)
throws IOException {
throw new IOException(KetchText.get().cannotFetchFromLocalReplica);
}
private void update(Repository git, ReplicaPushRequest req)
throws IOException {
RefDatabase refdb = git.getRefDatabase();
CommitMethod method = getCommitMethod();
// Local replica probably uses RefTreeDatabase, the request should
// be only for the txnNamespace, so drop to the bootstrap layer.
if (refdb instanceof RefTreeDatabase) {
if (!isOnlyTxnNamespace(req.getCommands())) {
return;
}
refdb = ((RefTreeDatabase) refdb).getBootstrap();
method = TXN_COMMITTED;
}
BatchRefUpdate batch = refdb.newBatchUpdate();
batch.setRefLogIdent(getSystem().newCommitter());
batch.setRefLogMessage("ketch", false); //$NON-NLS-1$
batch.setAllowNonFastForwards(true);
// RefDirectory updates multiple references sequentially.
// Run everything else first, then accepted (if present),
// then committed (if present). This ensures an earlier
// failure will not update these critical references.
ReceiveCommand accepted = null;
ReceiveCommand committed = null;
for (ReceiveCommand cmd : req.getCommands()) {
String name = cmd.getRefName();
if (name.equals(getSystem().getTxnAccepted())) {
accepted = cmd;
} else if (name.equals(getSystem().getTxnCommitted())) {
committed = cmd;
} else {
batch.addCommand(cmd);
}
}
if (committed != null && method == ALL_REFS) {
Map<String, Ref> refs = refdb.getRefs(ALL);
batch.addCommand(prepareCommit(git, refs, committed.getNewId()));
}
if (accepted != null) {
batch.addCommand(accepted);
}
if (committed != null) {
batch.addCommand(committed);
}
try (RevWalk rw = new RevWalk(git)) {
batch.execute(rw, NullProgressMonitor.INSTANCE);
}
// KetchReplica only cares about accepted and committed in
// advertisement. If they failed, store the current values
// back in the ReplicaPushRequest.
List<String> failed = new ArrayList<>(2);
checkFailed(failed, accepted);
checkFailed(failed, committed);
if (!failed.isEmpty()) {
String[] arr = failed.toArray(new String[failed.size()]);
req.setRefs(refdb.exactRef(arr));
}
}
private static void checkFailed(List<String> failed, ReceiveCommand cmd) {
if (cmd != null && cmd.getResult() != OK) {
failed.add(cmd.getRefName());
}
}
private boolean isOnlyTxnNamespace(Collection<ReceiveCommand> cmdList) {
// Be paranoid and reject non txnNamespace names, this
// is a programming error in Ketch that should not occur.
String txnNamespace = getSystem().getTxnNamespace();
for (ReceiveCommand cmd : cmdList) {
if (!cmd.getRefName().startsWith(txnNamespace)) {
cmd.setResult(REJECTED_OTHER_REASON,
MessageFormat.format(
KetchText.get().outsideTxnNamespace,
cmd.getRefName(), txnNamespace));
ReceiveCommand.abort(cmdList);
return false;
}
}
return true;
}
}

119
org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LogIndex.java

@ -0,0 +1,119 @@
/*
* Copyright (C) 2016, Google Inc.
* and other copyright owners as documented in the project's IP log.
*
* This program and the accompanying materials are made available
* under the terms of the Eclipse Distribution License v1.0 which
* accompanies this distribution, is reproduced below, and is
* available at http://www.eclipse.org/org/documents/edl-v10.php
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
*
* - Neither the name of the Eclipse Foundation, Inc. nor the
* names of its contributors may be used to endorse or promote
* products derived from this software without specific prior
* written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.eclipse.jgit.internal.ketch;
import org.eclipse.jgit.lib.AnyObjectId;
import org.eclipse.jgit.lib.ObjectId;
/**
* An ObjectId for a commit extended with incrementing log index.
* <p>
* For any two LogIndex instances, {@code A} is an ancestor of {@code C}
* reachable through parent edges in the graph if {@code A.index < C.index}.
* LogIndex provides a performance optimization for Ketch, the same information
* can be obtained from {@link org.eclipse.jgit.revwalk.RevWalk}.
* <p>
* Index values are only valid within a single {@link KetchLeader} instance
* after it has won an election. By restricting scope to a single leader new
* leaders do not need to traverse the entire history to determine the next
* {@code index} for new proposals. This differs from Raft, where leader
* election uses the log index and the term number to determine which replica
* holds a sufficiently up-to-date log. Since Ketch uses Git objects for storage
* of its replicated log, it keeps the term number as Raft does but uses
* standard Git operations to imply the log index.
* <p>
* {@link Round#runAsync(AnyObjectId)} bumps the index as each new round is
* constructed.
*/
public class LogIndex extends ObjectId {
static LogIndex unknown(AnyObjectId id) {
return new LogIndex(id, 0);
}
private final long index;
private LogIndex(AnyObjectId id, long index) {
super(id);
this.index = index;
}
LogIndex nextIndex(AnyObjectId id) {
return new LogIndex(id, index + 1);
}
/** @return index provided by the current leader instance. */
public long getIndex() {
return index;
}
/**
* Check if this log position committed before another log position.
* <p>
* Only valid for log positions in memory for the current leader.
*
* @param c
* other (more recent) log position.
* @return true if this log position was before {@code c} or equal to c and
* therefore any agreement of {@code c} implies agreement on this
* log position.
*/
boolean isBefore(LogIndex c) {
return index <= c.index;
}
/**
* @return string suitable for debug logging containing the log index and
* abbreviated ObjectId.
*/
@SuppressWarnings("boxing")
public String describeForLog() {
return String.format("%5d/%s", index, abbreviate(6).name()); //$NON-NLS-1$
}
@SuppressWarnings("boxing")
@Override
public String toString() {
return String.format("LogId[%5d/%s]", index, name()); //$NON-NLS-1$
}
}

391
org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/Proposal.java

@ -0,0 +1,391 @@
/*
* Copyright (C) 2016, Google Inc.
* and other copyright owners as documented in the project's IP log.
*
* This program and the accompanying materials are made available
* under the terms of the Eclipse Distribution License v1.0 which
* accompanies this distribution, is reproduced below, and is
* available at http://www.eclipse.org/org/documents/edl-v10.php
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
*
* - Neither the name of the Eclipse Foundation, Inc. nor the
* names of its contributors may be used to endorse or promote
* products derived from this software without specific prior
* written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.eclipse.jgit.internal.ketch;
import static org.eclipse.jgit.internal.ketch.Proposal.State.ABORTED;
import static org.eclipse.jgit.internal.ketch.Proposal.State.EXECUTED;
import static org.eclipse.jgit.internal.ketch.Proposal.State.NEW;
import static org.eclipse.jgit.transport.ReceiveCommand.Result.NOT_ATTEMPTED;
import static org.eclipse.jgit.transport.ReceiveCommand.Result.OK;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import org.eclipse.jgit.annotations.Nullable;
import org.eclipse.jgit.errors.MissingObjectException;
import org.eclipse.jgit.internal.storage.reftree.Command;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.PersonIdent;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.revwalk.RevWalk;
import org.eclipse.jgit.transport.PushCertificate;
import org.eclipse.jgit.transport.ReceiveCommand;
/**
* A proposal to be applied in a Ketch system.
* <p>
* Pushing to a Ketch leader results in the leader making a proposal. The
* proposal includes the list of reference updates. The leader attempts to send
* the proposal to a quorum of replicas by pushing the proposal to a "staging"
* area under the {@code refs/txn/stage/} namespace. If the proposal succeeds
* then the changes are durable and the leader can commit the proposal.
* <p>
* Proposals are executed by {@link KetchLeader#queueProposal(Proposal)}, which
* runs them asynchronously in the background. Proposals are thread-safe futures
* allowing callers to {@link #await()} for results or be notified by callback
* using {@link #addListener(Runnable)}.
*/
public class Proposal {
/** Current state of the proposal. */
public enum State {
/** Proposal has not yet been given to a {@link KetchLeader}. */
NEW(false),
/**
* Proposal was validated and has entered the queue, but a round
* containing this proposal has not started yet.
*/
QUEUED(false),
/** Round containing the proposal has begun and is in progress. */
RUNNING(false),
/**
* Proposal was executed through a round. Individual results from
* {@link Proposal#getCommands()}, {@link Command#getResult()} explain
* the success or failure outcome.
*/
EXECUTED(true),
/** Proposal was aborted and did not reach consensus. */
ABORTED(true);
private final boolean done;
private State(boolean done) {
this.done = done;
}
/** @return true if this is a terminal state. */
public boolean isDone() {
return done;
}
}
private final List<Command> commands;
private PersonIdent author;
private String message;
private PushCertificate pushCert;
private final List<Runnable> listeners = new CopyOnWriteArrayList<>();
private final AtomicReference<State> state = new AtomicReference<>(NEW);
/**
* Create a proposal from a list of Ketch commands.
*
* @param cmds
* prepared list of commands.
*/
public Proposal(List<Command> cmds) {
commands = Collections.unmodifiableList(new ArrayList<>(cmds));
}
/**
* Create a proposal from a collection of received commands.
*
* @param rw
* walker to assist in preparing commands.
* @param cmds
* list of pending commands.
* @throws MissingObjectException
* newId of a command is not found locally.
* @throws IOException
* local objects cannot be accessed.
*/
public Proposal(RevWalk rw, Collection<ReceiveCommand> cmds)
throws MissingObjectException, IOException {
commands = asCommandList(rw, cmds);
}
private static List<Command> asCommandList(RevWalk rw,
Collection<ReceiveCommand> cmds)
throws MissingObjectException, IOException {
List<Command> commands = new ArrayList<>(cmds.size());
for (ReceiveCommand cmd : cmds) {
commands.add(new Command(rw, cmd));
}
return Collections.unmodifiableList(commands);
}
/** @return commands from this proposal. */
public Collection<Command> getCommands() {
return commands;
}
/** @return optional author of the proposal. */
@Nullable
public PersonIdent getAuthor() {
return author;
}
/**
* Set the author for the proposal.
*
* @param who
* optional identity of the author of the proposal.
* @return {@code this}
*/
public Proposal setAuthor(@Nullable PersonIdent who) {
author = who;
return this;
}
/** @return optional message for the commit log of the RefTree. */
@Nullable
public String getMessage() {
return message;
}
/**
* Set the message to appear in the commit log of the RefTree.
*
* @param msg
* message text for the commit.
* @return {@code this}
*/
public Proposal setMessage(@Nullable String msg) {
message = msg != null && !msg.isEmpty() ? msg : null;
return this;
}
/** @return optional certificate signing the references. */
@Nullable
public PushCertificate getPushCertificate() {
return pushCert;
}
/**
* Set the push certificate signing the references.
*
* @param cert
* certificate, may be null.
* @return {@code this}
*/
public Proposal setPushCertificate(@Nullable PushCertificate cert) {
pushCert = cert;
return this;
}
/**
* Add a callback to be invoked when the proposal is done.
* <p>
* A proposal is done when it has entered either {@link State#EXECUTED} or
* {@link State#ABORTED} state. If the proposal is already done
* {@code callback.run()} is immediately invoked on the caller's thread.
*
* @param callback
* method to run after the proposal is done. The callback may be
* run on a Ketch system thread and should be completed quickly.
*/
public void addListener(Runnable callback) {
boolean runNow = false;
synchronized (state) {
if (state.get().isDone()) {
runNow = true;
} else {
listeners.add(callback);
}
}
if (runNow) {
callback.run();
}
}
/** Set command result as OK. */
void success() {
for (Command c : commands) {
if (c.getResult() == NOT_ATTEMPTED) {
c.setResult(OK);
}
}
notifyState(EXECUTED);
}
/** Mark commands as "transaction aborted". */
void abort() {
Command.abort(commands, null);
notifyState(ABORTED);
}
/** @return read the current state of the proposal. */
public State getState() {
return state.get();
}
/**
* @return {@code true} if the proposal was attempted. A true value does not
* mean consensus was reached, only that the proposal was considered
* and will not be making any more progress beyond its current
* state.
*/
public boolean isDone() {
return state.get().isDone();
}
/**
* Wait for the proposal to be attempted and {@link #isDone()} to be true.
*
* @throws InterruptedException
* caller was interrupted before proposal executed.
*/
public void await() throws InterruptedException {
synchronized (state) {
while (!state.get().isDone()) {
state.wait();
}
}
}
/**
* Wait for the proposal to be attempted and {@link #isDone()} to be true.
*
* @param wait
* how long to wait.
* @param unit
* unit describing the wait time.
* @return true if the proposal is done; false if the method timed out.
* @throws InterruptedException
* caller was interrupted before proposal executed.
*/
public boolean await(long wait, TimeUnit unit) throws InterruptedException {
synchronized (state) {
if (state.get().isDone()) {
return true;
}
state.wait(unit.toMillis(wait));
return state.get().isDone();
}
}
/**
* Wait for the proposal to exit a state.
*
* @param notIn
* state the proposal should not be in to return.
* @param wait
* how long to wait.
* @param unit
* unit describing the wait time.
* @return true if the proposal exited the state; false on time out.
* @throws InterruptedException
* caller was interrupted before proposal executed.
*/
public boolean awaitStateChange(State notIn, long wait, TimeUnit unit)
throws InterruptedException {
synchronized (state) {
if (state.get() != notIn) {
return true;
}
state.wait(unit.toMillis(wait));
return state.get() != notIn;
}
}
void notifyState(State s) {
synchronized (state) {
state.set(s);
state.notifyAll();
}
if (s.isDone()) {
for (Runnable callback : listeners) {
callback.run();
}
listeners.clear();
}
}
@Override
public String toString() {
StringBuilder s = new StringBuilder();
s.append("Ketch Proposal {\n"); //$NON-NLS-1$
s.append(" ").append(state.get()).append('\n'); //$NON-NLS-1$
if (author != null) {
s.append(" author ").append(author).append('\n'); //$NON-NLS-1$
}
if (message != null) {
s.append(" message ").append(message).append('\n'); //$NON-NLS-1$
}
for (Command c : commands) {
s.append(" "); //$NON-NLS-1$
format(s, c.getOldRef(), "CREATE"); //$NON-NLS-1$
s.append(' ');
format(s, c.getNewRef(), "DELETE"); //$NON-NLS-1$
s.append(' ').append(c.getRefName());
if (c.getResult() != ReceiveCommand.Result.NOT_ATTEMPTED) {
s.append(' ').append(c.getResult()); // $NON-NLS-1$
}
s.append('\n');
}
s.append('}');
return s.toString();
}
private static void format(StringBuilder s, @Nullable Ref r, String n) {
if (r == null) {
s.append(n);
} else if (r.isSymbolic()) {
s.append(r.getTarget().getName());
} else {
ObjectId id = r.getObjectId();
if (id != null) {
s.append(id.abbreviate(8).name());
}
}
}
}

299
org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ProposalRound.java

@ -0,0 +1,299 @@
/*
* Copyright (C) 2016, Google Inc.
* and other copyright owners as documented in the project's IP log.
*
* This program and the accompanying materials are made available
* under the terms of the Eclipse Distribution License v1.0 which
* accompanies this distribution, is reproduced below, and is
* available at http://www.eclipse.org/org/documents/edl-v10.php
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
*
* - Neither the name of the Eclipse Foundation, Inc. nor the
* names of its contributors may be used to endorse or promote
* products derived from this software without specific prior
* written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.eclipse.jgit.internal.ketch;
import static org.eclipse.jgit.internal.ketch.Proposal.State.RUNNING;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.eclipse.jgit.annotations.Nullable;
import org.eclipse.jgit.internal.storage.reftree.Command;
import org.eclipse.jgit.internal.storage.reftree.RefTree;
import org.eclipse.jgit.lib.CommitBuilder;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectInserter;
import org.eclipse.jgit.lib.PersonIdent;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.RevWalk;
import org.eclipse.jgit.transport.ReceiveCommand;
/** A {@link Round} that aggregates and sends user {@link Proposal}s. */
class ProposalRound extends Round {
private final List<Proposal> todo;
private RefTree queuedTree;
ProposalRound(KetchLeader leader, LogIndex head, List<Proposal> todo,
@Nullable RefTree tree) {
super(leader, head);
this.todo = todo;
if (tree != null && canCombine(todo)) {
this.queuedTree = tree;
} else {
leader.roundHoldsReferenceToRefTree = false;
}
}
private static boolean canCombine(List<Proposal> todo) {
Proposal first = todo.get(0);
for (int i = 1; i < todo.size(); i++) {
if (!canCombine(first, todo.get(i))) {
return false;
}
}
return true;
}
private static boolean canCombine(Proposal a, Proposal b) {
String aMsg = nullToEmpty(a.getMessage());
String bMsg = nullToEmpty(b.getMessage());
return aMsg.equals(bMsg) && canCombine(a.getAuthor(), b.getAuthor());
}
private static String nullToEmpty(@Nullable String str) {
return str != null ? str : ""; //$NON-NLS-1$
}
private static boolean canCombine(@Nullable PersonIdent a,
@Nullable PersonIdent b) {
if (a != null && b != null) {
// Same name and email address. Combine timestamp as the two
// proposals are running concurrently and appear together or
// not at all from the point of view of an outside reader.
return a.getName().equals(b.getName())
&& a.getEmailAddress().equals(b.getEmailAddress());
}
// If a and b are null, both will be the system identity.
return a == null && b == null;
}
void start() throws IOException {
for (Proposal p : todo) {
p.notifyState(RUNNING);
}
try {
ObjectId id;
try (Repository git = leader.openRepository()) {
id = insertProposals(git);
}
runAsync(id);
} catch (NoOp e) {
for (Proposal p : todo) {
p.success();
}
leader.lock.lock();
try {
leader.nextRound();
} finally {
leader.lock.unlock();
}
} catch (IOException e) {
abort();
throw e;
}
}
private ObjectId insertProposals(Repository git)
throws IOException, NoOp {
ObjectId id;
try (ObjectInserter inserter = git.newObjectInserter()) {
// TODO(sop) Process signed push certificates.
if (queuedTree != null) {
id = insertSingleProposal(git, inserter);
} else {
id = insertMultiProposal(git, inserter);
}
stageCommands = makeStageList(git, inserter);
inserter.flush();
}
return id;
}
private ObjectId insertSingleProposal(Repository git,
ObjectInserter inserter) throws IOException, NoOp {
// Fast path: tree is passed in with all proposals applied.
ObjectId treeId = queuedTree.writeTree(inserter);
queuedTree = null;
leader.roundHoldsReferenceToRefTree = false;
if (!ObjectId.zeroId().equals(acceptedOldIndex)) {
try (RevWalk rw = new RevWalk(git)) {
RevCommit c = rw.parseCommit(acceptedOldIndex);
if (treeId.equals(c.getTree())) {
throw new NoOp();
}
}
}
Proposal p = todo.get(0);
CommitBuilder b = new CommitBuilder();
b.setTreeId(treeId);
if (!ObjectId.zeroId().equals(acceptedOldIndex)) {
b.setParentId(acceptedOldIndex);
}
b.setCommitter(leader.getSystem().newCommitter());
b.setAuthor(p.getAuthor() != null ? p.getAuthor() : b.getCommitter());
b.setMessage(message(p));
return inserter.insert(b);
}
private ObjectId insertMultiProposal(Repository git,
ObjectInserter inserter) throws IOException, NoOp {
// The tree was not passed in, or there are multiple proposals
// each needing their own commit. Reset the tree and replay each
// proposal in order as individual commits.
ObjectId lastIndex = acceptedOldIndex;
ObjectId oldTreeId;
RefTree tree;
if (ObjectId.zeroId().equals(lastIndex)) {
oldTreeId = ObjectId.zeroId();
tree = RefTree.newEmptyTree();
} else {
try (RevWalk rw = new RevWalk(git)) {
RevCommit c = rw.parseCommit(lastIndex);
oldTreeId = c.getTree();
tree = RefTree.read(rw.getObjectReader(), c.getTree());
}
}
PersonIdent committer = leader.getSystem().newCommitter();
for (Proposal p : todo) {
if (!tree.apply(p.getCommands())) {
// This should not occur, previously during queuing the
// commands were successfully applied to the pending tree.
// Abort the entire round.
throw new IOException(
KetchText.get().queuedProposalFailedToApply);
}
ObjectId treeId = tree.writeTree(inserter);
if (treeId.equals(oldTreeId)) {
continue;
}
CommitBuilder b = new CommitBuilder();
b.setTreeId(treeId);
if (!ObjectId.zeroId().equals(lastIndex)) {
b.setParentId(lastIndex);
}
b.setAuthor(p.getAuthor() != null ? p.getAuthor() : committer);
b.setCommitter(committer);
b.setMessage(message(p));
lastIndex = inserter.insert(b);
}
if (lastIndex.equals(acceptedOldIndex)) {
throw new NoOp();
}
return lastIndex;
}
private String message(Proposal p) {
StringBuilder m = new StringBuilder();
String msg = p.getMessage();
if (msg != null && !msg.isEmpty()) {
m.append(msg);
while (m.length() < 2 || m.charAt(m.length() - 2) != '\n'
|| m.charAt(m.length() - 1) != '\n') {
m.append('\n');
}
}
m.append(KetchConstants.TERM.getName())
.append(": ") //$NON-NLS-1$
.append(leader.getTerm());
return m.toString();
}
void abort() {
for (Proposal p : todo) {
p.abort();
}
}
void success() {
for (Proposal p : todo) {
p.success();
}
}
private List<ReceiveCommand> makeStageList(Repository git,
ObjectInserter inserter) throws IOException {
// For each branch, collapse consecutive updates to only most recent,
// avoiding sending multiple objects in a rapid fast-forward chain, or
// rewritten content.
Map<String, ObjectId> byRef = new HashMap<>();
for (Proposal p : todo) {
for (Command c : p.getCommands()) {
Ref n = c.getNewRef();
if (n != null && !n.isSymbolic()) {
byRef.put(n.getName(), n.getObjectId());
}
}
}
if (byRef.isEmpty()) {
return Collections.emptyList();
}
Set<ObjectId> newObjs = new HashSet<>(byRef.values());
StageBuilder b = new StageBuilder(
leader.getSystem().getTxnStage(),
acceptedNewIndex);
return b.makeStageList(newObjs, git, inserter);
}
private static class NoOp extends Exception {
private static final long serialVersionUID = 1L;
}
}

316
org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/RemoteGitReplica.java

@ -0,0 +1,316 @@
/*
* Copyright (C) 2016, Google Inc.
* and other copyright owners as documented in the project's IP log.
*
* This program and the accompanying materials are made available
* under the terms of the Eclipse Distribution License v1.0 which
* accompanies this distribution, is reproduced below, and is
* available at http://www.eclipse.org/org/documents/edl-v10.php
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
*
* - Neither the name of the Eclipse Foundation, Inc. nor the
* names of its contributors may be used to endorse or promote
* products derived from this software without specific prior
* written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.eclipse.jgit.internal.ketch;
import static org.eclipse.jgit.internal.ketch.KetchReplica.CommitMethod.ALL_REFS;
import static org.eclipse.jgit.transport.ReceiveCommand.Result.LOCK_FAILURE;
import static org.eclipse.jgit.transport.ReceiveCommand.Result.NOT_ATTEMPTED;
import static org.eclipse.jgit.transport.ReceiveCommand.Result.OK;
import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_NODELETE;
import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_NONFASTFORWARD;
import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_OTHER_REASON;
import static org.eclipse.jgit.lib.Ref.Storage.NETWORK;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.eclipse.jgit.annotations.Nullable;
import org.eclipse.jgit.errors.NotSupportedException;
import org.eclipse.jgit.errors.TransportException;
import org.eclipse.jgit.lib.AnyObjectId;
import org.eclipse.jgit.lib.NullProgressMonitor;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectIdRef;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.transport.FetchConnection;
import org.eclipse.jgit.transport.PushConnection;
import org.eclipse.jgit.transport.ReceiveCommand;
import org.eclipse.jgit.transport.RemoteConfig;
import org.eclipse.jgit.transport.RemoteRefUpdate;
import org.eclipse.jgit.transport.Transport;
import org.eclipse.jgit.transport.URIish;
/**
* Representation of a Git repository on a remote replica system.
* <p>
* {@link KetchLeader} will contact the replica using the Git wire protocol.
* <p>
* The remote replica may be fully Ketch-aware, or a standard Git server.
*/
public class RemoteGitReplica extends KetchReplica {
private final URIish uri;
private final RemoteConfig remoteConfig;
/**
* Configure a new remote.
*
* @param leader
* instance this replica follows.
* @param name
* unique-ish name identifying this remote for debugging.
* @param uri
* URI to connect to the follower's repository.
* @param cfg
* how Ketch should treat the remote system.
* @param rc
* optional remote configuration describing how to contact the
* peer repository.
*/
public RemoteGitReplica(KetchLeader leader, String name, URIish uri,
ReplicaConfig cfg, @Nullable RemoteConfig rc) {
super(leader, name, cfg);
this.uri = uri;
this.remoteConfig = rc;
}
/** @return URI to contact the remote peer repository. */
public URIish getURI() {
return uri;
}
/** @return optional configuration describing how to contact the peer. */
@Nullable
protected RemoteConfig getRemoteConfig() {
return remoteConfig;
}
@Override
protected String describeForLog() {
return String.format("%s @ %s", getName(), getURI()); //$NON-NLS-1$
}
@Override
protected void startPush(final ReplicaPushRequest req) {
getSystem().getExecutor().execute(new Runnable() {
@Override
public void run() {
try (Repository git = getLeader().openRepository()) {
try {
push(git, req);
req.done(git);
} catch (Throwable err) {
req.setException(git, err);
}
} catch (IOException err) {
req.setException(null, err);
}
}
});
}
private void push(Repository repo, ReplicaPushRequest req)
throws NotSupportedException, TransportException, IOException {
Map<String, Ref> adv;
List<RemoteCommand> cmds = asUpdateList(req.getCommands());
try (Transport transport = Transport.open(repo, uri)) {
RemoteConfig rc = getRemoteConfig();
if (rc != null) {
transport.applyConfig(rc);
}
transport.setPushAtomic(true);
adv = push(repo, transport, cmds);
}
for (RemoteCommand c : cmds) {
c.copyStatusToResult();
}
req.setRefs(adv);
}
private Map<String, Ref> push(Repository git, Transport transport,
List<RemoteCommand> cmds) throws IOException {
Map<String, RemoteRefUpdate> updates = asUpdateMap(cmds);
try (PushConnection connection = transport.openPush()) {
Map<String, Ref> adv = connection.getRefsMap();
RemoteRefUpdate accepted = updates.get(getSystem().getTxnAccepted());
if (accepted != null && !isExpectedValue(adv, accepted)) {
abort(cmds);
return adv;
}
RemoteRefUpdate committed = updates.get(getSystem().getTxnCommitted());
if (committed != null && !isExpectedValue(adv, committed)) {
abort(cmds);
return adv;
}
if (committed != null && getCommitMethod() == ALL_REFS) {
prepareCommit(git, cmds, updates, adv,
committed.getNewObjectId());
}
connection.push(NullProgressMonitor.INSTANCE, updates);
return adv;
}
}
private static boolean isExpectedValue(Map<String, Ref> adv,
RemoteRefUpdate u) {
Ref r = adv.get(u.getRemoteName());
if (!AnyObjectId.equals(getId(r), u.getExpectedOldObjectId())) {
((RemoteCommand) u).cmd.setResult(LOCK_FAILURE);
return false;
}
return true;
}
private void prepareCommit(Repository git, List<RemoteCommand> cmds,
Map<String, RemoteRefUpdate> updates, Map<String, Ref> adv,
ObjectId committed) throws IOException {
for (ReceiveCommand cmd : prepareCommit(git, adv, committed)) {
RemoteCommand c = new RemoteCommand(cmd);
cmds.add(c);
updates.put(c.getRemoteName(), c);
}
}
private static List<RemoteCommand> asUpdateList(
Collection<ReceiveCommand> cmds) {
try {
List<RemoteCommand> toPush = new ArrayList<>(cmds.size());
for (ReceiveCommand cmd : cmds) {
toPush.add(new RemoteCommand(cmd));
}
return toPush;
} catch (IOException e) {
// Cannot occur as no IO was required to build the command.
throw new IllegalStateException(e);
}
}
private static Map<String, RemoteRefUpdate> asUpdateMap(
List<RemoteCommand> cmds) {
Map<String, RemoteRefUpdate> m = new LinkedHashMap<>();
for (RemoteCommand cmd : cmds) {
m.put(cmd.getRemoteName(), cmd);
}
return m;
}
private static void abort(List<RemoteCommand> cmds) {
List<ReceiveCommand> tmp = new ArrayList<>(cmds.size());
for (RemoteCommand cmd : cmds) {
tmp.add(cmd.cmd);
}
ReceiveCommand.abort(tmp);
}
protected void blockingFetch(Repository repo, ReplicaFetchRequest req)
throws NotSupportedException, TransportException {
try (Transport transport = Transport.open(repo, uri)) {
RemoteConfig rc = getRemoteConfig();
if (rc != null) {
transport.applyConfig(rc);
}
fetch(transport, req);
}
}
private void fetch(Transport transport, ReplicaFetchRequest req)
throws NotSupportedException, TransportException {
try (FetchConnection conn = transport.openFetch()) {
Map<String, Ref> remoteRefs = conn.getRefsMap();
req.setRefs(remoteRefs);
List<Ref> want = new ArrayList<>();
for (String name : req.getWantRefs()) {
Ref ref = remoteRefs.get(name);
if (ref != null && ref.getObjectId() != null) {
want.add(ref);
}
}
for (ObjectId id : req.getWantObjects()) {
want.add(new ObjectIdRef.Unpeeled(NETWORK, id.name(), id));
}
conn.fetch(NullProgressMonitor.INSTANCE, want,
Collections.<ObjectId> emptySet());
}
}
static class RemoteCommand extends RemoteRefUpdate {
final ReceiveCommand cmd;
RemoteCommand(ReceiveCommand cmd) throws IOException {
super(null, null,
cmd.getNewId(), cmd.getRefName(),
true /* force update */,
null /* no local tracking ref */,
cmd.getOldId());
this.cmd = cmd;
}
void copyStatusToResult() {
if (cmd.getResult() == NOT_ATTEMPTED) {
switch (getStatus()) {
case OK:
case UP_TO_DATE:
case NON_EXISTING:
cmd.setResult(OK);
break;
case REJECTED_NODELETE:
cmd.setResult(REJECTED_NODELETE);
break;
case REJECTED_NONFASTFORWARD:
cmd.setResult(REJECTED_NONFASTFORWARD);
break;
case REJECTED_OTHER_REASON:
cmd.setResult(REJECTED_OTHER_REASON, getMessage());
break;
default:
cmd.setResult(REJECTED_OTHER_REASON, getStatus().name());
break;
}
}
}
}
}

233
org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ReplicaConfig.java

@ -0,0 +1,233 @@
/*
* Copyright (C) 2016, Google Inc.
* and other copyright owners as documented in the project's IP log.
*
* This program and the accompanying materials are made available
* under the terms of the Eclipse Distribution License v1.0 which
* accompanies this distribution, is reproduced below, and is
* available at http://www.eclipse.org/org/documents/edl-v10.php
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
*
* - Neither the name of the Eclipse Foundation, Inc. nor the
* names of its contributors may be used to endorse or promote
* products derived from this software without specific prior
* written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.eclipse.jgit.internal.ketch;
import static java.util.concurrent.TimeUnit.DAYS;
import static java.util.concurrent.TimeUnit.HOURS;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static java.util.concurrent.TimeUnit.MINUTES;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.eclipse.jgit.internal.ketch.KetchConstants.CONFIG_KEY_COMMIT;
import static org.eclipse.jgit.internal.ketch.KetchConstants.CONFIG_KEY_SPEED;
import static org.eclipse.jgit.internal.ketch.KetchConstants.CONFIG_KEY_TYPE;
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_REMOTE;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.eclipse.jgit.internal.ketch.KetchReplica.CommitMethod;
import org.eclipse.jgit.internal.ketch.KetchReplica.CommitSpeed;
import org.eclipse.jgit.internal.ketch.KetchReplica.Participation;
import org.eclipse.jgit.lib.Config;
/** Configures a {@link KetchReplica}. */
public class ReplicaConfig {
/**
* Read a configuration from a config block.
*
* @param cfg
* configuration to read.
* @param name
* of the replica being configured.
* @return replica configuration for {@code name}.
*/
public static ReplicaConfig newFromConfig(Config cfg, String name) {
return new ReplicaConfig().fromConfig(cfg, name);
}
private Participation participation = Participation.FULL;
private CommitMethod commitMethod = CommitMethod.ALL_REFS;
private CommitSpeed commitSpeed = CommitSpeed.BATCHED;
private long minRetry = SECONDS.toMillis(5);
private long maxRetry = MINUTES.toMillis(1);
/** @return participation of the replica in the system. */
public Participation getParticipation() {
return participation;
}
/** @return how Ketch should apply committed changes. */
public CommitMethod getCommitMethod() {
return commitMethod;
}
/** @return how quickly should Ketch commit. */
public CommitSpeed getCommitSpeed() {
return commitSpeed;
}
/**
* Returns the minimum wait delay before retrying a failure.
*
* @param unit
* to get retry delay in.
* @return minimum delay before retrying a failure.
*/
public long getMinRetry(TimeUnit unit) {
return unit.convert(minRetry, MILLISECONDS);
}
/**
* Returns the maximum wait delay before retrying a failure.
*
* @param unit
* to get retry delay in.
* @return maximum delay before retrying a failure.
*/
public long getMaxRetry(TimeUnit unit) {
return unit.convert(maxRetry, MILLISECONDS);
}
/**
* Update the configuration from a config block.
*
* @param cfg
* configuration to read.
* @param name
* of the replica being configured.
* @return {@code this}
*/
public ReplicaConfig fromConfig(Config cfg, String name) {
participation = cfg.getEnum(
CONFIG_KEY_REMOTE, name, CONFIG_KEY_TYPE,
participation);
commitMethod = cfg.getEnum(
CONFIG_KEY_REMOTE, name, CONFIG_KEY_COMMIT,
commitMethod);
commitSpeed = cfg.getEnum(
CONFIG_KEY_REMOTE, name, CONFIG_KEY_SPEED,
commitSpeed);
minRetry = getMillis(cfg, name, "ketch-minRetry", minRetry); //$NON-NLS-1$
maxRetry = getMillis(cfg, name, "ketch-maxRetry", maxRetry); //$NON-NLS-1$
return this;
}
private static long getMillis(Config cfg, String name, String key,
long defaultValue) {
String valStr = cfg.getString(CONFIG_KEY_REMOTE, name, key);
if (valStr == null) {
return defaultValue;
}
valStr = valStr.trim();
if (valStr.isEmpty()) {
return defaultValue;
}
Matcher m = UnitMap.PATTERN.matcher(valStr);
if (!m.matches()) {
return defaultValue;
}
String digits = m.group(1);
String unitName = m.group(2).trim();
TimeUnit unit = UnitMap.UNITS.get(unitName);
if (unit == null) {
return defaultValue;
}
try {
if (digits.indexOf('.') == -1) {
return unit.toMillis(Long.parseLong(digits));
}
double val = Double.parseDouble(digits);
return (long) (val * unit.toMillis(1));
} catch (NumberFormatException nfe) {
return defaultValue;
}
}
static class UnitMap {
static final Pattern PATTERN = Pattern
.compile("^([1-9][0-9]*(?:\\.[0-9]*)?)\\s*(.*)$"); //$NON-NLS-1$
static final Map<String, TimeUnit> UNITS;
static {
Map<String, TimeUnit> m = new HashMap<>();
TimeUnit u = MILLISECONDS;
m.put("", u); //$NON-NLS-1$
m.put("ms", u); //$NON-NLS-1$
m.put("millis", u); //$NON-NLS-1$
m.put("millisecond", u); //$NON-NLS-1$
m.put("milliseconds", u); //$NON-NLS-1$
u = SECONDS;
m.put("s", u); //$NON-NLS-1$
m.put("sec", u); //$NON-NLS-1$
m.put("secs", u); //$NON-NLS-1$
m.put("second", u); //$NON-NLS-1$
m.put("seconds", u); //$NON-NLS-1$
u = MINUTES;
m.put("m", u); //$NON-NLS-1$
m.put("min", u); //$NON-NLS-1$
m.put("mins", u); //$NON-NLS-1$
m.put("minute", u); //$NON-NLS-1$
m.put("minutes", u); //$NON-NLS-1$
u = HOURS;
m.put("h", u); //$NON-NLS-1$
m.put("hr", u); //$NON-NLS-1$
m.put("hrs", u); //$NON-NLS-1$
m.put("hour", u); //$NON-NLS-1$
m.put("hours", u); //$NON-NLS-1$
u = DAYS;
m.put("d", u); //$NON-NLS-1$
m.put("day", u); //$NON-NLS-1$
m.put("days", u); //$NON-NLS-1$
UNITS = Collections.unmodifiableMap(m);
}
private UnitMap() {
}
}
}

96
org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ReplicaFetchRequest.java

@ -0,0 +1,96 @@
/*
* Copyright (C) 2016, Google Inc.
* and other copyright owners as documented in the project's IP log.
*
* This program and the accompanying materials are made available
* under the terms of the Eclipse Distribution License v1.0 which
* accompanies this distribution, is reproduced below, and is
* available at http://www.eclipse.org/org/documents/edl-v10.php
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
*
* - Neither the name of the Eclipse Foundation, Inc. nor the
* names of its contributors may be used to endorse or promote
* products derived from this software without specific prior
* written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.eclipse.jgit.internal.ketch;
import java.util.Map;
import java.util.Set;
import org.eclipse.jgit.annotations.Nullable;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Ref;
/** A fetch request to obtain objects from a replica, and its result. */
public class ReplicaFetchRequest {
private final Set<String> wantRefs;
private final Set<ObjectId> wantObjects;
private Map<String, Ref> refs;
/**
* Construct a new fetch request for a replica.
*
* @param wantRefs
* named references to be fetched.
* @param wantObjects
* specific objects to be fetched.
*/
public ReplicaFetchRequest(Set<String> wantRefs,
Set<ObjectId> wantObjects) {
this.wantRefs = wantRefs;
this.wantObjects = wantObjects;
}
/** @return references to be fetched. */
public Set<String> getWantRefs() {
return wantRefs;
}
/** @return objects to be fetched. */
public Set<ObjectId> getWantObjects() {
return wantObjects;
}
/** @return remote references, usually from the advertisement. */
@Nullable
public Map<String, Ref> getRefs() {
return refs;
}
/**
* @param refs
* references observed from the replica.
*/
public void setRefs(Map<String, Ref> refs) {
this.refs = refs;
}
}

168
org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ReplicaPushRequest.java

@ -0,0 +1,168 @@
/*
* Copyright (C) 2016, Google Inc.
* and other copyright owners as documented in the project's IP log.
*
* This program and the accompanying materials are made available
* under the terms of the Eclipse Distribution License v1.0 which
* accompanies this distribution, is reproduced below, and is
* available at http://www.eclipse.org/org/documents/edl-v10.php
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
*
* - Neither the name of the Eclipse Foundation, Inc. nor the
* names of its contributors may be used to endorse or promote
* products derived from this software without specific prior
* written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.eclipse.jgit.internal.ketch;
import java.util.Collection;
import java.util.Map;
import org.eclipse.jgit.annotations.Nullable;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.transport.ReceiveCommand;
/**
* A push request sending objects to a replica, and its result.
* <p>
* Implementors of {@link KetchReplica} must populate the command result fields,
* {@link #setRefs(Map)}, and call one of
* {@link #setException(Repository, Throwable)} or {@link #done(Repository)} to
* finish processing.
*/
public class ReplicaPushRequest {
private final KetchReplica replica;
private final Collection<ReceiveCommand> commands;
private Map<String, Ref> refs;
private Throwable exception;
private boolean notified;
/**
* Construct a new push request for a replica.
*
* @param replica
* the replica being pushed to.
* @param commands
* commands to be executed.
*/
public ReplicaPushRequest(KetchReplica replica,
Collection<ReceiveCommand> commands) {
this.replica = replica;
this.commands = commands;
}
/** @return commands to be executed, and their results. */
public Collection<ReceiveCommand> getCommands() {
return commands;
}
/** @return remote references, usually from the advertisement. */
@Nullable
public Map<String, Ref> getRefs() {
return refs;
}
/**
* @param refs
* references observed from the replica.
*/
public void setRefs(Map<String, Ref> refs) {
this.refs = refs;
}
/** @return exception thrown, if any. */
@Nullable
public Throwable getException() {
return exception;
}
/**
* Mark the request as crashing with a communication error.
* <p>
* This method may take significant time acquiring the leader lock and
* updating the Ketch state machine with the failure.
*
* @param repo
* local repository reference used by the push attempt.
* @param err
* exception thrown during communication.
*/
public void setException(@Nullable Repository repo, Throwable err) {
if (KetchReplica.log.isErrorEnabled()) {
KetchReplica.log.error(describe("failed"), err); //$NON-NLS-1$
}
if (!notified) {
notified = true;
exception = err;
replica.afterPush(repo, this);
}
}
/**
* Mark the request as completed without exception.
* <p>
* This method may take significant time acquiring the leader lock and
* updating the Ketch state machine with results from this replica.
*
* @param repo
* local repository reference used by the push attempt.
*/
public void done(Repository repo) {
if (KetchReplica.log.isDebugEnabled()) {
KetchReplica.log.debug(describe("completed")); //$NON-NLS-1$
}
if (!notified) {
notified = true;
replica.afterPush(repo, this);
}
}
private String describe(String heading) {
StringBuilder b = new StringBuilder();
b.append("push to "); //$NON-NLS-1$
b.append(replica.describeForLog());
b.append(' ').append(heading).append(":\n"); //$NON-NLS-1$
for (ReceiveCommand cmd : commands) {
b.append(String.format(
" %-12s %-12s %s %s", //$NON-NLS-1$
LeaderSnapshot.str(cmd.getOldId()),
LeaderSnapshot.str(cmd.getNewId()),
cmd.getRefName(),
cmd.getResult()));
if (cmd.getMessage() != null) {
b.append(' ').append(cmd.getMessage());
}
b.append('\n');
}
return b.toString();
}
}

109
org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ReplicaSnapshot.java

@ -0,0 +1,109 @@
/*
* Copyright (C) 2016, Google Inc.
* and other copyright owners as documented in the project's IP log.
*
* This program and the accompanying materials are made available
* under the terms of the Eclipse Distribution License v1.0 which
* accompanies this distribution, is reproduced below, and is
* available at http://www.eclipse.org/org/documents/edl-v10.php
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
*
* - Neither the name of the Eclipse Foundation, Inc. nor the
* names of its contributors may be used to endorse or promote
* products derived from this software without specific prior
* written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.eclipse.jgit.internal.ketch;
import java.util.Date;
import org.eclipse.jgit.annotations.Nullable;
import org.eclipse.jgit.lib.ObjectId;
/**
* A snapshot of a replica.
*
* @see LeaderSnapshot
*/
public class ReplicaSnapshot {
final KetchReplica replica;
ObjectId accepted;
ObjectId committed;
KetchReplica.State state;
String error;
long retryAtMillis;
ReplicaSnapshot(KetchReplica replica) {
this.replica = replica;
}
/** @return the replica this snapshot describes the state of. */
public KetchReplica getReplica() {
return replica;
}
/** @return current state of the replica. */
public KetchReplica.State getState() {
return state;
}
/** @return last known Git commit at {@code refs/txn/accepted}. */
@Nullable
public ObjectId getAccepted() {
return accepted;
}
/** @return last known Git commit at {@code refs/txn/committed}. */
@Nullable
public ObjectId getCommitted() {
return committed;
}
/**
* @return if {@link #getState()} == {@link KetchReplica.State#OFFLINE} an
* optional human-readable message from the transport system
* explaining the failure.
*/
@Nullable
public String getErrorMessage() {
return error;
}
/**
* @return time (usually in the future) when the leader will retry
* communication with the offline or lagging replica; null if no
* retry is scheduled or necessary.
*/
@Nullable
public Date getRetryAt() {
return retryAtMillis > 0 ? new Date(retryAtMillis) : null;
}
}

116
org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/Round.java

@ -0,0 +1,116 @@
/*
* Copyright (C) 2016, Google Inc.
* and other copyright owners as documented in the project's IP log.
*
* This program and the accompanying materials are made available
* under the terms of the Eclipse Distribution License v1.0 which
* accompanies this distribution, is reproduced below, and is
* available at http://www.eclipse.org/org/documents/edl-v10.php
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
*
* - Neither the name of the Eclipse Foundation, Inc. nor the
* names of its contributors may be used to endorse or promote
* products derived from this software without specific prior
* written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.eclipse.jgit.internal.ketch;
import java.io.IOException;
import java.util.List;
import org.eclipse.jgit.lib.AnyObjectId;
import org.eclipse.jgit.transport.ReceiveCommand;
/**
* One round-trip to all replicas proposing a log entry.
* <p>
* In Raft a log entry represents a state transition at a specific index in the
* replicated log. The leader can only append log entries to the log.
* <p>
* In Ketch a log entry is recorded under the {@code refs/txn} namespace. This
* occurs when:
* <ul>
* <li>a replica wants to establish itself as a new leader by proposing a new
* term (see {@link ElectionRound})
* <li>an established leader wants to gain consensus on new {@link Proposal}s
* (see {@link ProposalRound})
* </ul>
*/
abstract class Round {
final KetchLeader leader;
final LogIndex acceptedOldIndex;
LogIndex acceptedNewIndex;
List<ReceiveCommand> stageCommands;
Round(KetchLeader leader, LogIndex head) {
this.leader = leader;
this.acceptedOldIndex = head;
}
/**
* Creates a commit for {@code refs/txn/accepted} and calls
* {@link #runAsync(AnyObjectId)} to begin execution of the round across
* the system.
* <p>
* If references are being updated (such as in a {@link ProposalRound}) the
* RefTree may be modified.
* <p>
* Invoked without {@link KetchLeader#lock} to build objects.
*
* @throws IOException
* the round cannot build new objects within the leader's
* repository. The leader may be unable to execute.
*/
abstract void start() throws IOException;
/**
* Asynchronously distribute the round's new value for
* {@code refs/txn/accepted} to all replicas.
* <p>
* Invoked by {@link #start()} after new commits have been created for the
* log. The method passes {@code newId} to {@link KetchLeader} to be
* distributed to all known replicas.
*
* @param newId
* new value for {@code refs/txn/accepted}.
*/
void runAsync(AnyObjectId newId) {
acceptedNewIndex = acceptedOldIndex.nextIndex(newId);
leader.runAsync(this);
}
/**
* Notify the round it was accepted by a majority of the system.
* <p>
* Invoked by the leader with {@link KetchLeader#lock} held by the caller.
*/
abstract void success();
}

270
org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/StageBuilder.java

@ -0,0 +1,270 @@
/*
* Copyright (C) 2016, Google Inc.
* and other copyright owners as documented in the project's IP log.
*
* This program and the accompanying materials are made available
* under the terms of the Eclipse Distribution License v1.0 which
* accompanies this distribution, is reproduced below, and is
* available at http://www.eclipse.org/org/documents/edl-v10.php
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
*
* - Neither the name of the Eclipse Foundation, Inc. nor the
* names of its contributors may be used to endorse or promote
* products derived from this software without specific prior
* written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.eclipse.jgit.internal.ketch;
import static org.eclipse.jgit.lib.FileMode.TYPE_GITLINK;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.eclipse.jgit.annotations.Nullable;
import org.eclipse.jgit.lib.AnyObjectId;
import org.eclipse.jgit.lib.CommitBuilder;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectInserter;
import org.eclipse.jgit.lib.PersonIdent;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.RevObject;
import org.eclipse.jgit.revwalk.RevWalk;
import org.eclipse.jgit.transport.ReceiveCommand;
import org.eclipse.jgit.treewalk.EmptyTreeIterator;
import org.eclipse.jgit.treewalk.TreeWalk;
import org.eclipse.jgit.treewalk.filter.TreeFilter;
/** Constructs a set of commands to stage content during a proposal. */
public class StageBuilder {
/**
* Acceptable number of references to send in a single stage transaction.
* <p>
* If the number of unique objects exceeds this amount the builder will
* attempt to decrease the reference count by chaining commits..
*/
private static final int SMALL_BATCH_SIZE = 5;
/**
* Acceptable number of commits to chain together using parent pointers.
* <p>
* When staging many unique commits the {@link StageBuilder} batches
* together unrelated commits as parents of a temporary commit. After the
* proposal completes the temporary commit is discarded and can be garbage
* collected by all replicas.
*/
private static final int TEMP_PARENT_BATCH_SIZE = 128;
private static final byte[] PEEL = { ' ', '^' };
private final String txnStage;
private final String txnId;
/**
* Construct a stage builder for a transaction.
*
* @param txnStageNamespace
* namespace for transaction references to build
* {@code "txnStageNamespace/txnId.n"} style names.
* @param txnId
* identifier used to name temporary staging refs.
*/
public StageBuilder(String txnStageNamespace, ObjectId txnId) {
this.txnStage = txnStageNamespace;
this.txnId = txnId.name();
}
/**
* Compare two RefTrees and return commands to stage new objects.
* <p>
* This method ignores the lineage between the two RefTrees and does a
* straight diff on the two trees. New objects will be staged. The diff
* strategy is useful to catch-up a lagging replica, without sending every
* intermediate step. This may mean the replica does not have the same
* object set as other replicas if there are rewinds or branch deletes.
*
* @param git
* source repository to read {@code oldTree} and {@code newTree}
* from.
* @param oldTree
* accepted RefTree on the replica ({@code refs/txn/accepted}).
* Use {@link ObjectId#zeroId()} if the remote does not have any
* ref tree, e.g. a new replica catching up.
* @param newTree
* RefTree being sent to the replica. The trees will be compared.
* @return list of commands to create {@code "refs/txn/stage/..."}
* references on replicas anchoring new objects into the repository
* while a transaction gains consensus.
* @throws IOException
* {@code git} cannot be accessed to compare {@code oldTree} and
* {@code newTree} to build the object set.
*/
public List<ReceiveCommand> makeStageList(Repository git, ObjectId oldTree,
ObjectId newTree) throws IOException {
try (RevWalk rw = new RevWalk(git);
TreeWalk tw = new TreeWalk(rw.getObjectReader());
ObjectInserter ins = git.newObjectInserter()) {
if (AnyObjectId.equals(oldTree, ObjectId.zeroId())) {
tw.addTree(new EmptyTreeIterator());
} else {
tw.addTree(rw.parseTree(oldTree));
}
tw.addTree(rw.parseTree(newTree));
tw.setFilter(TreeFilter.ANY_DIFF);
tw.setRecursive(true);
Set<ObjectId> newObjs = new HashSet<>();
while (tw.next()) {
if (tw.getRawMode(1) == TYPE_GITLINK
&& !tw.isPathSuffix(PEEL, 2)) {
newObjs.add(tw.getObjectId(1));
}
}
List<ReceiveCommand> cmds = makeStageList(newObjs, git, ins);
ins.flush();
return cmds;
}
}
/**
* Construct a set of commands to stage objects on a replica.
*
* @param newObjs
* objects to send to a replica.
* @param git
* local repository to read source objects from. Required to
* perform minification of {@code newObjs}.
* @param inserter
* inserter to write temporary commit objects during minification
* if many new branches are created by {@code newObjs}.
* @return list of commands to create {@code "refs/txn/stage/..."}
* references on replicas anchoring {@code newObjs} into the
* repository while a transaction gains consensus.
* @throws IOException
* {@code git} cannot be accessed to perform minification of
* {@code newObjs}.
*/
public List<ReceiveCommand> makeStageList(Set<ObjectId> newObjs,
@Nullable Repository git, @Nullable ObjectInserter inserter)
throws IOException {
if (git == null || newObjs.size() <= SMALL_BATCH_SIZE) {
// Without a source repository can only construct unique set.
List<ReceiveCommand> cmds = new ArrayList<>(newObjs.size());
for (ObjectId id : newObjs) {
stage(cmds, id);
}
return cmds;
}
List<ReceiveCommand> cmds = new ArrayList<>();
List<RevCommit> commits = new ArrayList<>();
reduceObjects(cmds, commits, git, newObjs);
if (inserter == null || commits.size() <= 1
|| (cmds.size() + commits.size()) <= SMALL_BATCH_SIZE) {
// Without an inserter to aggregate commits, or for a small set of
// commits just send one stage ref per commit.
for (RevCommit c : commits) {
stage(cmds, c.copy());
}
return cmds;
}
// 'commits' is sorted most recent to least recent commit.
// Group batches of commits and build a chain.
// TODO(sop) Cluster by restricted graphs to support filtering.
ObjectId tip = null;
for (int end = commits.size(); end > 0;) {
int start = Math.max(0, end - TEMP_PARENT_BATCH_SIZE);
List<RevCommit> batch = commits.subList(start, end);
List<ObjectId> parents = new ArrayList<>(1 + batch.size());
if (tip != null) {
parents.add(tip);
}
parents.addAll(batch);
CommitBuilder b = new CommitBuilder();
b.setTreeId(batch.get(0).getTree());
b.setParentIds(parents);
b.setAuthor(tmpAuthor(batch));
b.setCommitter(b.getAuthor());
tip = inserter.insert(b);
end = start;
}
stage(cmds, tip);
return cmds;
}
private static PersonIdent tmpAuthor(List<RevCommit> commits) {
// Construct a predictable author using most recent commit time.
int t = 0;
for (int i = 0; i < commits.size();) {
t = Math.max(t, commits.get(i).getCommitTime());
}
String name = "Ketch Stage"; //$NON-NLS-1$
String email = "tmp@tmp"; //$NON-NLS-1$
return new PersonIdent(name, email, t * 1000L, 0);
}
private void reduceObjects(List<ReceiveCommand> cmds,
List<RevCommit> commits, Repository git,
Set<ObjectId> newObjs) throws IOException {
try (RevWalk rw = new RevWalk(git)) {
rw.setRetainBody(false);
for (ObjectId id : newObjs) {
RevObject obj = rw.parseAny(id);
if (obj instanceof RevCommit) {
rw.markStart((RevCommit) obj);
} else {
stage(cmds, id);
}
}
for (RevCommit c; (c = rw.next()) != null;) {
commits.add(c);
rw.markUninteresting(c);
}
}
}
private void stage(List<ReceiveCommand> cmds, ObjectId id) {
int estLen = txnStage.length() + txnId.length() + 5;
StringBuilder n = new StringBuilder(estLen);
n.append(txnStage).append(txnId).append('.');
n.append(Integer.toHexString(cmds.size()));
cmds.add(new ReceiveCommand(ObjectId.zeroId(), id, n.toString()));
}
}

4
org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/package-info.java

@ -0,0 +1,4 @@
/**
* Distributed consensus system built on Git.
*/
package org.eclipse.jgit.internal.ketch;

10
org.eclipse.jgit/src/org/eclipse/jgit/transport/Daemon.java

@ -255,6 +255,16 @@ public class Daemon {
uploadPackFactory = (UploadPackFactory<DaemonClient>) UploadPackFactory.DISABLED; uploadPackFactory = (UploadPackFactory<DaemonClient>) UploadPackFactory.DISABLED;
} }
/**
* Get the factory used to construct per-request ReceivePack.
*
* @return the factory.
* @since 4.2
*/
public ReceivePackFactory<DaemonClient> getReceivePackFactory() {
return receivePackFactory;
}
/** /**
* Set the factory to construct and configure per-request ReceivePack. * Set the factory to construct and configure per-request ReceivePack.
* *

Loading…
Cancel
Save