Browse Source

Merge changes I7cdb563b,I7f60ae68,I7bd1e769,I92683805,I0e51a8e6

* changes:
  UploadPack: Fix races in smart HTTP negotiation
  PackWriter: Export more statistics
  Do not requeue state vector in stateless RPC fetch
  Wrap excessively long line in BasePackFetchConnection
  Fix smart HTTP client stream alignment errors
stable-1.2
Shawn O. Pearce 13 years ago committed by Code Review
parent
commit
cc03e27093
  1. 28
      org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/UnionInputStreamTest.java
  2. 24
      org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java
  3. 8
      org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackFetchConnection.java
  4. 5
      org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java
  5. 118
      org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java
  6. 7
      org.eclipse.jgit/src/org/eclipse/jgit/util/io/UnionInputStream.java

28
org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/UnionInputStreamTest.java

@ -106,13 +106,21 @@ public class UnionInputStreamTest {
u.add(new ByteArrayInputStream(new byte[] { 4, 5 }));
final byte[] r = new byte[5];
assertEquals(5, u.read(r, 0, 5));
assertTrue(Arrays.equals(new byte[] { 1, 0, 2, 3, 4 }, r));
assertEquals(3, u.read(r, 0, 5));
assertTrue(Arrays.equals(new byte[] { 1, 0, 2, }, slice(r, 3)));
assertEquals(1, u.read(r, 0, 5));
assertEquals(5, r[0]);
assertEquals(3, r[0]);
assertEquals(2, u.read(r, 0, 5));
assertTrue(Arrays.equals(new byte[] { 4, 5, }, slice(r, 2)));
assertEquals(-1, u.read(r, 0, 5));
}
private static byte[] slice(byte[] in, int len) {
byte[] r = new byte[len];
System.arraycopy(in, 0, r, 0, len);
return r;
}
@Test
public void testArrayConstructor() throws IOException {
final UnionInputStream u = new UnionInputStream(
@ -121,10 +129,12 @@ public class UnionInputStreamTest {
new ByteArrayInputStream(new byte[] { 4, 5 }));
final byte[] r = new byte[5];
assertEquals(5, u.read(r, 0, 5));
assertTrue(Arrays.equals(new byte[] { 1, 0, 2, 3, 4 }, r));
assertEquals(3, u.read(r, 0, 5));
assertTrue(Arrays.equals(new byte[] { 1, 0, 2, }, slice(r, 3)));
assertEquals(1, u.read(r, 0, 5));
assertEquals(5, r[0]);
assertEquals(3, r[0]);
assertEquals(2, u.read(r, 0, 5));
assertTrue(Arrays.equals(new byte[] { 4, 5, }, slice(r, 2)));
assertEquals(-1, u.read(r, 0, 5));
}
@ -143,9 +153,9 @@ public class UnionInputStreamTest {
u.add(new ByteArrayInputStream(new byte[] { 3 }));
u.add(new ByteArrayInputStream(new byte[] { 4, 5 }));
assertEquals(0, u.skip(0));
assertEquals(4, u.skip(4));
assertEquals(4, u.read());
assertEquals(1, u.skip(5));
assertEquals(3, u.skip(3));
assertEquals(3, u.read());
assertEquals(2, u.skip(5));
assertEquals(0, u.skip(5));
assertEquals(-1, u.read());

24
org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java

@ -714,6 +714,7 @@ public class PackWriter {
if (!cachedPacks.isEmpty())
throw new IOException(JGitText.get().cachedPacksPreventsIndexCreation);
long writeStart = System.currentTimeMillis();
final List<ObjectToPack> list = sortByName();
final PackIndexWriter iw;
int indexVersion = config.getIndexVersion();
@ -722,6 +723,7 @@ public class PackWriter {
else
iw = PackIndexWriter.createVersion(indexStream, indexVersion);
iw.write(list, packcsum);
stats.timeWriting += System.currentTimeMillis() - writeStart;
}
private List<ObjectToPack> sortByName() {
@ -828,6 +830,7 @@ public class PackWriter {
stats.timeWriting = System.currentTimeMillis() - writeStart;
stats.totalBytes = out.length();
stats.reusedPacks = Collections.unmodifiableList(cachedPacks);
stats.depth = depth;
for (Statistics.ObjectType typeStat : stats.objectTypes) {
if (typeStat == null)
@ -1870,6 +1873,8 @@ public class PackWriter {
Collection<CachedPack> reusedPacks;
int depth;
int deltaSearchNonEdgeObjects;
int deltasFound;
@ -2012,6 +2017,16 @@ public class PackWriter {
return objectTypes[typeCode];
}
/** @return true if the resulting pack file was a shallow pack. */
public boolean isShallow() {
return depth > 0;
}
/** @return depth (in commits) the pack includes if shallow. */
public int getDepth() {
return depth;
}
/**
* @return time in milliseconds spent enumerating the objects that need
* to be included in the output. This time includes any restarts
@ -2060,6 +2075,15 @@ public class PackWriter {
return timeWriting;
}
/** @return total time spent processing this pack. */
public long getTimeTotal() {
return timeCounting
+ timeSearchingForReuse
+ timeSearchingForSizes
+ timeCompressing
+ timeWriting;
}
/**
* @return get the average output speed in terms of bytes-per-second.
* {@code getTotalBytes() / (getTimeWriting() / 1000.0)}.

8
org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackFetchConnection.java

@ -433,7 +433,9 @@ public abstract class BasePackFetchConnection extends BasePackConnection
// ACK status to tell us common objects for reuse in future
// requests. If its not enabled, we can't talk to the peer.
//
throw new PackProtocolException(uri, MessageFormat.format(JGitText.get().statelessRPCRequiresOptionToBeEnabled, OPTION_MULTI_ACK_DETAILED));
throw new PackProtocolException(uri, MessageFormat.format(
JGitText.get().statelessRPCRequiresOptionToBeEnabled,
OPTION_MULTI_ACK_DETAILED));
}
return line.toString();
@ -453,7 +455,7 @@ public abstract class BasePackFetchConnection extends BasePackConnection
state.writeTo(out, null);
negotiateBegin();
SEND_HAVES: while (!receivedReady) {
SEND_HAVES: for (;;) {
final RevCommit c = walk.next();
if (c == null)
break SEND_HAVES;
@ -528,6 +530,8 @@ public abstract class BasePackFetchConnection extends BasePackConnection
throw new CancelledException();
}
if (noDone & receivedReady)
break SEND_HAVES;
if (statelessRPC)
state.writeTo(out, null);

5
org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java

@ -851,11 +851,6 @@ public class TransportHttp extends HttpTransport implements WalkTransport,
}
class HttpExecuteStream extends InputStream {
public int available() throws IOException {
execute();
return 0;
}
public int read() throws IOException {
execute();
return -1;

118
org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java

@ -49,6 +49,7 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@ -107,6 +108,16 @@ public class UploadPack {
static final String OPTION_SHALLOW = BasePackFetchConnection.OPTION_SHALLOW;
/** Policy the server uses to validate client requests */
public static enum RequestPolicy {
/** Client may only ask for objects the server advertised a reference for. */
ADVERTISED,
/** Client may ask for any commit reachable from a reference. */
REACHABLE_COMMIT,
/** Client may ask for any SHA-1 in the repository. */
ANY;
}
/** Database we read the objects from. */
private final Repository db;
@ -198,6 +209,8 @@ public class UploadPack {
private final RevFlagSet SAVE;
private RequestPolicy requestPolicy = RequestPolicy.ADVERTISED;
private MultiAck multiAck = MultiAck.OFF;
private boolean noDone;
@ -243,12 +256,22 @@ public class UploadPack {
/** @return all refs which were advertised to the client. */
public final Map<String, Ref> getAdvertisedRefs() {
if (refs == null) {
refs = refFilter.filter(db.getAllRefs());
}
if (refs == null)
setAdvertisedRefs(db.getAllRefs());
return refs;
}
/**
* @param allRefs
* explicit set of references to claim as advertised by this
* UploadPack instance. This overrides any references that
* may exist in the source repository. The map is passed
* to the configured {@link #getRefFilter()}.
*/
public void setAdvertisedRefs(Map<String, Ref> allRefs) {
refs = refFilter.filter(allRefs);
}
/** @return timeout (in seconds) before aborting an IO operation. */
public int getTimeout() {
return timeout;
@ -285,6 +308,26 @@ public class UploadPack {
*/
public void setBiDirectionalPipe(final boolean twoWay) {
biDirectionalPipe = twoWay;
if (!biDirectionalPipe && requestPolicy == RequestPolicy.ADVERTISED)
requestPolicy = RequestPolicy.REACHABLE_COMMIT;
}
/** @return policy used by the service to validate client requests. */
public RequestPolicy getRequestPolicy() {
return requestPolicy;
}
/**
* @param policy
* the policy used to enforce validation of a client's want list.
* By default the policy is {@link RequestPolicy#ADVERTISED},
* which is the Git default requiring clients to only ask for an
* object that a reference directly points to. This may be relaxed
* to {@link RequestPolicy#REACHABLE_COMMIT} when callers
* have {@link #setBiDirectionalPipe(boolean)} set to false.
*/
public void setRequestPolicy(RequestPolicy policy) {
requestPolicy = policy != null ? policy : RequestPolicy.ADVERTISED;
}
/** @return the filter used while advertising the refs to the client */
@ -406,6 +449,8 @@ public class UploadPack {
private void service() throws IOException {
if (biDirectionalPipe)
sendAdvertisedRefs(new PacketLineOutRefAdvertiser(pckOut));
else if (requestPolicy == RequestPolicy.ANY)
advertised = Collections.emptySet();
else {
advertised = new HashSet<ObjectId>();
for (Ref ref : getAdvertisedRefs().values()) {
@ -659,6 +704,7 @@ public class UploadPack {
needMissing = true;
}
Set<RevObject> notAdvertisedWants = null;
int haveCnt = 0;
AsyncRevObjectQueue q = walk.parseAny(toParse, needMissing);
try {
@ -682,10 +728,10 @@ public class UploadPack {
// list wasn't parsed earlier, and was done in this batch.
//
if (wantIds.remove(obj)) {
if (!advertised.contains(obj)) {
String msg = MessageFormat.format(
JGitText.get().wantNotValid, obj.name());
throw new PackProtocolException(msg);
if (!advertised.contains(obj) && requestPolicy != RequestPolicy.ANY) {
if (notAdvertisedWants == null)
notAdvertisedWants = new HashSet<RevObject>();
notAdvertisedWants.add(obj);
}
if (!obj.has(WANT)) {
@ -745,6 +791,26 @@ public class UploadPack {
} finally {
q.release();
}
// If the client asked for non advertised object, check our policy.
if (notAdvertisedWants != null && !notAdvertisedWants.isEmpty()) {
switch (requestPolicy) {
case ADVERTISED:
default:
throw new PackProtocolException(MessageFormat.format(
JGitText.get().wantNotValid,
notAdvertisedWants.iterator().next().name()));
case REACHABLE_COMMIT:
checkNotAdvertisedWants(notAdvertisedWants);
break;
case ANY:
// Allow whatever was asked for.
break;
}
}
int missCnt = peerHas.size() - haveCnt;
// If we don't have one of the objects but we're also willing to
@ -787,6 +853,40 @@ public class UploadPack {
return last;
}
private void checkNotAdvertisedWants(Set<RevObject> notAdvertisedWants)
throws MissingObjectException, IncorrectObjectTypeException, IOException {
// Walk the requested commits back to the advertised commits.
// If any commit exists, a branch was deleted or rewound and
// the repository owner no longer exports that requested item.
// If the requested commit is merged into an advertised branch
// it will be marked UNINTERESTING and no commits return.
for (RevObject o : notAdvertisedWants) {
if (!(o instanceof RevCommit)) {
throw new PackProtocolException(MessageFormat.format(
JGitText.get().wantNotValid,
notAdvertisedWants.iterator().next().name()));
}
walk.markStart((RevCommit) o);
}
for (ObjectId id : advertised) {
try {
walk.markUninteresting(walk.parseCommit(id));
} catch (IncorrectObjectTypeException notCommit) {
continue;
}
}
RevCommit bad = walk.next();
if (bad != null) {
throw new PackProtocolException(MessageFormat.format(
JGitText.get().wantNotValid,
bad.name()));
}
walk.reset();
}
private void addCommonBase(final RevObject o) {
if (!o.has(COMMON)) {
o.add(COMMON);
@ -941,7 +1041,7 @@ public class UploadPack {
pw.setThin(options.contains(OPTION_THIN_PACK));
pw.setReuseValidatingObjects(false);
if (commonBase.isEmpty()) {
if (commonBase.isEmpty() && refs != null) {
Set<ObjectId> tagTargets = new HashSet<ObjectId>();
for (Ref ref : refs.values()) {
if (ref.getPeeledObjectId() != null)
@ -968,7 +1068,7 @@ public class UploadPack {
rw = ow;
}
if (options.contains(OPTION_INCLUDE_TAG)) {
if (options.contains(OPTION_INCLUDE_TAG) && refs != null) {
for (Ref ref : refs.values()) {
ObjectId objectId = ref.getObjectId();

7
org.eclipse.jgit/src/org/eclipse/jgit/util/io/UnionInputStream.java

@ -148,8 +148,11 @@ public class UnionInputStream extends InputStream {
len -= n;
} else if (in == EOF)
return 0 < cnt ? cnt : -1;
else
else {
pop();
if (0 < cnt)
break;
}
}
return cnt;
}
@ -180,6 +183,8 @@ public class UnionInputStream extends InputStream {
final int r = in.read();
if (r < 0) {
pop();
if (0 < cnt)
break;
} else {
cnt += 1;
len -= 1;

Loading…
Cancel
Save