@ -1,4 +1,5 @@
/ * / *
* Copyright ( C ) 2009 - 2010 , Google Inc .
* Copyright ( C ) 2008 , Shawn O . Pearce < spearce @spearce.org > * Copyright ( C ) 2008 , Shawn O . Pearce < spearce @spearce.org >
* and other copyright owners as documented in the project ' s IP log . * and other copyright owners as documented in the project ' s IP log .
* *
@ -43,10 +44,22 @@
package org.eclipse.jgit.transport ; package org.eclipse.jgit.transport ;
import static org.eclipse.jgit.util.HttpSupport.ENCODING_GZIP ;
import static org.eclipse.jgit.util.HttpSupport.HDR_ACCEPT ;
import static org.eclipse.jgit.util.HttpSupport.HDR_ACCEPT_ENCODING ;
import static org.eclipse.jgit.util.HttpSupport.HDR_CONTENT_ENCODING ;
import static org.eclipse.jgit.util.HttpSupport.HDR_CONTENT_TYPE ;
import static org.eclipse.jgit.util.HttpSupport.HDR_PRAGMA ;
import static org.eclipse.jgit.util.HttpSupport.HDR_USER_AGENT ;
import static org.eclipse.jgit.util.HttpSupport.METHOD_POST ;
import java.io.BufferedReader ; import java.io.BufferedReader ;
import java.io.ByteArrayInputStream ;
import java.io.FileNotFoundException ; import java.io.FileNotFoundException ;
import java.io.IOException ; import java.io.IOException ;
import java.io.InputStream ; import java.io.InputStream ;
import java.io.InputStreamReader ;
import java.io.OutputStream ;
import java.net.HttpURLConnection ; import java.net.HttpURLConnection ;
import java.net.MalformedURLException ; import java.net.MalformedURLException ;
import java.net.Proxy ; import java.net.Proxy ;
@ -55,27 +68,53 @@ import java.net.URL;
import java.util.ArrayList ; import java.util.ArrayList ;
import java.util.Collection ; import java.util.Collection ;
import java.util.Map ; import java.util.Map ;
import java.util.Set ;
import java.util.TreeMap ; import java.util.TreeMap ;
import java.util.zip.GZIPInputStream ;
import java.util.zip.GZIPOutputStream ;
import org.eclipse.jgit.errors.NoRemoteRepositoryException ;
import org.eclipse.jgit.errors.NotSupportedException ; import org.eclipse.jgit.errors.NotSupportedException ;
import org.eclipse.jgit.errors.PackProtocolException ; import org.eclipse.jgit.errors.PackProtocolException ;
import org.eclipse.jgit.errors.TransportException ; import org.eclipse.jgit.errors.TransportException ;
import org.eclipse.jgit.lib.Config ;
import org.eclipse.jgit.lib.Constants ;
import org.eclipse.jgit.lib.ObjectId ; import org.eclipse.jgit.lib.ObjectId ;
import org.eclipse.jgit.lib.ProgressMonitor ;
import org.eclipse.jgit.lib.Ref ; import org.eclipse.jgit.lib.Ref ;
import org.eclipse.jgit.lib.Repository ; import org.eclipse.jgit.lib.Repository ;
import org.eclipse.jgit.lib.Config.SectionParser ;
import org.eclipse.jgit.util.HttpSupport ; import org.eclipse.jgit.util.HttpSupport ;
import org.eclipse.jgit.util.IO ;
import org.eclipse.jgit.util.RawParseUtils ;
import org.eclipse.jgit.util.TemporaryBuffer ;
import org.eclipse.jgit.util.io.DisabledOutputStream ;
import org.eclipse.jgit.util.io.UnionInputStream ;
/ * * / * *
* Transport over the non - Git aware HTTP and FTP protocol . * Transport over HTTP and FTP protocols .
* < p >
* If the transport is using HTTP and the remote HTTP service is Git - aware
* ( speaks the "smart-http protocol" ) this client will automatically take
* advantage of the additional Git - specific HTTP extensions . If the remote
* service does not support these extensions , the client will degrade to direct
* file fetching .
* < p > * < p >
* The HTTP transport does not require any specialized Git support on the remote * If the remote ( server side ) repository does not have the specialized Git
* ( server side ) repository . Object files are retrieved directly through * support , object files are retrieved directly through standard HTTP GET ( or
* standard HTTP GET requests , making it easy to serve a Git repository through * binary FTP GET ) requests . This make it easy to serve a Git repository through
* a standard web host provider that does not offer specific support for Git . * a standard web host provider that does not offer specific support for Git .
* *
* @see WalkFetchConnection * @see WalkFetchConnection
* / * /
public class TransportHttp extends HttpTransport implements WalkTransport { public class TransportHttp extends HttpTransport implements WalkTransport ,
PackTransport {
private static final String SVC_UPLOAD_PACK = "git-upload-pack" ;
private static final String SVC_RECEIVE_PACK = "git-receive-pack" ;
private static final String userAgent = computeUserAgent ( ) ;
static boolean canHandle ( final URIish uri ) { static boolean canHandle ( final URIish uri ) {
if ( ! uri . isRemote ( ) ) if ( ! uri . isRemote ( ) )
return false ; return false ;
@ -83,10 +122,37 @@ public class TransportHttp extends HttpTransport implements WalkTransport {
return "http" . equals ( s ) | | "https" . equals ( s ) | | "ftp" . equals ( s ) ; return "http" . equals ( s ) | | "https" . equals ( s ) | | "ftp" . equals ( s ) ;
} }
private static String computeUserAgent ( ) {
String version ;
final Package pkg = TransportHttp . class . getPackage ( ) ;
if ( pkg ! = null & & pkg . getImplementationVersion ( ) ! = null ) {
version = pkg . getImplementationVersion ( ) ;
} else {
version = "unknown" ; //$NON-NLS-1$
}
return "JGit/" + version ; //$NON-NLS-1$
}
private static final Config . SectionParser < HttpConfig > HTTP_KEY = new SectionParser < HttpConfig > ( ) {
public HttpConfig parse ( final Config cfg ) {
return new HttpConfig ( cfg ) ;
}
} ;
private static class HttpConfig {
final int postBuffer ;
HttpConfig ( final Config rc ) {
postBuffer = rc . getInt ( "http" , "postbuffer" , 1 * 1024 * 1024 ) ;
}
}
private final URL baseUrl ; private final URL baseUrl ;
private final URL objectsUrl ; private final URL objectsUrl ;
private final HttpConfig http ;
private final ProxySelector proxySelector ; private final ProxySelector proxySelector ;
TransportHttp ( final Repository local , final URIish uri ) TransportHttp ( final Repository local , final URIish uri )
@ -101,22 +167,75 @@ public class TransportHttp extends HttpTransport implements WalkTransport {
} catch ( MalformedURLException e ) { } catch ( MalformedURLException e ) {
throw new NotSupportedException ( "Invalid URL " + uri , e ) ; throw new NotSupportedException ( "Invalid URL " + uri , e ) ;
} }
http = local . getConfig ( ) . get ( HTTP_KEY ) ;
proxySelector = ProxySelector . getDefault ( ) ; proxySelector = ProxySelector . getDefault ( ) ;
} }
@Override @Override
public FetchConnection openFetch ( ) throws TransportException { public FetchConnection openFetch ( ) throws TransportException ,
final HttpObjectDB c = new HttpObjectDB ( objectsUrl ) ; NotSupportedException {
final WalkFetchConnection r = new WalkFetchConnection ( this , c ) ; final String service = SVC_UPLOAD_PACK ;
r . available ( c . readAdvertisedRefs ( ) ) ; try {
return r ; final HttpURLConnection c = connect ( service ) ;
final InputStream in = openInputStream ( c ) ;
try {
if ( isSmartHttp ( c , service ) ) {
readSmartHeaders ( in , service ) ;
return new SmartHttpFetchConnection ( in ) ;
} else {
// Assume this server doesn't support smart HTTP fetch
// and fall back on dumb object walking.
//
HttpObjectDB d = new HttpObjectDB ( objectsUrl ) ;
WalkFetchConnection wfc = new WalkFetchConnection ( this , d ) ;
BufferedReader br = new BufferedReader (
new InputStreamReader ( in , Constants . CHARSET ) ) ;
try {
wfc . available ( d . readAdvertisedImpl ( br ) ) ;
} finally {
br . close ( ) ;
}
return wfc ;
}
} finally {
in . close ( ) ;
}
} catch ( NotSupportedException err ) {
throw err ;
} catch ( TransportException err ) {
throw err ;
} catch ( IOException err ) {
throw new TransportException ( uri , "error reading info/refs" , err ) ;
}
} }
@Override @Override
public PushConnection openPush ( ) throws NotSupportedException , public PushConnection openPush ( ) throws NotSupportedException ,
TransportException { TransportException {
final String s = getURI ( ) . getScheme ( ) ; final String service = SVC_RECEIVE_PACK ;
throw new NotSupportedException ( "Push not supported over " + s + "." ) ; try {
final HttpURLConnection c = connect ( service ) ;
final InputStream in = openInputStream ( c ) ;
try {
if ( isSmartHttp ( c , service ) ) {
readSmartHeaders ( in , service ) ;
return new SmartHttpPushConnection ( in ) ;
} else {
final String msg = "remote does not support smart HTTP push" ;
throw new NotSupportedException ( msg ) ;
}
} finally {
in . close ( ) ;
}
} catch ( NotSupportedException err ) {
throw err ;
} catch ( TransportException err ) {
throw err ;
} catch ( IOException err ) {
throw new TransportException ( uri , "error reading info/refs" , err ) ;
}
} }
@Override @Override
@ -124,6 +243,112 @@ public class TransportHttp extends HttpTransport implements WalkTransport {
// No explicit connections are maintained.
// No explicit connections are maintained.
} }
private HttpURLConnection connect ( final String service )
throws TransportException , NotSupportedException {
final URL u ;
try {
final StringBuilder b = new StringBuilder ( ) ;
b . append ( baseUrl ) ;
if ( b . charAt ( b . length ( ) - 1 ) ! = '/' )
b . append ( '/' ) ;
b . append ( Constants . INFO_REFS ) ;
b . append ( b . indexOf ( "?" ) < 0 ? '?' : '&' ) ;
b . append ( "service=" ) ;
b . append ( service ) ;
u = new URL ( b . toString ( ) ) ;
} catch ( MalformedURLException e ) {
throw new NotSupportedException ( "Invalid URL " + uri , e ) ;
}
try {
final HttpURLConnection conn = httpOpen ( u ) ;
String expType = "application/x-" + service + "-advertisement" ;
conn . setRequestProperty ( HDR_ACCEPT , expType + ", */*" ) ;
final int status = HttpSupport . response ( conn ) ;
switch ( status ) {
case HttpURLConnection . HTTP_OK :
return conn ;
case HttpURLConnection . HTTP_NOT_FOUND :
throw new NoRemoteRepositoryException ( uri , u + " not found" ) ;
case HttpURLConnection . HTTP_FORBIDDEN :
throw new TransportException ( uri , service + " not permitted" ) ;
default :
String err = status + " " + conn . getResponseMessage ( ) ;
throw new TransportException ( uri , err ) ;
}
} catch ( NotSupportedException e ) {
throw e ;
} catch ( TransportException e ) {
throw e ;
} catch ( IOException e ) {
throw new TransportException ( uri , "cannot open " + service , e ) ;
}
}
final HttpURLConnection httpOpen ( final URL u ) throws IOException {
final Proxy proxy = HttpSupport . proxyFor ( proxySelector , u ) ;
HttpURLConnection conn = ( HttpURLConnection ) u . openConnection ( proxy ) ;
conn . setRequestProperty ( HDR_ACCEPT_ENCODING , ENCODING_GZIP ) ;
conn . setRequestProperty ( HDR_PRAGMA , "no-cache" ) ; //$NON-NLS-1$
conn . setRequestProperty ( HDR_USER_AGENT , userAgent ) ;
return conn ;
}
final InputStream openInputStream ( HttpURLConnection conn )
throws IOException {
InputStream input = conn . getInputStream ( ) ;
if ( ENCODING_GZIP . equals ( conn . getHeaderField ( HDR_CONTENT_ENCODING ) ) )
input = new GZIPInputStream ( input ) ;
return input ;
}
IOException wrongContentType ( String expType , String actType ) {
final String why = "expected Content-Type " + expType
+ "; received Content-Type " + actType ;
return new TransportException ( uri , why ) ;
}
private boolean isSmartHttp ( final HttpURLConnection c , final String service ) {
final String expType = "application/x-" + service + "-advertisement" ;
final String actType = c . getContentType ( ) ;
return expType . equals ( actType ) ;
}
private void readSmartHeaders ( final InputStream in , final String service )
throws IOException {
// A smart reply will have a '#' after the first 4 bytes, but
// a dumb reply cannot contain a '#' until after byte 41. Do a
// quick check to make sure its a smart reply before we parse
// as a pkt-line stream.
//
final byte [ ] magic = new byte [ 5 ] ;
IO . readFully ( in , magic , 0 , magic . length ) ;
if ( magic [ 4 ] ! = '#' ) {
throw new TransportException ( uri , "expected pkt-line with"
+ " '# service=', got '" + RawParseUtils . decode ( magic )
+ "'" ) ;
}
final PacketLineIn pckIn = new PacketLineIn ( new UnionInputStream (
new ByteArrayInputStream ( magic ) , in ) ) ;
final String exp = "# service=" + service ;
final String act = pckIn . readString ( ) ;
if ( ! exp . equals ( act ) ) {
throw new TransportException ( uri , "expected '" + exp + "', got '"
+ act + "'" ) ;
}
while ( pckIn . readString ( ) ! = PacketLineIn . END ) {
// for now, ignore the remaining header lines
}
}
class HttpObjectDB extends WalkRemoteObjectDatabase { class HttpObjectDB extends WalkRemoteObjectDatabase {
private final URL objectsUrl ; private final URL objectsUrl ;
@ -186,13 +411,10 @@ public class TransportHttp extends HttpTransport implements WalkTransport {
FileStream open ( final String path ) throws IOException { FileStream open ( final String path ) throws IOException {
final URL base = objectsUrl ; final URL base = objectsUrl ;
final URL u = new URL ( base , path ) ; final URL u = new URL ( base , path ) ;
final Proxy proxy = HttpSupport . proxyFor ( proxySelector , u ) ; final HttpURLConnection c = httpOpen ( u ) ;
final HttpURLConnection c ;
c = ( HttpURLConnection ) u . openConnection ( proxy ) ;
switch ( HttpSupport . response ( c ) ) { switch ( HttpSupport . response ( c ) ) {
case HttpURLConnection . HTTP_OK : case HttpURLConnection . HTTP_OK :
final InputStream in = c . getInputStream ( ) ; final InputStream in = openInputStream ( c ) ;
final int len = c . getContentLength ( ) ; final int len = c . getContentLength ( ) ;
return new FileStream ( in , len ) ; return new FileStream ( in , len ) ;
case HttpURLConnection . HTTP_NOT_FOUND : case HttpURLConnection . HTTP_NOT_FOUND :
@ -204,26 +426,7 @@ public class TransportHttp extends HttpTransport implements WalkTransport {
} }
} }
Map < String , Ref > readAdvertisedRefs ( ) throws TransportException { Map < String , Ref > readAdvertisedImpl ( final BufferedReader br )
try {
final BufferedReader br = openReader ( INFO_REFS ) ;
try {
return readAdvertisedImpl ( br ) ;
} finally {
br . close ( ) ;
}
} catch ( IOException err ) {
try {
throw new TransportException ( new URL ( objectsUrl , INFO_REFS )
+ ": cannot read available refs" , err ) ;
} catch ( MalformedURLException mue ) {
throw new TransportException ( objectsUrl + INFO_REFS
+ ": cannot read available refs" , err ) ;
}
}
}
private Map < String , Ref > readAdvertisedImpl ( final BufferedReader br )
throws IOException , PackProtocolException { throws IOException , PackProtocolException {
final TreeMap < String , Ref > avail = new TreeMap < String , Ref > ( ) ; final TreeMap < String , Ref > avail = new TreeMap < String , Ref > ( ) ;
for ( ; ; ) { for ( ; ; ) {
@ -279,4 +482,220 @@ public class TransportHttp extends HttpTransport implements WalkTransport {
// We do not maintain persistent connections.
// We do not maintain persistent connections.
} }
} }
class SmartHttpFetchConnection extends BasePackFetchConnection {
SmartHttpFetchConnection ( final InputStream advertisement )
throws TransportException {
super ( TransportHttp . this ) ;
statelessRPC = true ;
init ( advertisement , DisabledOutputStream . INSTANCE ) ;
outNeedsEnd = false ;
try {
readAdvertisedRefs ( ) ;
} catch ( IOException err ) {
close ( ) ;
throw new TransportException ( uri , "remote hung up" , err ) ;
}
}
@Override
protected void doFetch ( final ProgressMonitor monitor ,
final Collection < Ref > want , final Set < ObjectId > have )
throws TransportException {
final Service svc = new Service ( SVC_UPLOAD_PACK ) ;
init ( svc . in , svc . out ) ;
super . doFetch ( monitor , want , have ) ;
}
}
class SmartHttpPushConnection extends BasePackPushConnection {
SmartHttpPushConnection ( final InputStream advertisement )
throws TransportException {
super ( TransportHttp . this ) ;
statelessRPC = true ;
init ( advertisement , DisabledOutputStream . INSTANCE ) ;
outNeedsEnd = false ;
try {
readAdvertisedRefs ( ) ;
} catch ( IOException err ) {
close ( ) ;
throw new TransportException ( uri , "remote hung up" , err ) ;
}
}
protected void doPush ( final ProgressMonitor monitor ,
final Map < String , RemoteRefUpdate > refUpdates )
throws TransportException {
final Service svc = new Service ( SVC_RECEIVE_PACK ) ;
init ( svc . in , svc . out ) ;
super . doPush ( monitor , refUpdates ) ;
}
}
/ * *
* State required to speak multiple HTTP requests with the remote .
* < p >
* A service wrapper provides a normal looking InputStream and OutputStream
* pair which are connected via HTTP to the named remote service . Writing to
* the OutputStream is buffered until either the buffer overflows , or
* reading from the InputStream occurs . If overflow occurs HTTP / 1 . 1 and its
* chunked transfer encoding is used to stream the request data to the
* remote service . If the entire request fits in the memory buffer , the
* older HTTP / 1 . 0 standard and a fixed content length is used instead .
* < p >
* It is an error to attempt to read without there being outstanding data
* ready for transmission on the OutputStream .
* < p >
* No state is preserved between write - read request pairs . The caller is
* responsible for replaying state vector information as part of the request
* data written to the OutputStream . Any session HTTP cookies may or may not
* be preserved between requests , it is left up to the JVM ' s implementation
* of the HTTP client .
* /
class Service {
private final String serviceName ;
private final String requestType ;
private final String responseType ;
private final UnionInputStream httpIn ;
final HttpInputStream in ;
final HttpOutputStream out ;
HttpURLConnection conn ;
Service ( final String serviceName ) {
this . serviceName = serviceName ;
this . requestType = "application/x-" + serviceName + "-request" ;
this . responseType = "application/x-" + serviceName + "-result" ;
this . httpIn = new UnionInputStream ( ) ;
this . in = new HttpInputStream ( httpIn ) ;
this . out = new HttpOutputStream ( ) ;
}
void openStream ( ) throws IOException {
conn = httpOpen ( new URL ( baseUrl , serviceName ) ) ;
conn . setRequestMethod ( METHOD_POST ) ;
conn . setInstanceFollowRedirects ( false ) ;
conn . setDoOutput ( true ) ;
conn . setRequestProperty ( HDR_CONTENT_TYPE , requestType ) ;
conn . setRequestProperty ( HDR_ACCEPT , responseType ) ;
}
void execute ( ) throws IOException {
out . close ( ) ;
if ( conn = = null ) {
// Output hasn't started yet, because everything fit into
// our request buffer. Send with a Content-Length header.
//
if ( out . length ( ) = = 0 ) {
throw new TransportException ( uri , "Starting read stage"
+ " without written request data pending"
+ " is not supported" ) ;
}
// Try to compress the content, but only if that is smaller.
TemporaryBuffer buf = new TemporaryBuffer . Heap ( http . postBuffer ) ;
try {
GZIPOutputStream gzip = new GZIPOutputStream ( buf ) ;
out . writeTo ( gzip , null ) ;
gzip . close ( ) ;
if ( out . length ( ) < buf . length ( ) )
buf = out ;
} catch ( IOException err ) {
// Most likely caused by overflowing the buffer, meaning
// its larger if it were compressed. Don't compress.
buf = out ;
}
openStream ( ) ;
if ( buf ! = out )
conn . setRequestProperty ( HDR_CONTENT_ENCODING , ENCODING_GZIP ) ;
conn . setFixedLengthStreamingMode ( ( int ) buf . length ( ) ) ;
final OutputStream httpOut = conn . getOutputStream ( ) ;
try {
buf . writeTo ( httpOut , null ) ;
} finally {
httpOut . close ( ) ;
}
}
out . reset ( ) ;
final int status = HttpSupport . response ( conn ) ;
if ( status ! = HttpURLConnection . HTTP_OK ) {
throw new TransportException ( uri , status + " "
+ conn . getResponseMessage ( ) ) ;
}
final String contentType = conn . getContentType ( ) ;
if ( ! responseType . equals ( contentType ) ) {
conn . getInputStream ( ) . close ( ) ;
throw wrongContentType ( responseType , contentType ) ;
}
httpIn . add ( openInputStream ( conn ) ) ;
conn = null ;
}
class HttpOutputStream extends TemporaryBuffer {
HttpOutputStream ( ) {
super ( http . postBuffer ) ;
}
@Override
protected OutputStream overflow ( ) throws IOException {
openStream ( ) ;
conn . setChunkedStreamingMode ( 0 ) ;
return conn . getOutputStream ( ) ;
}
}
class HttpInputStream extends InputStream {
private final UnionInputStream src ;
HttpInputStream ( UnionInputStream httpIn ) {
this . src = httpIn ;
}
private InputStream self ( ) throws IOException {
if ( src . isEmpty ( ) ) {
// If we have no InputStreams available it means we must
// have written data previously to the service, but have
// not yet finished the HTTP request in order to get the
// response from the service. Ensure we get it now.
//
execute ( ) ;
}
return src ;
}
public int available ( ) throws IOException {
return self ( ) . available ( ) ;
}
public int read ( ) throws IOException {
return self ( ) . read ( ) ;
}
public int read ( byte [ ] b , int off , int len ) throws IOException {
return self ( ) . read ( b , off , len ) ;
}
public long skip ( long n ) throws IOException {
return self ( ) . skip ( n ) ;
}
public void close ( ) throws IOException {
src . close ( ) ;
}
}
}
} }
Shawn O. Pearce
15 years ago