Browse Source
Add a storage implementation storing large objects in Amazon S3. The AmazonS3Repository pre-signs download and upload requests. AWS access and secret key are expected to be in the $HOME/.aws/credentials file in the following format: [default] accessKey = ... secretKey = ... Use AWS version 4 request signing [1] because it is more secure and supported by all regions. The version 3 signing is not supported in newer regions. In follow up changes we should: - implement getVerifyAction() and do actual verification. Subclasses of S3Repository can implement caching for object meta data (size) in order to avoid extra roundtrips to S3. Verification should ensure that meta data store and content of S3 storage are in sync - HEAD request used in S3Repository.getSize() seems to always return Content-length 0 in contrast to the documentation [2]. So getSize() does detect if the object exists in S3 or not but in case the object exists it always returns size 0 [1] http://docs.aws.amazon.com/general/latest/gr/signature-version-4.html [2] https://forums.aws.amazon.com/thread.jspa?threadID=223616 Change-Id: Ic47f094928a259e5264c92b3aacf6d90210907a8 Signed-off-by: Matthias Sohn <matthias.sohn@sap.com> Signed-off-by: Sasa Zivkov <sasa.zivkov@sap.com>stable-4.3
Matthias Sohn
9 years ago
committed by
Sasa Zivkov
11 changed files with 915 additions and 11 deletions
@ -1,4 +1,12 @@ |
|||||||
corruptLongObject=The content hash ''{0}'' of the long object ''{1}'' doesn''t match its id, the corrupt object will be deleted. |
corruptLongObject=The content hash ''{0}'' of the long object ''{1}'' doesn''t match its id, the corrupt object will be deleted. |
||||||
|
failedToCalcSignature=Failed to calculate a request signature: {0} |
||||||
invalidPathInfo=Invalid pathInfo ''{0}'' does not match ''/'{'SHA-256'}''' |
invalidPathInfo=Invalid pathInfo ''{0}'' does not match ''/'{'SHA-256'}''' |
||||||
objectNotFound=Object ''{0}'' not found |
objectNotFound=Object ''{0}'' not found |
||||||
|
undefinedS3AccessKey=S3 configuration: ''accessKey'' is undefined |
||||||
|
undefinedS3Bucket=S3 configuration: ''bucket'' is undefined |
||||||
|
undefinedS3Region=S3 configuration: ''region'' is undefined |
||||||
|
undefinedS3SecretKey=S3 configuration: ''secretKey'' is undefined |
||||||
|
undefinedS3StorageClass=S3 configuration: ''storageClass'' is undefined |
||||||
|
unparsableEndpoint=Unable to parse service endpoint: {0} |
||||||
unsupportedOperation=Operation ''{0}'' is not supported |
unsupportedOperation=Operation ''{0}'' is not supported |
||||||
|
unsupportedUtf8=UTF-8 encoding is not supported. |
||||||
|
|||||||
@ -0,0 +1,141 @@ |
|||||||
|
/* |
||||||
|
* Copyright (C) 2015, Matthias Sohn <matthias.sohn@sap.com> |
||||||
|
* Copyright (C) 2015, Sasa Zivkov <sasa.zivkov@sap.com> |
||||||
|
* and other copyright owners as documented in the project's IP log. |
||||||
|
* |
||||||
|
* This program and the accompanying materials are made available |
||||||
|
* under the terms of the Eclipse Distribution License v1.0 which |
||||||
|
* accompanies this distribution, is reproduced below, and is |
||||||
|
* available at http://www.eclipse.org/org/documents/edl-v10.php
|
||||||
|
* |
||||||
|
* All rights reserved. |
||||||
|
* |
||||||
|
* Redistribution and use in source and binary forms, with or |
||||||
|
* without modification, are permitted provided that the following |
||||||
|
* conditions are met: |
||||||
|
* |
||||||
|
* - Redistributions of source code must retain the above copyright |
||||||
|
* notice, this list of conditions and the following disclaimer. |
||||||
|
* |
||||||
|
* - Redistributions in binary form must reproduce the above |
||||||
|
* copyright notice, this list of conditions and the following |
||||||
|
* disclaimer in the documentation and/or other materials provided |
||||||
|
* with the distribution. |
||||||
|
* |
||||||
|
* - Neither the name of the Eclipse Foundation, Inc. nor the |
||||||
|
* names of its contributors may be used to endorse or promote |
||||||
|
* products derived from this software without specific prior |
||||||
|
* written permission. |
||||||
|
* |
||||||
|
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND |
||||||
|
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, |
||||||
|
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
||||||
|
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
||||||
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR |
||||||
|
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
||||||
|
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
||||||
|
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
||||||
|
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER |
||||||
|
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, |
||||||
|
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
||||||
|
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF |
||||||
|
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||||||
|
*/ |
||||||
|
package org.eclipse.jgit.lfs.server.s3; |
||||||
|
|
||||||
|
/** |
||||||
|
* Configuration for an Amazon AWS S3 bucket |
||||||
|
* |
||||||
|
* @since 4.3 |
||||||
|
*/ |
||||||
|
public class S3Config { |
||||||
|
private final String region; |
||||||
|
private final String bucket; |
||||||
|
private final String storageClass; |
||||||
|
private final String accessKey; |
||||||
|
private final String secretKey; |
||||||
|
private final int expirationSeconds; |
||||||
|
private final boolean disableSslVerify; |
||||||
|
|
||||||
|
/** |
||||||
|
* @param region |
||||||
|
* AWS region |
||||||
|
* @param bucket |
||||||
|
* S3 storage bucket |
||||||
|
* @param storageClass |
||||||
|
* S3 storage class
|
||||||
|
* @param accessKey |
||||||
|
* access key for authenticating to AWS |
||||||
|
* @param secretKey |
||||||
|
* secret key for authenticating to AWS |
||||||
|
* @param expirationSeconds |
||||||
|
* period in seconds after which requests signed for this bucket |
||||||
|
* will expire |
||||||
|
* @param disableSslVerify |
||||||
|
* if {@code true} disable Amazon server certificate and hostname |
||||||
|
* verification |
||||||
|
*/ |
||||||
|
public S3Config(String region, String bucket, String storageClass, |
||||||
|
String accessKey, String secretKey, int expirationSeconds, |
||||||
|
boolean disableSslVerify) { |
||||||
|
this.region = region; |
||||||
|
this.bucket = bucket; |
||||||
|
this.storageClass = storageClass; |
||||||
|
this.accessKey = accessKey; |
||||||
|
this.secretKey = secretKey; |
||||||
|
this.expirationSeconds = expirationSeconds; |
||||||
|
this.disableSslVerify = disableSslVerify; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* @return Get name of AWS region this bucket resides in |
||||||
|
*/ |
||||||
|
public String getRegion() { |
||||||
|
return region; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* @return Get S3 storage bucket name |
||||||
|
*/ |
||||||
|
public String getBucket() { |
||||||
|
return bucket; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* @return S3 storage class to use for objects stored in this bucket |
||||||
|
*/ |
||||||
|
public String getStorageClass() { |
||||||
|
return storageClass; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* @return access key for authenticating to AWS |
||||||
|
*/ |
||||||
|
public String getAccessKey() { |
||||||
|
return accessKey; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* @return secret key for authenticating to AWS |
||||||
|
*/ |
||||||
|
public String getSecretKey() { |
||||||
|
return secretKey; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* @return period in seconds after which requests signed for this bucket |
||||||
|
* will expire |
||||||
|
*/ |
||||||
|
public int getExpirationSeconds() { |
||||||
|
return expirationSeconds; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* @return {@code true} if Amazon server certificate and hostname |
||||||
|
* verification is disabled |
||||||
|
*/ |
||||||
|
boolean isDisableSslVerify() { |
||||||
|
return disableSslVerify; |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
@ -0,0 +1,217 @@ |
|||||||
|
/* |
||||||
|
* Copyright (C) 2015, Matthias Sohn <matthias.sohn@sap.com> |
||||||
|
* Copyright (C) 2015, Sasa Zivkov <sasa.zivkov@sap.com> |
||||||
|
* and other copyright owners as documented in the project's IP log. |
||||||
|
* |
||||||
|
* This program and the accompanying materials are made available |
||||||
|
* under the terms of the Eclipse Distribution License v1.0 which |
||||||
|
* accompanies this distribution, is reproduced below, and is |
||||||
|
* available at http://www.eclipse.org/org/documents/edl-v10.php
|
||||||
|
* |
||||||
|
* All rights reserved. |
||||||
|
* |
||||||
|
* Redistribution and use in source and binary forms, with or |
||||||
|
* without modification, are permitted provided that the following |
||||||
|
* conditions are met: |
||||||
|
* |
||||||
|
* - Redistributions of source code must retain the above copyright |
||||||
|
* notice, this list of conditions and the following disclaimer. |
||||||
|
* |
||||||
|
* - Redistributions in binary form must reproduce the above |
||||||
|
* copyright notice, this list of conditions and the following |
||||||
|
* disclaimer in the documentation and/or other materials provided |
||||||
|
* with the distribution. |
||||||
|
* |
||||||
|
* - Neither the name of the Eclipse Foundation, Inc. nor the |
||||||
|
* names of its contributors may be used to endorse or promote |
||||||
|
* products derived from this software without specific prior |
||||||
|
* written permission. |
||||||
|
* |
||||||
|
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND |
||||||
|
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, |
||||||
|
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
||||||
|
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
||||||
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR |
||||||
|
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
||||||
|
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
||||||
|
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
||||||
|
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER |
||||||
|
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, |
||||||
|
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
||||||
|
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF |
||||||
|
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||||||
|
*/ |
||||||
|
package org.eclipse.jgit.lfs.server.s3; |
||||||
|
|
||||||
|
import static javax.servlet.http.HttpServletResponse.SC_OK; |
||||||
|
import static org.eclipse.jgit.lfs.server.s3.SignerV4.UNSIGNED_PAYLOAD; |
||||||
|
import static org.eclipse.jgit.lfs.server.s3.SignerV4.X_AMZ_CONTENT_SHA256; |
||||||
|
import static org.eclipse.jgit.lfs.server.s3.SignerV4.X_AMZ_EXPIRES; |
||||||
|
import static org.eclipse.jgit.lfs.server.s3.SignerV4.X_AMZ_STORAGE_CLASS; |
||||||
|
import static org.eclipse.jgit.util.HttpSupport.HDR_CONTENT_LENGTH; |
||||||
|
import static org.eclipse.jgit.util.HttpSupport.METHOD_GET; |
||||||
|
import static org.eclipse.jgit.util.HttpSupport.METHOD_HEAD; |
||||||
|
import static org.eclipse.jgit.util.HttpSupport.METHOD_PUT; |
||||||
|
|
||||||
|
import java.io.IOException; |
||||||
|
import java.net.MalformedURLException; |
||||||
|
import java.net.Proxy; |
||||||
|
import java.net.ProxySelector; |
||||||
|
import java.net.URL; |
||||||
|
import java.text.MessageFormat; |
||||||
|
import java.util.HashMap; |
||||||
|
import java.util.Map; |
||||||
|
|
||||||
|
import org.eclipse.jgit.lfs.lib.AnyLongObjectId; |
||||||
|
import org.eclipse.jgit.lfs.server.LargeFileRepository; |
||||||
|
import org.eclipse.jgit.lfs.server.Response; |
||||||
|
import org.eclipse.jgit.lfs.server.Response.Action; |
||||||
|
import org.eclipse.jgit.lfs.server.internal.LfsServerText; |
||||||
|
import org.eclipse.jgit.transport.http.HttpConnection; |
||||||
|
import org.eclipse.jgit.transport.http.apache.HttpClientConnectionFactory; |
||||||
|
import org.eclipse.jgit.util.HttpSupport; |
||||||
|
|
||||||
|
/** |
||||||
|
* Repository storing LFS objects in Amazon S3 |
||||||
|
* |
||||||
|
* @since 4.3 |
||||||
|
*/ |
||||||
|
public class S3Repository implements LargeFileRepository { |
||||||
|
|
||||||
|
private S3Config s3Config; |
||||||
|
|
||||||
|
/** |
||||||
|
* Construct a LFS repository storing large objects in Amazon S3 |
||||||
|
* |
||||||
|
* @param config |
||||||
|
* AWS S3 storage bucket configuration |
||||||
|
*/ |
||||||
|
public S3Repository(S3Config config) { |
||||||
|
validateConfig(config); |
||||||
|
this.s3Config = config; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public Response.Action getDownloadAction(AnyLongObjectId oid) { |
||||||
|
URL endpointUrl = getObjectUrl(oid); |
||||||
|
Map<String, String> queryParams = new HashMap<String, String>(); |
||||||
|
queryParams.put(X_AMZ_EXPIRES, |
||||||
|
Integer.toString(s3Config.getExpirationSeconds())); |
||||||
|
Map<String, String> headers = new HashMap<String, String>(); |
||||||
|
String authorizationQueryParameters = SignerV4.createAuthorizationQuery( |
||||||
|
s3Config, endpointUrl, METHOD_GET, headers, queryParams, |
||||||
|
UNSIGNED_PAYLOAD); |
||||||
|
|
||||||
|
Response.Action a = new Response.Action(); |
||||||
|
a.href = endpointUrl.toString() + "?" + authorizationQueryParameters; //$NON-NLS-1$
|
||||||
|
return a; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public Response.Action getUploadAction(AnyLongObjectId oid, long size) { |
||||||
|
cacheObjectMetaData(oid, size); |
||||||
|
URL objectUrl = getObjectUrl(oid); |
||||||
|
Map<String, String> headers = new HashMap<String, String>(); |
||||||
|
headers.put(X_AMZ_CONTENT_SHA256, oid.getName()); |
||||||
|
headers.put(HDR_CONTENT_LENGTH, Long.toString(size)); |
||||||
|
headers.put(X_AMZ_STORAGE_CLASS, s3Config.getStorageClass()); |
||||||
|
headers.put(HttpSupport.HDR_CONTENT_TYPE, "application/octet-stream"); //$NON-NLS-1$
|
||||||
|
headers = SignerV4.createHeaderAuthorization(s3Config, objectUrl, |
||||||
|
METHOD_PUT, headers, oid.getName()); |
||||||
|
|
||||||
|
Response.Action a = new Response.Action(); |
||||||
|
a.href = objectUrl.toString(); |
||||||
|
a.header = new HashMap<>(); |
||||||
|
a.header.putAll(headers); |
||||||
|
return a; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public Action getVerifyAction(AnyLongObjectId id) { |
||||||
|
return null; // TODO(ms) implement this
|
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public long getSize(AnyLongObjectId oid) throws IOException { |
||||||
|
URL endpointUrl = getObjectUrl(oid); |
||||||
|
Map<String, String> queryParams = new HashMap<String, String>(); |
||||||
|
queryParams.put(X_AMZ_EXPIRES, |
||||||
|
Integer.toString(s3Config.getExpirationSeconds())); |
||||||
|
Map<String, String> headers = new HashMap<String, String>(); |
||||||
|
|
||||||
|
String authorizationQueryParameters = SignerV4.createAuthorizationQuery( |
||||||
|
s3Config, endpointUrl, METHOD_HEAD, headers, queryParams, |
||||||
|
UNSIGNED_PAYLOAD); |
||||||
|
String href = endpointUrl.toString() + "?" //$NON-NLS-1$
|
||||||
|
+ authorizationQueryParameters; |
||||||
|
|
||||||
|
Proxy proxy = HttpSupport.proxyFor(ProxySelector.getDefault(), |
||||||
|
endpointUrl); |
||||||
|
HttpClientConnectionFactory f = new HttpClientConnectionFactory(); |
||||||
|
HttpConnection conn = f.create(new URL(href), proxy); |
||||||
|
if (s3Config.isDisableSslVerify()) { |
||||||
|
HttpSupport.disableSslVerify(conn); |
||||||
|
} |
||||||
|
conn.setRequestMethod(METHOD_HEAD); |
||||||
|
conn.connect(); |
||||||
|
int status = conn.getResponseCode(); |
||||||
|
if (status == SC_OK) { |
||||||
|
String contentLengthHeader = conn |
||||||
|
.getHeaderField(HDR_CONTENT_LENGTH); |
||||||
|
if (contentLengthHeader != null) { |
||||||
|
return Integer.parseInt(contentLengthHeader); |
||||||
|
} |
||||||
|
} |
||||||
|
return -1; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Cache metadata (size) for an object to avoid extra roundtrip to S3 in |
||||||
|
* order to retrieve this metadata for a given object. Subclasses can |
||||||
|
* implement a local cache and override {{@link #getSize(AnyLongObjectId)} |
||||||
|
* to retrieve the object size from the local cache to eliminate the need |
||||||
|
* for another roundtrip to S3 |
||||||
|
* |
||||||
|
* @param oid |
||||||
|
* the object id identifying the object to be cached |
||||||
|
* @param size |
||||||
|
* the object's size (in bytes) |
||||||
|
*/ |
||||||
|
protected void cacheObjectMetaData(AnyLongObjectId oid, long size) { |
||||||
|
// no caching
|
||||||
|
} |
||||||
|
|
||||||
|
private void validateConfig(S3Config config) { |
||||||
|
assertNotEmpty(LfsServerText.get().undefinedS3AccessKey, |
||||||
|
config.getAccessKey()); |
||||||
|
assertNotEmpty(LfsServerText.get().undefinedS3Bucket, |
||||||
|
config.getBucket()); |
||||||
|
assertNotEmpty(LfsServerText.get().undefinedS3Region, |
||||||
|
config.getRegion()); |
||||||
|
assertNotEmpty(LfsServerText.get().undefinedS3SecretKey, |
||||||
|
config.getSecretKey()); |
||||||
|
assertNotEmpty(LfsServerText.get().undefinedS3StorageClass, |
||||||
|
config.getStorageClass()); |
||||||
|
} |
||||||
|
|
||||||
|
private void assertNotEmpty(String message, String value) { |
||||||
|
if (value == null || value.trim().length() == 0) { |
||||||
|
throw new IllegalArgumentException(message); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
private URL getObjectUrl(AnyLongObjectId oid) { |
||||||
|
try { |
||||||
|
return new URL(String.format("https://s3-%s.amazonaws.com/%s/%s", //$NON-NLS-1$
|
||||||
|
s3Config.getRegion(), s3Config.getBucket(), |
||||||
|
getPath(oid))); |
||||||
|
} catch (MalformedURLException e) { |
||||||
|
throw new IllegalArgumentException(MessageFormat.format( |
||||||
|
LfsServerText.get().unparsableEndpoint, e.getMessage())); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
private String getPath(AnyLongObjectId oid) { |
||||||
|
return oid.getName(); |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,420 @@ |
|||||||
|
/* |
||||||
|
* Copyright (C) 2015, Matthias Sohn <matthias.sohn@sap.com> |
||||||
|
* Copyright (C) 2015, Sasa Zivkov <sasa.zivkov@sap.com> |
||||||
|
* and other copyright owners as documented in the project's IP log. |
||||||
|
* |
||||||
|
* This program and the accompanying materials are made available |
||||||
|
* under the terms of the Eclipse Distribution License v1.0 which |
||||||
|
* accompanies this distribution, is reproduced below, and is |
||||||
|
* available at http://www.eclipse.org/org/documents/edl-v10.php
|
||||||
|
* |
||||||
|
* All rights reserved. |
||||||
|
* |
||||||
|
* Redistribution and use in source and binary forms, with or |
||||||
|
* without modification, are permitted provided that the following |
||||||
|
* conditions are met: |
||||||
|
* |
||||||
|
* - Redistributions of source code must retain the above copyright |
||||||
|
* notice, this list of conditions and the following disclaimer. |
||||||
|
* |
||||||
|
* - Redistributions in binary form must reproduce the above |
||||||
|
* copyright notice, this list of conditions and the following |
||||||
|
* disclaimer in the documentation and/or other materials provided |
||||||
|
* with the distribution. |
||||||
|
* |
||||||
|
* - Neither the name of the Eclipse Foundation, Inc. nor the |
||||||
|
* names of its contributors may be used to endorse or promote |
||||||
|
* products derived from this software without specific prior |
||||||
|
* written permission. |
||||||
|
* |
||||||
|
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND |
||||||
|
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, |
||||||
|
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
||||||
|
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
||||||
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR |
||||||
|
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
||||||
|
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
||||||
|
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
||||||
|
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER |
||||||
|
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, |
||||||
|
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
||||||
|
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF |
||||||
|
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||||||
|
*/ |
||||||
|
package org.eclipse.jgit.lfs.server.s3; |
||||||
|
|
||||||
|
import static org.eclipse.jgit.util.HttpSupport.HDR_AUTHORIZATION; |
||||||
|
|
||||||
|
import java.io.UnsupportedEncodingException; |
||||||
|
import java.net.URL; |
||||||
|
import java.net.URLEncoder; |
||||||
|
import java.nio.charset.StandardCharsets; |
||||||
|
import java.security.MessageDigest; |
||||||
|
import java.text.MessageFormat; |
||||||
|
import java.text.SimpleDateFormat; |
||||||
|
import java.util.ArrayList; |
||||||
|
import java.util.Collections; |
||||||
|
import java.util.Date; |
||||||
|
import java.util.Iterator; |
||||||
|
import java.util.List; |
||||||
|
import java.util.Map; |
||||||
|
import java.util.SimpleTimeZone; |
||||||
|
import java.util.SortedMap; |
||||||
|
import java.util.TreeMap; |
||||||
|
|
||||||
|
import javax.crypto.Mac; |
||||||
|
import javax.crypto.spec.SecretKeySpec; |
||||||
|
|
||||||
|
import org.eclipse.jgit.lfs.lib.Constants; |
||||||
|
import org.eclipse.jgit.lfs.server.internal.LfsServerText; |
||||||
|
|
||||||
|
/** |
||||||
|
* Signing support for Amazon AWS signing V4 |
||||||
|
* <p> |
||||||
|
* See |
||||||
|
* http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html
|
||||||
|
*/ |
||||||
|
class SignerV4 { |
||||||
|
static final String UNSIGNED_PAYLOAD = "UNSIGNED-PAYLOAD"; //$NON-NLS-1$
|
||||||
|
|
||||||
|
private static final String ALGORITHM = "HMAC-SHA256"; //$NON-NLS-1$
|
||||||
|
private static final String DATE_STRING_FORMAT = "yyyyMMdd"; //$NON-NLS-1$
|
||||||
|
private static final String HEX = "0123456789abcdef"; //$NON-NLS-1$
|
||||||
|
private static final String HMACSHA256 = "HmacSHA256"; //$NON-NLS-1$
|
||||||
|
private static final String ISO8601_BASIC_FORMAT = "yyyyMMdd'T'HHmmss'Z'"; //$NON-NLS-1$
|
||||||
|
private static final String S3 = "s3"; //$NON-NLS-1$
|
||||||
|
private static final String SCHEME = "AWS4"; //$NON-NLS-1$
|
||||||
|
private static final String TERMINATOR = "aws4_request"; //$NON-NLS-1$
|
||||||
|
private static final String UTC = "UTC"; //$NON-NLS-1$
|
||||||
|
private static final String X_AMZ_ALGORITHM = "X-Amz-Algorithm"; //$NON-NLS-1$
|
||||||
|
private static final String X_AMZ_CREDENTIAL = "X-Amz-Credential"; //$NON-NLS-1$
|
||||||
|
private static final String X_AMZ_DATE = "X-Amz-Date"; //$NON-NLS-1$
|
||||||
|
private static final String X_AMZ_SIGNATURE = "X-Amz-Signature"; //$NON-NLS-1$
|
||||||
|
private static final String X_AMZ_SIGNED_HEADERS = "X-Amz-SignedHeaders"; //$NON-NLS-1$
|
||||||
|
|
||||||
|
static final String X_AMZ_CONTENT_SHA256 = "x-amz-content-sha256"; //$NON-NLS-1$
|
||||||
|
static final String X_AMZ_EXPIRES = "X-Amz-Expires"; //$NON-NLS-1$
|
||||||
|
static final String X_AMZ_STORAGE_CLASS = "x-amz-storage-class"; //$NON-NLS-1$
|
||||||
|
|
||||||
|
/** |
||||||
|
* Create an AWSV4 authorization for a request, suitable for embedding in |
||||||
|
* query parameters. |
||||||
|
* |
||||||
|
* @param bucketConfig |
||||||
|
* configuration of S3 storage bucket this request should be |
||||||
|
* signed for |
||||||
|
* @param url |
||||||
|
* HTTP request URL |
||||||
|
* @param httpMethod |
||||||
|
* HTTP method |
||||||
|
* @param headers |
||||||
|
* The HTTP request headers; 'Host' and 'X-Amz-Date' will be |
||||||
|
* added to this set. |
||||||
|
* @param queryParameters |
||||||
|
* Any query parameters that will be added to the endpoint. The |
||||||
|
* parameters should be specified in canonical format. |
||||||
|
* @param bodyHash |
||||||
|
* Pre-computed SHA256 hash of the request body content; this |
||||||
|
* value should also be set as the header 'X-Amz-Content-SHA256' |
||||||
|
* for non-streaming uploads. |
||||||
|
* @return The computed authorization string for the request. This value |
||||||
|
* needs to be set as the header 'Authorization' on the subsequent |
||||||
|
* HTTP request. |
||||||
|
*/ |
||||||
|
static String createAuthorizationQuery(S3Config bucketConfig, URL url, |
||||||
|
String httpMethod, Map<String, String> headers, |
||||||
|
Map<String, String> queryParameters, String bodyHash) { |
||||||
|
addHostHeader(url, headers); |
||||||
|
|
||||||
|
queryParameters.put(X_AMZ_ALGORITHM, SCHEME + "-" + ALGORITHM); //$NON-NLS-1$
|
||||||
|
|
||||||
|
Date now = new Date(); |
||||||
|
String dateStamp = dateStamp(now); |
||||||
|
String scope = scope(bucketConfig.getRegion(), dateStamp); |
||||||
|
queryParameters.put(X_AMZ_CREDENTIAL, |
||||||
|
bucketConfig.getAccessKey() + "/" + scope); //$NON-NLS-1$
|
||||||
|
|
||||||
|
String dateTimeStampISO8601 = dateTimeStampISO8601(now); |
||||||
|
queryParameters.put(X_AMZ_DATE, dateTimeStampISO8601); |
||||||
|
|
||||||
|
String canonicalizedHeaderNames = canonicalizeHeaderNames(headers); |
||||||
|
queryParameters.put(X_AMZ_SIGNED_HEADERS, canonicalizedHeaderNames); |
||||||
|
|
||||||
|
String canonicalizedQueryParameters = canonicalizeQueryString( |
||||||
|
queryParameters); |
||||||
|
String canonicalizedHeaders = canonicalizeHeaderString(headers); |
||||||
|
String canonicalRequest = canonicalRequest(url, httpMethod, |
||||||
|
canonicalizedQueryParameters, canonicalizedHeaderNames, |
||||||
|
canonicalizedHeaders, bodyHash); |
||||||
|
byte[] signature = createSignature(bucketConfig, dateTimeStampISO8601, |
||||||
|
dateStamp, scope, canonicalRequest); |
||||||
|
queryParameters.put(X_AMZ_SIGNATURE, toHex(signature)); |
||||||
|
|
||||||
|
return formatAuthorizationQuery(queryParameters); |
||||||
|
} |
||||||
|
|
||||||
|
private static String formatAuthorizationQuery( |
||||||
|
Map<String, String> queryParameters) { |
||||||
|
StringBuilder s = new StringBuilder(); |
||||||
|
for (String key : queryParameters.keySet()) { |
||||||
|
appendQuery(s, key, queryParameters.get(key)); |
||||||
|
} |
||||||
|
return s.toString(); |
||||||
|
} |
||||||
|
|
||||||
|
private static void appendQuery(StringBuilder s, String key, |
||||||
|
String value) { |
||||||
|
if (s.length() != 0) { |
||||||
|
s.append("&"); //$NON-NLS-1$
|
||||||
|
} |
||||||
|
s.append(key).append("=").append(value); //$NON-NLS-1$
|
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Sign headers for given bucket, url and HTTP method and add signature in |
||||||
|
* Authorization header. |
||||||
|
* |
||||||
|
* @param bucketConfig |
||||||
|
* configuration of S3 storage bucket this request should be |
||||||
|
* signed for |
||||||
|
* @param url |
||||||
|
* HTTP request URL |
||||||
|
* @param httpMethod |
||||||
|
* HTTP method |
||||||
|
* @param headers |
||||||
|
* HTTP headers to sign |
||||||
|
* @param bodyHash |
||||||
|
* Pre-computed SHA256 hash of the request body content; this |
||||||
|
* value should also be set as the header 'X-Amz-Content-SHA256' |
||||||
|
* for non-streaming uploads. |
||||||
|
* @return HTTP headers signd by an Authorization header added to the |
||||||
|
* headers |
||||||
|
*/ |
||||||
|
static Map<String, String> createHeaderAuthorization( |
||||||
|
S3Config bucketConfig, URL url, String httpMethod, |
||||||
|
Map<String, String> headers, String bodyHash) { |
||||||
|
addHostHeader(url, headers); |
||||||
|
|
||||||
|
Date now = new Date(); |
||||||
|
String dateTimeStamp = dateTimeStampISO8601(now); |
||||||
|
headers.put(X_AMZ_DATE, dateTimeStamp); |
||||||
|
|
||||||
|
String canonicalizedHeaderNames = canonicalizeHeaderNames(headers); |
||||||
|
String canonicalizedHeaders = canonicalizeHeaderString(headers); |
||||||
|
String canonicalRequest = canonicalRequest(url, httpMethod, "", //$NON-NLS-1$
|
||||||
|
canonicalizedHeaderNames, canonicalizedHeaders, bodyHash); |
||||||
|
String dateStamp = dateStamp(now); |
||||||
|
String scope = scope(bucketConfig.getRegion(), dateStamp); |
||||||
|
|
||||||
|
byte[] signature = createSignature(bucketConfig, dateTimeStamp, |
||||||
|
dateStamp, scope, canonicalRequest); |
||||||
|
|
||||||
|
headers.put(HDR_AUTHORIZATION, formatAuthorizationHeader(bucketConfig, |
||||||
|
canonicalizedHeaderNames, scope, signature)); // $NON-NLS-1$
|
||||||
|
|
||||||
|
return headers; |
||||||
|
} |
||||||
|
|
||||||
|
private static String formatAuthorizationHeader( |
||||||
|
S3Config bucketConfig, String canonicalizedHeaderNames, |
||||||
|
String scope, byte[] signature) { |
||||||
|
StringBuilder s = new StringBuilder(); |
||||||
|
s.append(SCHEME).append("-").append(ALGORITHM).append(" "); //$NON-NLS-1$ //$NON-NLS-2$
|
||||||
|
s.append("Credential=").append(bucketConfig.getAccessKey()).append("/") //$NON-NLS-1$//$NON-NLS-2$
|
||||||
|
.append(scope).append(","); //$NON-NLS-1$
|
||||||
|
s.append("SignedHeaders=").append(canonicalizedHeaderNames).append(","); //$NON-NLS-1$ //$NON-NLS-2$
|
||||||
|
s.append("Signature=").append(toHex(signature)); //$NON-NLS-1$
|
||||||
|
return s.toString(); |
||||||
|
} |
||||||
|
|
||||||
|
private static void addHostHeader(URL url, |
||||||
|
Map<String, String> headers) { |
||||||
|
String hostHeader = url.getHost(); |
||||||
|
int port = url.getPort(); |
||||||
|
if (port > -1) { |
||||||
|
hostHeader.concat(":" + Integer.toString(port)); //$NON-NLS-1$
|
||||||
|
} |
||||||
|
headers.put("Host", hostHeader); //$NON-NLS-1$
|
||||||
|
} |
||||||
|
|
||||||
|
private static String canonicalizeHeaderNames( |
||||||
|
Map<String, String> headers) { |
||||||
|
List<String> sortedHeaders = new ArrayList<String>(); |
||||||
|
sortedHeaders.addAll(headers.keySet()); |
||||||
|
Collections.sort(sortedHeaders, String.CASE_INSENSITIVE_ORDER); |
||||||
|
|
||||||
|
StringBuilder buffer = new StringBuilder(); |
||||||
|
for (String header : sortedHeaders) { |
||||||
|
if (buffer.length() > 0) |
||||||
|
buffer.append(";"); //$NON-NLS-1$
|
||||||
|
buffer.append(header.toLowerCase()); |
||||||
|
} |
||||||
|
|
||||||
|
return buffer.toString(); |
||||||
|
} |
||||||
|
|
||||||
|
private static String canonicalizeHeaderString( |
||||||
|
Map<String, String> headers) { |
||||||
|
if (headers == null || headers.isEmpty()) { |
||||||
|
return ""; //$NON-NLS-1$
|
||||||
|
} |
||||||
|
|
||||||
|
List<String> sortedHeaders = new ArrayList<String>(); |
||||||
|
sortedHeaders.addAll(headers.keySet()); |
||||||
|
Collections.sort(sortedHeaders, String.CASE_INSENSITIVE_ORDER); |
||||||
|
|
||||||
|
StringBuilder buffer = new StringBuilder(); |
||||||
|
for (String key : sortedHeaders) { |
||||||
|
buffer.append(key.toLowerCase().replaceAll("\\s+", " ") + ":" //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
|
||||||
|
+ headers.get(key).replaceAll("\\s+", " ")); //$NON-NLS-1$//$NON-NLS-2$
|
||||||
|
buffer.append("\n"); //$NON-NLS-1$
|
||||||
|
} |
||||||
|
|
||||||
|
return buffer.toString(); |
||||||
|
} |
||||||
|
|
||||||
|
private static String dateStamp(Date now) { |
||||||
|
// TODO(ms) cache and reuse DateFormat instances
|
||||||
|
SimpleDateFormat dateStampFormat = new SimpleDateFormat( |
||||||
|
DATE_STRING_FORMAT); |
||||||
|
dateStampFormat.setTimeZone(new SimpleTimeZone(0, UTC)); |
||||||
|
String dateStamp = dateStampFormat.format(now); |
||||||
|
return dateStamp; |
||||||
|
} |
||||||
|
|
||||||
|
private static String dateTimeStampISO8601(Date now) { |
||||||
|
// TODO(ms) cache and reuse DateFormat instances
|
||||||
|
SimpleDateFormat dateTimeFormat = new SimpleDateFormat( |
||||||
|
ISO8601_BASIC_FORMAT); |
||||||
|
dateTimeFormat.setTimeZone(new SimpleTimeZone(0, UTC)); |
||||||
|
String dateTimeStamp = dateTimeFormat.format(now); |
||||||
|
return dateTimeStamp; |
||||||
|
} |
||||||
|
|
||||||
|
private static String scope(String region, String dateStamp) { |
||||||
|
String scope = String.format("%s/%s/%s/%s", dateStamp, region, S3, //$NON-NLS-1$
|
||||||
|
TERMINATOR); |
||||||
|
return scope; |
||||||
|
} |
||||||
|
|
||||||
|
private static String canonicalizeQueryString( |
||||||
|
Map<String, String> parameters) { |
||||||
|
if (parameters == null || parameters.isEmpty()) { |
||||||
|
return ""; //$NON-NLS-1$
|
||||||
|
} |
||||||
|
|
||||||
|
SortedMap<String, String> sorted = new TreeMap<String, String>(); |
||||||
|
|
||||||
|
Iterator<Map.Entry<String, String>> pairs = parameters.entrySet() |
||||||
|
.iterator(); |
||||||
|
while (pairs.hasNext()) { |
||||||
|
Map.Entry<String, String> pair = pairs.next(); |
||||||
|
String key = pair.getKey(); |
||||||
|
String value = pair.getValue(); |
||||||
|
sorted.put(urlEncode(key, false), urlEncode(value, false)); |
||||||
|
} |
||||||
|
|
||||||
|
StringBuilder builder = new StringBuilder(); |
||||||
|
pairs = sorted.entrySet().iterator(); |
||||||
|
while (pairs.hasNext()) { |
||||||
|
Map.Entry<String, String> pair = pairs.next(); |
||||||
|
builder.append(pair.getKey()); |
||||||
|
builder.append("="); //$NON-NLS-1$
|
||||||
|
builder.append(pair.getValue()); |
||||||
|
if (pairs.hasNext()) { |
||||||
|
builder.append("&"); //$NON-NLS-1$
|
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
return builder.toString(); |
||||||
|
} |
||||||
|
|
||||||
|
private static String canonicalRequest(URL endpoint, String httpMethod, |
||||||
|
String queryParameters, String canonicalizedHeaderNames, |
||||||
|
String canonicalizedHeaders, String bodyHash) { |
||||||
|
return String.format("%s\n%s\n%s\n%s\n%s\n%s", //$NON-NLS-1$
|
||||||
|
httpMethod, canonicalizeResourcePath(endpoint), |
||||||
|
queryParameters, canonicalizedHeaders, canonicalizedHeaderNames, |
||||||
|
bodyHash); |
||||||
|
} |
||||||
|
|
||||||
|
private static String canonicalizeResourcePath(URL endpoint) { |
||||||
|
if (endpoint == null) { |
||||||
|
return "/"; //$NON-NLS-1$
|
||||||
|
} |
||||||
|
String path = endpoint.getPath(); |
||||||
|
if (path == null || path.isEmpty()) { |
||||||
|
return "/"; //$NON-NLS-1$
|
||||||
|
} |
||||||
|
|
||||||
|
String encodedPath = urlEncode(path, true); |
||||||
|
if (encodedPath.startsWith("/")) { //$NON-NLS-1$
|
||||||
|
return encodedPath; |
||||||
|
} else { |
||||||
|
return "/" + encodedPath; //$NON-NLS-1$
|
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
private static byte[] hash(String s) { |
||||||
|
MessageDigest md = Constants.newMessageDigest(); |
||||||
|
md.update(s.getBytes(StandardCharsets.UTF_8)); |
||||||
|
return md.digest(); |
||||||
|
} |
||||||
|
|
||||||
|
private static byte[] sign(String stringData, byte[] key) { |
||||||
|
try { |
||||||
|
byte[] data = stringData.getBytes("UTF-8"); //$NON-NLS-1$
|
||||||
|
Mac mac = Mac.getInstance(HMACSHA256); |
||||||
|
mac.init(new SecretKeySpec(key, HMACSHA256)); |
||||||
|
return mac.doFinal(data); |
||||||
|
} catch (Exception e) { |
||||||
|
throw new RuntimeException(MessageFormat.format( |
||||||
|
LfsServerText.get().failedToCalcSignature, e.getMessage()), |
||||||
|
e); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
private static String stringToSign(String scheme, String algorithm, |
||||||
|
String dateTime, String scope, String canonicalRequest) { |
||||||
|
return String.format("%s-%s\n%s\n%s\n%s", //$NON-NLS-1$
|
||||||
|
scheme, algorithm, dateTime, scope, |
||||||
|
toHex(hash(canonicalRequest))); |
||||||
|
} |
||||||
|
|
||||||
|
private static String toHex(byte[] bytes) { |
||||||
|
StringBuilder builder = new StringBuilder(2 * bytes.length); |
||||||
|
for (byte b : bytes) { |
||||||
|
builder.append(HEX.charAt((b & 0xF0) >> 4)); |
||||||
|
builder.append(HEX.charAt(b & 0xF)); |
||||||
|
} |
||||||
|
return builder.toString(); |
||||||
|
} |
||||||
|
|
||||||
|
private static String urlEncode(String url, boolean keepPathSlash) { |
||||||
|
String encoded; |
||||||
|
try { |
||||||
|
encoded = URLEncoder.encode(url, StandardCharsets.UTF_8.name()); |
||||||
|
} catch (UnsupportedEncodingException e) { |
||||||
|
throw new RuntimeException(LfsServerText.get().unsupportedUtf8, e); |
||||||
|
} |
||||||
|
if (keepPathSlash) { |
||||||
|
encoded = encoded.replace("%2F", "/"); //$NON-NLS-1$ //$NON-NLS-2$
|
||||||
|
} |
||||||
|
return encoded; |
||||||
|
} |
||||||
|
|
||||||
|
private static byte[] createSignature(S3Config bucketConfig, |
||||||
|
String dateTimeStamp, String dateStamp, |
||||||
|
String scope, String canonicalRequest) { |
||||||
|
String stringToSign = stringToSign(SCHEME, ALGORITHM, dateTimeStamp, |
||||||
|
scope, canonicalRequest); |
||||||
|
|
||||||
|
byte[] signature = (SCHEME + bucketConfig.getSecretKey()).getBytes(); |
||||||
|
signature = sign(dateStamp, signature); |
||||||
|
signature = sign(bucketConfig.getRegion(), signature); |
||||||
|
signature = sign(S3, signature); |
||||||
|
signature = sign(TERMINATOR, signature); |
||||||
|
signature = sign(stringToSign, signature); |
||||||
|
return signature; |
||||||
|
} |
||||||
|
} |
||||||
Loading…
Reference in new issue