Wenjun Ruan
5 months ago
committed by
GitHub
49 changed files with 899 additions and 592 deletions
@ -0,0 +1,64 @@ |
|||||||
|
<?xml version="1.0" encoding="UTF-8"?> |
||||||
|
<!-- |
||||||
|
~ Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
~ contributor license agreements. See the NOTICE file distributed with |
||||||
|
~ this work for additional information regarding copyright ownership. |
||||||
|
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
~ (the "License"); you may not use this file except in compliance with |
||||||
|
~ the License. You may obtain a copy of the License at |
||||||
|
~ |
||||||
|
~ http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
~ |
||||||
|
~ Unless required by applicable law or agreed to in writing, software |
||||||
|
~ distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
~ See the License for the specific language governing permissions and |
||||||
|
~ limitations under the License. |
||||||
|
--> |
||||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" |
||||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> |
||||||
|
<modelVersion>4.0.0</modelVersion> |
||||||
|
<parent> |
||||||
|
<groupId>org.apache.dolphinscheduler</groupId> |
||||||
|
<artifactId>dolphinscheduler-authentication</artifactId> |
||||||
|
<version>dev-SNAPSHOT</version> |
||||||
|
</parent> |
||||||
|
|
||||||
|
<artifactId>dolphinscheduler-aws-authentication</artifactId> |
||||||
|
|
||||||
|
<dependencies> |
||||||
|
|
||||||
|
<dependency> |
||||||
|
<groupId>com.amazonaws</groupId> |
||||||
|
<artifactId>aws-java-sdk-emr</artifactId> |
||||||
|
</dependency> |
||||||
|
|
||||||
|
<dependency> |
||||||
|
<groupId>com.amazonaws</groupId> |
||||||
|
<artifactId>aws-java-sdk-s3</artifactId> |
||||||
|
</dependency> |
||||||
|
|
||||||
|
<dependency> |
||||||
|
<groupId>com.amazonaws</groupId> |
||||||
|
<artifactId>aws-java-sdk-sagemaker</artifactId> |
||||||
|
</dependency> |
||||||
|
|
||||||
|
<dependency> |
||||||
|
<groupId>com.amazonaws</groupId> |
||||||
|
<artifactId>aws-java-sdk-dms</artifactId> |
||||||
|
</dependency> |
||||||
|
|
||||||
|
<dependency> |
||||||
|
<groupId>software.amazon.awssdk</groupId> |
||||||
|
<artifactId>datasync</artifactId> |
||||||
|
</dependency> |
||||||
|
|
||||||
|
<dependency> |
||||||
|
<groupId>org.slf4j</groupId> |
||||||
|
<artifactId>slf4j-api</artifactId> |
||||||
|
<scope>provided</scope> |
||||||
|
</dependency> |
||||||
|
|
||||||
|
</dependencies> |
||||||
|
|
||||||
|
</project> |
||||||
@ -0,0 +1,74 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.authentication.aws; |
||||||
|
|
||||||
|
import static org.apache.dolphinscheduler.authentication.aws.AwsConfigurationKeys.AWS_AUTHENTICATION_TYPE; |
||||||
|
|
||||||
|
import java.util.Map; |
||||||
|
|
||||||
|
import lombok.experimental.UtilityClass; |
||||||
|
import lombok.extern.slf4j.Slf4j; |
||||||
|
|
||||||
|
import com.amazonaws.auth.AWSCredentialsProvider; |
||||||
|
import com.amazonaws.auth.AWSStaticCredentialsProvider; |
||||||
|
import com.amazonaws.auth.BasicAWSCredentials; |
||||||
|
import com.amazonaws.auth.InstanceProfileCredentialsProvider; |
||||||
|
|
||||||
|
@Slf4j |
||||||
|
@UtilityClass |
||||||
|
public class AWSCredentialsProviderFactor { |
||||||
|
|
||||||
|
public static AWSCredentialsProvider credentialsProvider(Map<String, String> awsProperties) { |
||||||
|
String awsAuthenticationType = awsProperties.getOrDefault( |
||||||
|
AWS_AUTHENTICATION_TYPE, AWSCredentialsProviderType.STATIC_CREDENTIALS_PROVIDER.getName()); |
||||||
|
AWSCredentialsProviderType awsCredentialsProviderType = |
||||||
|
AWSCredentialsProviderType.of(awsAuthenticationType).orElse(null); |
||||||
|
if (awsCredentialsProviderType == null) { |
||||||
|
throw new IllegalArgumentException( |
||||||
|
"The aws.credentials.provider.type: " + awsAuthenticationType + " is invalidated"); |
||||||
|
} |
||||||
|
switch (awsCredentialsProviderType) { |
||||||
|
case STATIC_CREDENTIALS_PROVIDER: |
||||||
|
return createAWSStaticCredentialsProvider(awsProperties); |
||||||
|
case INSTANCE_PROFILE_CREDENTIALS_PROVIDER: |
||||||
|
return createInstanceProfileCredentialsProvider(); |
||||||
|
default: |
||||||
|
throw new IllegalArgumentException( |
||||||
|
"The aws.credentials.provider.type: " + awsAuthenticationType + " is invalidated"); |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
private static AWSCredentialsProvider createAWSStaticCredentialsProvider(Map<String, String> awsProperties) { |
||||||
|
String awsAccessKeyId = awsProperties.get(AwsConfigurationKeys.AWS_ACCESS_KEY_ID); |
||||||
|
String awsSecretAccessKey = awsProperties.get(AwsConfigurationKeys.AWS_SECRET); |
||||||
|
final BasicAWSCredentials basicAWSCredentials = new BasicAWSCredentials(awsAccessKeyId, awsSecretAccessKey); |
||||||
|
AWSStaticCredentialsProvider awsStaticCredentialsProvider = |
||||||
|
new AWSStaticCredentialsProvider(basicAWSCredentials); |
||||||
|
log.info("AWSStaticCredentialsProvider created successfully"); |
||||||
|
return awsStaticCredentialsProvider; |
||||||
|
} |
||||||
|
|
||||||
|
private static AWSCredentialsProvider createInstanceProfileCredentialsProvider() { |
||||||
|
InstanceProfileCredentialsProvider instanceProfileCredentialsProvider = |
||||||
|
InstanceProfileCredentialsProvider.getInstance(); |
||||||
|
log.info("InstanceProfileCredentialsProvider created successfully"); |
||||||
|
return instanceProfileCredentialsProvider; |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
@ -0,0 +1,49 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.authentication.aws; |
||||||
|
|
||||||
|
import java.util.Optional; |
||||||
|
|
||||||
|
import lombok.Getter; |
||||||
|
|
||||||
|
@Getter |
||||||
|
public enum AWSCredentialsProviderType { |
||||||
|
|
||||||
|
STATIC_CREDENTIALS_PROVIDER("AWSStaticCredentialsProvider"), |
||||||
|
INSTANCE_PROFILE_CREDENTIALS_PROVIDER("InstanceProfileCredentialsProvider"), |
||||||
|
; |
||||||
|
|
||||||
|
private final String name; |
||||||
|
|
||||||
|
AWSCredentialsProviderType(String name) { |
||||||
|
this.name = name; |
||||||
|
} |
||||||
|
|
||||||
|
public static Optional<AWSCredentialsProviderType> of(String name) { |
||||||
|
if (name == null) { |
||||||
|
return Optional.empty(); |
||||||
|
} |
||||||
|
for (AWSCredentialsProviderType type : values()) { |
||||||
|
if (type.getName().equalsIgnoreCase(name)) { |
||||||
|
return Optional.of(type); |
||||||
|
} |
||||||
|
} |
||||||
|
return Optional.empty(); |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
@ -0,0 +1,53 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.authentication.aws; |
||||||
|
|
||||||
|
import java.util.Map; |
||||||
|
|
||||||
|
import lombok.experimental.UtilityClass; |
||||||
|
|
||||||
|
import com.amazonaws.auth.AWSCredentialsProvider; |
||||||
|
import com.amazonaws.client.builder.AwsClientBuilder; |
||||||
|
import com.amazonaws.regions.Regions; |
||||||
|
import com.amazonaws.services.databasemigrationservice.AWSDatabaseMigrationService; |
||||||
|
import com.amazonaws.services.databasemigrationservice.AWSDatabaseMigrationServiceClientBuilder; |
||||||
|
|
||||||
|
@UtilityClass |
||||||
|
public class AWSDatabaseMigrationServiceClientFactory { |
||||||
|
|
||||||
|
public AWSDatabaseMigrationService createAWSDatabaseMigrationServiceClient(Map<String, String> awsProperties) { |
||||||
|
AWSCredentialsProvider awsCredentialsProvider = AWSCredentialsProviderFactor.credentialsProvider(awsProperties); |
||||||
|
Regions regions = Regions.fromName(awsProperties.get(AwsConfigurationKeys.AWS_REGION)); |
||||||
|
String endpoint = awsProperties.get(AwsConfigurationKeys.AWS_ENDPOINT); |
||||||
|
|
||||||
|
if (endpoint != null && !endpoint.isEmpty()) { |
||||||
|
return AWSDatabaseMigrationServiceClientBuilder |
||||||
|
.standard() |
||||||
|
.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, regions.getName())) |
||||||
|
.withCredentials(awsCredentialsProvider) |
||||||
|
.build(); |
||||||
|
} else { |
||||||
|
return AWSDatabaseMigrationServiceClientBuilder |
||||||
|
.standard() |
||||||
|
.withCredentials(awsCredentialsProvider) |
||||||
|
.withRegion(regions) |
||||||
|
.build(); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
@ -0,0 +1,53 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.authentication.aws; |
||||||
|
|
||||||
|
import java.util.Map; |
||||||
|
|
||||||
|
import lombok.experimental.UtilityClass; |
||||||
|
|
||||||
|
import com.amazonaws.auth.AWSCredentialsProvider; |
||||||
|
import com.amazonaws.client.builder.AwsClientBuilder; |
||||||
|
import com.amazonaws.regions.Regions; |
||||||
|
import com.amazonaws.services.elasticmapreduce.AmazonElasticMapReduce; |
||||||
|
import com.amazonaws.services.elasticmapreduce.AmazonElasticMapReduceClientBuilder; |
||||||
|
|
||||||
|
@UtilityClass |
||||||
|
public class AmazonElasticMapReduceClientFactory { |
||||||
|
|
||||||
|
public AmazonElasticMapReduce createAmazonElasticMapReduceClient(Map<String, String> awsProperties) { |
||||||
|
AWSCredentialsProvider awsCredentialsProvider = AWSCredentialsProviderFactor.credentialsProvider(awsProperties); |
||||||
|
Regions regions = Regions.fromName(awsProperties.get(AwsConfigurationKeys.AWS_REGION)); |
||||||
|
String endpoint = awsProperties.get(AwsConfigurationKeys.AWS_ENDPOINT); |
||||||
|
|
||||||
|
if (endpoint != null && !endpoint.isEmpty()) { |
||||||
|
return AmazonElasticMapReduceClientBuilder |
||||||
|
.standard() |
||||||
|
.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, regions.getName())) |
||||||
|
.withCredentials(awsCredentialsProvider) |
||||||
|
.build(); |
||||||
|
} else { |
||||||
|
return AmazonElasticMapReduceClientBuilder |
||||||
|
.standard() |
||||||
|
.withCredentials(awsCredentialsProvider) |
||||||
|
.withRegion(regions) |
||||||
|
.build(); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
@ -0,0 +1,54 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.authentication.aws; |
||||||
|
|
||||||
|
import java.util.Map; |
||||||
|
|
||||||
|
import lombok.experimental.UtilityClass; |
||||||
|
|
||||||
|
import com.amazonaws.auth.AWSCredentialsProvider; |
||||||
|
import com.amazonaws.client.builder.AwsClientBuilder; |
||||||
|
import com.amazonaws.regions.Regions; |
||||||
|
import com.amazonaws.services.s3.AmazonS3; |
||||||
|
import com.amazonaws.services.s3.AmazonS3ClientBuilder; |
||||||
|
|
||||||
|
@UtilityClass |
||||||
|
public class AmazonS3ClientFactory { |
||||||
|
|
||||||
|
public AmazonS3 createAmazonS3Client(Map<String, String> awsProperties) { |
||||||
|
AWSCredentialsProvider awsCredentialsProvider = AWSCredentialsProviderFactor.credentialsProvider(awsProperties); |
||||||
|
Regions regions = Regions.fromName(awsProperties.get(AwsConfigurationKeys.AWS_REGION)); |
||||||
|
String endpoint = awsProperties.get(AwsConfigurationKeys.AWS_ENDPOINT); |
||||||
|
|
||||||
|
if (endpoint != null && !endpoint.isEmpty()) { |
||||||
|
return AmazonS3ClientBuilder |
||||||
|
.standard() |
||||||
|
.withPathStyleAccessEnabled(true) |
||||||
|
.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, regions.getName())) |
||||||
|
.withCredentials(awsCredentialsProvider) |
||||||
|
.build(); |
||||||
|
} else { |
||||||
|
return AmazonS3ClientBuilder |
||||||
|
.standard() |
||||||
|
.withCredentials(awsCredentialsProvider) |
||||||
|
.withRegion(regions) |
||||||
|
.build(); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
@ -0,0 +1,53 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.authentication.aws; |
||||||
|
|
||||||
|
import java.util.Map; |
||||||
|
|
||||||
|
import lombok.experimental.UtilityClass; |
||||||
|
|
||||||
|
import com.amazonaws.auth.AWSCredentialsProvider; |
||||||
|
import com.amazonaws.client.builder.AwsClientBuilder; |
||||||
|
import com.amazonaws.regions.Regions; |
||||||
|
import com.amazonaws.services.sagemaker.AmazonSageMaker; |
||||||
|
import com.amazonaws.services.sagemaker.AmazonSageMakerClientBuilder; |
||||||
|
|
||||||
|
@UtilityClass |
||||||
|
public class AmazonSageMakerClientFactory { |
||||||
|
|
||||||
|
public AmazonSageMaker createAmazonSageMakerClient(Map<String, String> awsProperties) { |
||||||
|
AWSCredentialsProvider awsCredentialsProvider = AWSCredentialsProviderFactor.credentialsProvider(awsProperties); |
||||||
|
Regions regions = Regions.fromName(awsProperties.get(AwsConfigurationKeys.AWS_REGION)); |
||||||
|
String endpoint = awsProperties.get(AwsConfigurationKeys.AWS_ENDPOINT); |
||||||
|
|
||||||
|
if (endpoint != null && !endpoint.isEmpty()) { |
||||||
|
return AmazonSageMakerClientBuilder |
||||||
|
.standard() |
||||||
|
.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, regions.getName())) |
||||||
|
.withCredentials(awsCredentialsProvider) |
||||||
|
.build(); |
||||||
|
} else { |
||||||
|
return AmazonSageMakerClientBuilder |
||||||
|
.standard() |
||||||
|
.withCredentials(awsCredentialsProvider) |
||||||
|
.withRegion(regions) |
||||||
|
.build(); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
@ -0,0 +1,28 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.authentication.aws; |
||||||
|
|
||||||
|
public class AwsConfigurationKeys { |
||||||
|
|
||||||
|
public static final String AWS_AUTHENTICATION_TYPE = "credentials.provider.type"; |
||||||
|
public static final String AWS_REGION = "region"; |
||||||
|
public static final String AWS_ENDPOINT = "endpoint"; |
||||||
|
|
||||||
|
public static final String AWS_ACCESS_KEY_ID = "access.key.id"; |
||||||
|
public static final String AWS_SECRET = "access.key.secret"; |
||||||
|
} |
||||||
@ -0,0 +1,47 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.authentication.aws; |
||||||
|
|
||||||
|
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; |
||||||
|
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; |
||||||
|
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; |
||||||
|
import software.amazon.awssdk.regions.Region; |
||||||
|
import software.amazon.awssdk.services.datasync.DataSyncClient; |
||||||
|
|
||||||
|
import java.util.Map; |
||||||
|
|
||||||
|
import lombok.experimental.UtilityClass; |
||||||
|
|
||||||
|
@UtilityClass |
||||||
|
public class DataSyncClientFactory { |
||||||
|
|
||||||
|
public DataSyncClient createDataSyncClient(Map<String, String> awsProperties) { |
||||||
|
// todo: upgrade the version of aws sdk
|
||||||
|
String awsAccessKeyId = awsProperties.get(AwsConfigurationKeys.AWS_ACCESS_KEY_ID); |
||||||
|
String awsSecretAccessKey = awsProperties.get(AwsConfigurationKeys.AWS_SECRET); |
||||||
|
final AwsBasicCredentials basicAWSCredentials = AwsBasicCredentials.create(awsAccessKeyId, awsSecretAccessKey); |
||||||
|
final AwsCredentialsProvider awsCredentialsProvider = StaticCredentialsProvider.create(basicAWSCredentials); |
||||||
|
|
||||||
|
// create a datasync client
|
||||||
|
return DataSyncClient.builder() |
||||||
|
.region(Region.of(awsProperties.get(AwsConfigurationKeys.AWS_REGION))) |
||||||
|
.credentialsProvider(awsCredentialsProvider) |
||||||
|
.build(); |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
@ -0,0 +1,65 @@ |
|||||||
|
# |
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
# contributor license agreements. See the NOTICE file distributed with |
||||||
|
# this work for additional information regarding copyright ownership. |
||||||
|
# The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
# (the "License"); you may not use this file except in compliance with |
||||||
|
# the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, software |
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
# See the License for the specific language governing permissions and |
||||||
|
# limitations under the License. |
||||||
|
# |
||||||
|
|
||||||
|
aws: |
||||||
|
s3: |
||||||
|
# The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider |
||||||
|
# AWSStaticCredentialsProvider: use the access key and secret key to authenticate |
||||||
|
# InstanceProfileCredentialsProvider: use the IAM role to authenticate |
||||||
|
credentials.provider.type: AWSStaticCredentialsProvider |
||||||
|
access.key.id: accessKey123 |
||||||
|
access.key.secret: secretKey123 |
||||||
|
region: us-east-1 |
||||||
|
bucket.name: dolphinscheduler |
||||||
|
endpoint: http://s3:9000 |
||||||
|
emr: |
||||||
|
# The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider |
||||||
|
# AWSStaticCredentialsProvider: use the access key and secret key to authenticate |
||||||
|
# InstanceProfileCredentialsProvider: use the IAM role to authenticate |
||||||
|
credentials.provider.type: AWSStaticCredentialsProvider |
||||||
|
access.key.id: minioadmin |
||||||
|
access.key.secret: minioadmin |
||||||
|
region: cn-north-1 |
||||||
|
endpoint: http://localhost:9000 |
||||||
|
sagemaker: |
||||||
|
# The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider |
||||||
|
# AWSStaticCredentialsProvider: use the access key and secret key to authenticate |
||||||
|
# InstanceProfileCredentialsProvider: use the IAM role to authenticate |
||||||
|
credentials.provider.type: AWSStaticCredentialsProvider |
||||||
|
access.key.id: minioadmin |
||||||
|
access.key.secret: minioadmin |
||||||
|
region: cn-north-1 |
||||||
|
endpoint: http://localhost:9000 |
||||||
|
dms: |
||||||
|
# The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider |
||||||
|
# AWSStaticCredentialsProvider: use the access key and secret key to authenticate |
||||||
|
# InstanceProfileCredentialsProvider: use the IAM role to authenticate |
||||||
|
credentials.provider.type: AWSStaticCredentialsProvider |
||||||
|
access.key.id: minioadmin |
||||||
|
access.key.secret: minioadmin |
||||||
|
region: cn-north-1 |
||||||
|
endpoint: http://localhost:9000 |
||||||
|
datasync: |
||||||
|
# The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider |
||||||
|
# AWSStaticCredentialsProvider: use the access key and secret key to authenticate |
||||||
|
# InstanceProfileCredentialsProvider: use the IAM role to authenticate |
||||||
|
credentials.provider.type: AWSStaticCredentialsProvider |
||||||
|
access.key.id: minioadmin |
||||||
|
access.key.secret: minioadmin |
||||||
|
region: cn-north-1 |
||||||
|
endpoint: http://localhost:9000 |
||||||
|
|
||||||
@ -0,0 +1,46 @@ |
|||||||
|
<?xml version="1.0" encoding="UTF-8"?> |
||||||
|
<!-- |
||||||
|
~ Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
~ contributor license agreements. See the NOTICE file distributed with |
||||||
|
~ this work for additional information regarding copyright ownership. |
||||||
|
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
~ (the "License"); you may not use this file except in compliance with |
||||||
|
~ the License. You may obtain a copy of the License at |
||||||
|
~ |
||||||
|
~ http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
~ |
||||||
|
~ Unless required by applicable law or agreed to in writing, software |
||||||
|
~ distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
~ See the License for the specific language governing permissions and |
||||||
|
~ limitations under the License. |
||||||
|
--> |
||||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" |
||||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> |
||||||
|
<modelVersion>4.0.0</modelVersion> |
||||||
|
<parent> |
||||||
|
<groupId>org.apache.dolphinscheduler</groupId> |
||||||
|
<artifactId>dolphinscheduler</artifactId> |
||||||
|
<version>dev-SNAPSHOT</version> |
||||||
|
</parent> |
||||||
|
|
||||||
|
<artifactId>dolphinscheduler-authentication</artifactId> |
||||||
|
<packaging>pom</packaging> |
||||||
|
|
||||||
|
<modules> |
||||||
|
<module>dolphinscheduler-aws-authentication</module> |
||||||
|
</modules> |
||||||
|
|
||||||
|
<dependencyManagement> |
||||||
|
<dependencies> |
||||||
|
<dependency> |
||||||
|
<groupId>org.apache.dolphinscheduler</groupId> |
||||||
|
<artifactId>dolphinscheduler-bom</artifactId> |
||||||
|
<version>${project.version}</version> |
||||||
|
<type>pom</type> |
||||||
|
<scope>import</scope> |
||||||
|
</dependency> |
||||||
|
</dependencies> |
||||||
|
</dependencyManagement> |
||||||
|
|
||||||
|
</project> |
||||||
@ -0,0 +1,65 @@ |
|||||||
|
# |
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
# contributor license agreements. See the NOTICE file distributed with |
||||||
|
# this work for additional information regarding copyright ownership. |
||||||
|
# The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
# (the "License"); you may not use this file except in compliance with |
||||||
|
# the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, software |
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
# See the License for the specific language governing permissions and |
||||||
|
# limitations under the License. |
||||||
|
# |
||||||
|
|
||||||
|
aws: |
||||||
|
s3: |
||||||
|
# The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider |
||||||
|
# AWSStaticCredentialsProvider: use the access key and secret key to authenticate |
||||||
|
# InstanceProfileCredentialsProvider: use the IAM role to authenticate |
||||||
|
credentials.provider.type: AWSStaticCredentialsProvider |
||||||
|
access.key.id: accessKey123 |
||||||
|
access.key.secret: secretKey123 |
||||||
|
region: us-east-1 |
||||||
|
bucket.name: dolphinscheduler |
||||||
|
endpoint: http://s3:9000 |
||||||
|
emr: |
||||||
|
# The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider |
||||||
|
# AWSStaticCredentialsProvider: use the access key and secret key to authenticate |
||||||
|
# InstanceProfileCredentialsProvider: use the IAM role to authenticate |
||||||
|
credentials.provider.type: AWSStaticCredentialsProvider |
||||||
|
access.key.id: minioadmin |
||||||
|
access.key.secret: minioadmin |
||||||
|
region: cn-north-1 |
||||||
|
endpoint: http://localhost:9000 |
||||||
|
sagemaker: |
||||||
|
# The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider |
||||||
|
# AWSStaticCredentialsProvider: use the access key and secret key to authenticate |
||||||
|
# InstanceProfileCredentialsProvider: use the IAM role to authenticate |
||||||
|
credentials.provider.type: AWSStaticCredentialsProvider |
||||||
|
access.key.id: minioadmin |
||||||
|
access.key.secret: minioadmin |
||||||
|
region: cn-north-1 |
||||||
|
endpoint: http://localhost:9000 |
||||||
|
dms: |
||||||
|
# The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider |
||||||
|
# AWSStaticCredentialsProvider: use the access key and secret key to authenticate |
||||||
|
# InstanceProfileCredentialsProvider: use the IAM role to authenticate |
||||||
|
credentials.provider.type: AWSStaticCredentialsProvider |
||||||
|
access.key.id: minioadmin |
||||||
|
access.key.secret: minioadmin |
||||||
|
region: cn-north-1 |
||||||
|
endpoint: http://localhost:9000 |
||||||
|
datasync: |
||||||
|
# The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider |
||||||
|
# AWSStaticCredentialsProvider: use the access key and secret key to authenticate |
||||||
|
# InstanceProfileCredentialsProvider: use the IAM role to authenticate |
||||||
|
credentials.provider.type: AWSStaticCredentialsProvider |
||||||
|
access.key.id: minioadmin |
||||||
|
access.key.secret: minioadmin |
||||||
|
region: cn-north-1 |
||||||
|
endpoint: http://localhost:9000 |
||||||
|
|
||||||
Loading…
Reference in new issue