Browse Source
* Add the ProgramType parameter to distinguish task types * EmrAddStepsTask supports Add-Steps * UI supports Add-Steps * EmrTask modify the name of the class to EmrJobFlowTask * add ERM Task abstract base class AbstractEmrTask * add testcase for EmrAddStepsTask * Modifying help Documents3.1.0-release
ZhaoGuodong
2 years ago
committed by
GitHub
21 changed files with 769 additions and 125 deletions
|
After Width: | Height: | Size: 129 KiB |
|
After Width: | Height: | Size: 130 KiB |
|
After Width: | Height: | Size: 136 KiB |
@ -0,0 +1,113 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.plugin.task.emr; |
||||||
|
|
||||||
|
import static com.fasterxml.jackson.databind.DeserializationFeature.ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT; |
||||||
|
import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES; |
||||||
|
import static com.fasterxml.jackson.databind.DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL; |
||||||
|
import static com.fasterxml.jackson.databind.MapperFeature.REQUIRE_SETTERS_FOR_GETTERS; |
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.plugin.task.api.AbstractTaskExecutor; |
||||||
|
import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; |
||||||
|
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; |
||||||
|
import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; |
||||||
|
import org.apache.dolphinscheduler.spi.utils.JSONUtils; |
||||||
|
import org.apache.dolphinscheduler.spi.utils.PropertyUtils; |
||||||
|
|
||||||
|
import java.util.TimeZone; |
||||||
|
|
||||||
|
import com.amazonaws.auth.AWSCredentialsProvider; |
||||||
|
import com.amazonaws.auth.AWSStaticCredentialsProvider; |
||||||
|
import com.amazonaws.auth.BasicAWSCredentials; |
||||||
|
import com.amazonaws.services.elasticmapreduce.AmazonElasticMapReduce; |
||||||
|
import com.amazonaws.services.elasticmapreduce.AmazonElasticMapReduceClientBuilder; |
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper; |
||||||
|
import com.fasterxml.jackson.databind.PropertyNamingStrategy; |
||||||
|
|
||||||
|
/** |
||||||
|
* ERM Task abstract base class
|
||||||
|
* |
||||||
|
* @since v3.1.0 |
||||||
|
*/ |
||||||
|
public abstract class AbstractEmrTask extends AbstractTaskExecutor { |
||||||
|
|
||||||
|
final TaskExecutionContext taskExecutionContext; |
||||||
|
EmrParameters emrParameters; |
||||||
|
AmazonElasticMapReduce emrClient; |
||||||
|
String clusterId; |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* config ObjectMapper features and propertyNamingStrategy |
||||||
|
* use UpperCamelCaseStrategy support capital letters parse |
||||||
|
* |
||||||
|
* @see PropertyNamingStrategy.UpperCamelCaseStrategy |
||||||
|
*/ |
||||||
|
static final ObjectMapper objectMapper = new ObjectMapper() |
||||||
|
.configure(FAIL_ON_UNKNOWN_PROPERTIES, false) |
||||||
|
.configure(ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT, true) |
||||||
|
.configure(READ_UNKNOWN_ENUM_VALUES_AS_NULL, true) |
||||||
|
.configure(REQUIRE_SETTERS_FOR_GETTERS, true) |
||||||
|
.setTimeZone(TimeZone.getDefault()) |
||||||
|
.setPropertyNamingStrategy(new PropertyNamingStrategy.UpperCamelCaseStrategy()); |
||||||
|
|
||||||
|
/** |
||||||
|
* constructor |
||||||
|
* |
||||||
|
* @param taskExecutionContext taskExecutionContext |
||||||
|
*/ |
||||||
|
protected AbstractEmrTask(TaskExecutionContext taskExecutionContext) { |
||||||
|
super(taskExecutionContext); |
||||||
|
this.taskExecutionContext = taskExecutionContext; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public void init() { |
||||||
|
final String taskParams = taskExecutionContext.getTaskParams(); |
||||||
|
logger.info("emr task params:{}", taskParams); |
||||||
|
emrParameters = JSONUtils.parseObject(taskParams, EmrParameters.class); |
||||||
|
if (emrParameters == null || !emrParameters.checkParameters()) { |
||||||
|
throw new EmrTaskException("emr task params is not valid"); |
||||||
|
} |
||||||
|
emrClient = createEmrClient(); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public AbstractParameters getParameters() { |
||||||
|
return emrParameters; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* create emr client from BasicAWSCredentials |
||||||
|
* |
||||||
|
* @return AmazonElasticMapReduce |
||||||
|
*/ |
||||||
|
private AmazonElasticMapReduce createEmrClient() { |
||||||
|
|
||||||
|
final String awsAccessKeyId = PropertyUtils.getString(TaskConstants.AWS_ACCESS_KEY_ID); |
||||||
|
final String awsSecretAccessKey = PropertyUtils.getString(TaskConstants.AWS_SECRET_ACCESS_KEY); |
||||||
|
final String awsRegion = PropertyUtils.getString(TaskConstants.AWS_REGION); |
||||||
|
final BasicAWSCredentials basicAWSCredentials = new BasicAWSCredentials(awsAccessKeyId, awsSecretAccessKey); |
||||||
|
final AWSCredentialsProvider awsCredentialsProvider = new AWSStaticCredentialsProvider(basicAWSCredentials); |
||||||
|
// create an EMR client
|
||||||
|
return AmazonElasticMapReduceClientBuilder.standard() |
||||||
|
.withCredentials(awsCredentialsProvider) |
||||||
|
.withRegion(awsRegion) |
||||||
|
.build(); |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,177 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.plugin.task.emr; |
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; |
||||||
|
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; |
||||||
|
|
||||||
|
import java.util.HashSet; |
||||||
|
import java.util.concurrent.TimeUnit; |
||||||
|
|
||||||
|
import com.amazonaws.SdkBaseException; |
||||||
|
import com.amazonaws.services.elasticmapreduce.model.AddJobFlowStepsRequest; |
||||||
|
import com.amazonaws.services.elasticmapreduce.model.AddJobFlowStepsResult; |
||||||
|
import com.amazonaws.services.elasticmapreduce.model.CancelStepsInfo; |
||||||
|
import com.amazonaws.services.elasticmapreduce.model.CancelStepsRequest; |
||||||
|
import com.amazonaws.services.elasticmapreduce.model.CancelStepsRequestStatus; |
||||||
|
import com.amazonaws.services.elasticmapreduce.model.CancelStepsResult; |
||||||
|
import com.amazonaws.services.elasticmapreduce.model.DescribeStepRequest; |
||||||
|
import com.amazonaws.services.elasticmapreduce.model.DescribeStepResult; |
||||||
|
import com.amazonaws.services.elasticmapreduce.model.StepState; |
||||||
|
import com.amazonaws.services.elasticmapreduce.model.StepStatus; |
||||||
|
import com.fasterxml.jackson.core.JsonProcessingException; |
||||||
|
import com.google.common.collect.Sets; |
||||||
|
|
||||||
|
/** |
||||||
|
* AddJobFlowSteps task executor |
||||||
|
* |
||||||
|
* @since v3.1.0 |
||||||
|
*/ |
||||||
|
public class EmrAddStepsTask extends AbstractEmrTask { |
||||||
|
|
||||||
|
private String stepId; |
||||||
|
|
||||||
|
private final HashSet<String> waitingStateSet = Sets.newHashSet( |
||||||
|
StepState.PENDING.toString(), |
||||||
|
StepState.CANCEL_PENDING.toString(), |
||||||
|
StepState.RUNNING.toString() |
||||||
|
); |
||||||
|
|
||||||
|
/** |
||||||
|
* constructor |
||||||
|
* |
||||||
|
* @param taskExecutionContext taskExecutionContext |
||||||
|
*/ |
||||||
|
protected EmrAddStepsTask(TaskExecutionContext taskExecutionContext) { |
||||||
|
super(taskExecutionContext); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public void handle() throws InterruptedException { |
||||||
|
StepStatus stepStatus = null; |
||||||
|
try { |
||||||
|
AddJobFlowStepsRequest addJobFlowStepsRequest = createAddJobFlowStepsRequest(); |
||||||
|
|
||||||
|
// submit addJobFlowStepsRequest to aws
|
||||||
|
AddJobFlowStepsResult result = emrClient.addJobFlowSteps(addJobFlowStepsRequest); |
||||||
|
|
||||||
|
clusterId = addJobFlowStepsRequest.getJobFlowId(); |
||||||
|
stepId = result.getStepIds().get(0); |
||||||
|
// use clusterId-stepId as appIds
|
||||||
|
setAppIds(clusterId + TaskConstants.SUBTRACT_STRING + stepId); |
||||||
|
|
||||||
|
stepStatus = getStepStatus(); |
||||||
|
|
||||||
|
while (waitingStateSet.contains(stepStatus.getState())) { |
||||||
|
TimeUnit.SECONDS.sleep(10); |
||||||
|
stepStatus = getStepStatus(); |
||||||
|
} |
||||||
|
|
||||||
|
} catch (EmrTaskException | SdkBaseException e) { |
||||||
|
logger.error("emr task submit failed with error", e); |
||||||
|
} finally { |
||||||
|
final int exitStatusCode = calculateExitStatusCode(stepStatus); |
||||||
|
setExitStatusCode(exitStatusCode); |
||||||
|
logger.info("emr task finished with step status : {}", stepStatus); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* parse json string to AddJobFlowStepsRequest |
||||||
|
* |
||||||
|
* @return AddJobFlowStepsRequest |
||||||
|
*/ |
||||||
|
private AddJobFlowStepsRequest createAddJobFlowStepsRequest() { |
||||||
|
|
||||||
|
final AddJobFlowStepsRequest addJobFlowStepsRequest; |
||||||
|
try { |
||||||
|
addJobFlowStepsRequest = objectMapper.readValue(emrParameters.getStepsDefineJson(), AddJobFlowStepsRequest.class); |
||||||
|
} catch (JsonProcessingException e) { |
||||||
|
throw new EmrTaskException("can not parse AddJobFlowStepsRequest from json", e); |
||||||
|
} |
||||||
|
|
||||||
|
// When a single task definition is associated with multiple steps, the state tracking will have high complexity.
|
||||||
|
// Therefore, A task definition only supports the association of a single step, which can better ensure the reliability of the task state.
|
||||||
|
if (addJobFlowStepsRequest.getSteps().size() > 1) { |
||||||
|
throw new EmrTaskException("ds emr addJobFlowStepsTask only support one step"); |
||||||
|
} |
||||||
|
|
||||||
|
return addJobFlowStepsRequest; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* calculate task exitStatusCode |
||||||
|
* |
||||||
|
* @param stepStatus aws emr execution status details of the cluster step. |
||||||
|
* @return exitStatusCode |
||||||
|
*/ |
||||||
|
private int calculateExitStatusCode(StepStatus stepStatus) { |
||||||
|
if (stepStatus == null) { |
||||||
|
return TaskConstants.EXIT_CODE_FAILURE; |
||||||
|
} else { |
||||||
|
String state = stepStatus.getState(); |
||||||
|
StepState stepState = StepState.valueOf(state); |
||||||
|
switch (stepState) { |
||||||
|
case COMPLETED: |
||||||
|
return TaskConstants.EXIT_CODE_SUCCESS; |
||||||
|
case CANCELLED: |
||||||
|
return TaskConstants.EXIT_CODE_KILL; |
||||||
|
default: |
||||||
|
return TaskConstants.EXIT_CODE_FAILURE; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
private StepStatus getStepStatus() { |
||||||
|
DescribeStepRequest describeStepRequest = new DescribeStepRequest().withClusterId(clusterId).withStepId(stepId); |
||||||
|
DescribeStepResult result = emrClient.describeStep(describeStepRequest); |
||||||
|
if (result == null) { |
||||||
|
throw new EmrTaskException("fetch step status failed"); |
||||||
|
} |
||||||
|
StepStatus stepStatus = result.getStep().getStatus(); |
||||||
|
logger.info("emr step [clusterId:{}, stepId:{}] running with status:{}", clusterId, stepId, stepStatus); |
||||||
|
return stepStatus; |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public void cancelApplication(boolean status) throws Exception { |
||||||
|
super.cancelApplication(status); |
||||||
|
logger.info("trying cancel emr step, taskId:{}, clusterId:{}, stepId:{}", this.taskExecutionContext.getTaskInstanceId(), clusterId, stepId); |
||||||
|
CancelStepsRequest cancelStepsRequest = new CancelStepsRequest().withClusterId(clusterId).withStepIds(stepId); |
||||||
|
CancelStepsResult cancelStepsResult = emrClient.cancelSteps(cancelStepsRequest); |
||||||
|
|
||||||
|
if (cancelStepsResult == null) { |
||||||
|
throw new EmrTaskException("cancel emr step failed"); |
||||||
|
} |
||||||
|
|
||||||
|
CancelStepsInfo cancelEmrStepInfo = cancelStepsResult.getCancelStepsInfoList() |
||||||
|
.stream() |
||||||
|
.filter(cancelStepsInfo -> cancelStepsInfo.getStepId().equals(stepId)) |
||||||
|
.findFirst() |
||||||
|
.orElseThrow(() -> new EmrTaskException("cancel emr step failed")); |
||||||
|
|
||||||
|
if (CancelStepsRequestStatus.FAILED.toString().equals(cancelEmrStepInfo.getStatus())) { |
||||||
|
throw new EmrTaskException("cancel emr step failed, message:" + cancelEmrStepInfo.getReason()); |
||||||
|
} |
||||||
|
|
||||||
|
logger.info("the result of cancel emr step is:{}", cancelStepsResult); |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
@ -0,0 +1,34 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.plugin.task.emr; |
||||||
|
|
||||||
|
/** |
||||||
|
* emr program type |
||||||
|
* |
||||||
|
* @since v3.1.0 |
||||||
|
*/ |
||||||
|
public enum ProgramType { |
||||||
|
/** |
||||||
|
* RunJobFlow |
||||||
|
*/ |
||||||
|
RUN_JOB_FLOW, |
||||||
|
/** |
||||||
|
* AddJobFlowSteps |
||||||
|
*/ |
||||||
|
ADD_JOB_FLOW_STEPS |
||||||
|
} |
||||||
@ -0,0 +1,198 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.plugin.task.emr; |
||||||
|
|
||||||
|
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_FAILURE; |
||||||
|
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_KILL; |
||||||
|
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_SUCCESS; |
||||||
|
|
||||||
|
import static org.mockito.ArgumentMatchers.any; |
||||||
|
import static org.powermock.api.mockito.PowerMockito.doReturn; |
||||||
|
import static org.powermock.api.mockito.PowerMockito.mock; |
||||||
|
import static org.powermock.api.mockito.PowerMockito.mockStatic; |
||||||
|
import static org.powermock.api.mockito.PowerMockito.spy; |
||||||
|
import static org.powermock.api.mockito.PowerMockito.when; |
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; |
||||||
|
import org.apache.dolphinscheduler.spi.utils.JSONUtils; |
||||||
|
|
||||||
|
import org.apache.commons.io.IOUtils; |
||||||
|
|
||||||
|
import java.io.InputStream; |
||||||
|
import java.nio.charset.StandardCharsets; |
||||||
|
import java.util.Collections; |
||||||
|
|
||||||
|
import org.junit.Assert; |
||||||
|
import org.junit.Before; |
||||||
|
import org.junit.Test; |
||||||
|
import org.junit.runner.RunWith; |
||||||
|
import org.powermock.api.mockito.PowerMockito; |
||||||
|
import org.powermock.core.classloader.annotations.PowerMockIgnore; |
||||||
|
import org.powermock.core.classloader.annotations.PrepareForTest; |
||||||
|
import org.powermock.modules.junit4.PowerMockRunner; |
||||||
|
|
||||||
|
import com.amazonaws.services.elasticmapreduce.AmazonElasticMapReduce; |
||||||
|
import com.amazonaws.services.elasticmapreduce.AmazonElasticMapReduceClientBuilder; |
||||||
|
import com.amazonaws.services.elasticmapreduce.model.AddJobFlowStepsResult; |
||||||
|
import com.amazonaws.services.elasticmapreduce.model.AmazonElasticMapReduceException; |
||||||
|
import com.amazonaws.services.elasticmapreduce.model.DescribeStepResult; |
||||||
|
import com.amazonaws.services.elasticmapreduce.model.Step; |
||||||
|
import com.amazonaws.services.elasticmapreduce.model.StepState; |
||||||
|
import com.amazonaws.services.elasticmapreduce.model.StepStatus; |
||||||
|
|
||||||
|
/** |
||||||
|
* EmrAddStepsTask Test |
||||||
|
* |
||||||
|
* @since v3.1.0 |
||||||
|
*/ |
||||||
|
@RunWith(PowerMockRunner.class) |
||||||
|
@PrepareForTest({ |
||||||
|
AmazonElasticMapReduceClientBuilder.class, |
||||||
|
EmrAddStepsTask.class, |
||||||
|
AmazonElasticMapReduce.class, |
||||||
|
JSONUtils.class |
||||||
|
}) |
||||||
|
@PowerMockIgnore({"javax.*"}) |
||||||
|
public class EmrAddStepsTaskTest { |
||||||
|
|
||||||
|
private final StepStatus pendingState = |
||||||
|
new StepStatus().withState(StepState.PENDING); |
||||||
|
|
||||||
|
private final StepStatus runningState = |
||||||
|
new StepStatus().withState(StepState.RUNNING); |
||||||
|
|
||||||
|
private final StepStatus completedState = |
||||||
|
new StepStatus().withState(StepState.COMPLETED); |
||||||
|
|
||||||
|
private final StepStatus cancelledState = |
||||||
|
new StepStatus().withState(StepState.CANCELLED); |
||||||
|
|
||||||
|
private final StepStatus failedState = |
||||||
|
new StepStatus().withState(StepState.FAILED); |
||||||
|
|
||||||
|
private EmrAddStepsTask emrAddStepsTask; |
||||||
|
private AmazonElasticMapReduce emrClient; |
||||||
|
private Step step; |
||||||
|
|
||||||
|
@Before |
||||||
|
public void before() throws Exception { |
||||||
|
// mock EmrParameters and EmrAddStepsTask
|
||||||
|
EmrParameters emrParameters = buildEmrTaskParameters(); |
||||||
|
String emrParametersString = JSONUtils.toJsonString(emrParameters); |
||||||
|
TaskExecutionContext taskExecutionContext = PowerMockito.mock(TaskExecutionContext.class); |
||||||
|
when(taskExecutionContext.getTaskParams()).thenReturn(emrParametersString); |
||||||
|
emrAddStepsTask = spy(new EmrAddStepsTask(taskExecutionContext)); |
||||||
|
|
||||||
|
// mock emrClient and behavior
|
||||||
|
emrClient = mock(AmazonElasticMapReduce.class); |
||||||
|
|
||||||
|
AddJobFlowStepsResult addJobFlowStepsResult = mock(AddJobFlowStepsResult.class); |
||||||
|
when(emrClient.addJobFlowSteps(any())).thenReturn(addJobFlowStepsResult); |
||||||
|
when(addJobFlowStepsResult.getStepIds()).thenReturn(Collections.singletonList("step-xx")); |
||||||
|
|
||||||
|
doReturn(emrClient).when(emrAddStepsTask, "createEmrClient"); |
||||||
|
DescribeStepResult describeStepResult = mock(DescribeStepResult.class); |
||||||
|
when(emrClient.describeStep(any())).thenReturn(describeStepResult); |
||||||
|
|
||||||
|
// mock step
|
||||||
|
step = mock(Step.class); |
||||||
|
when(describeStepResult.getStep()).thenReturn(step); |
||||||
|
|
||||||
|
emrAddStepsTask.init(); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void testCanNotParseJson() throws Exception { |
||||||
|
mockStatic(JSONUtils.class); |
||||||
|
when(emrAddStepsTask, "createAddJobFlowStepsRequest").thenThrow(new EmrTaskException("can not parse AddJobFlowStepsRequest from json", new Exception("error"))); |
||||||
|
emrAddStepsTask.handle(); |
||||||
|
Assert.assertEquals(EXIT_CODE_FAILURE, emrAddStepsTask.getExitStatusCode()); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void testDefineJsonStepNotOne() throws Exception { |
||||||
|
// mock EmrParameters and EmrAddStepsTask
|
||||||
|
EmrParameters emrParameters = buildErrorEmrTaskParameters(); |
||||||
|
String emrParametersString = JSONUtils.toJsonString(emrParameters); |
||||||
|
TaskExecutionContext taskExecutionContext = PowerMockito.mock(TaskExecutionContext.class); |
||||||
|
when(taskExecutionContext.getTaskParams()).thenReturn(emrParametersString); |
||||||
|
emrAddStepsTask = spy(new EmrAddStepsTask(taskExecutionContext)); |
||||||
|
doReturn(emrClient).when(emrAddStepsTask, "createEmrClient"); |
||||||
|
emrAddStepsTask.init(); |
||||||
|
emrAddStepsTask.handle(); |
||||||
|
|
||||||
|
Assert.assertEquals(EXIT_CODE_FAILURE, emrAddStepsTask.getExitStatusCode()); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void testHandle() throws Exception { |
||||||
|
when(step.getStatus()).thenReturn(pendingState, runningState, completedState); |
||||||
|
|
||||||
|
emrAddStepsTask.handle(); |
||||||
|
Assert.assertEquals(EXIT_CODE_SUCCESS, emrAddStepsTask.getExitStatusCode()); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void testHandleUserRequestTerminate() throws Exception { |
||||||
|
when(step.getStatus()).thenReturn(pendingState, runningState, cancelledState); |
||||||
|
|
||||||
|
emrAddStepsTask.handle(); |
||||||
|
Assert.assertEquals(EXIT_CODE_KILL, emrAddStepsTask.getExitStatusCode()); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void testHandleError() throws Exception { |
||||||
|
when(step.getStatus()).thenReturn(pendingState, runningState, failedState); |
||||||
|
emrAddStepsTask.handle(); |
||||||
|
Assert.assertEquals(EXIT_CODE_FAILURE, emrAddStepsTask.getExitStatusCode()); |
||||||
|
|
||||||
|
when(emrClient.addJobFlowSteps(any())).thenThrow(new AmazonElasticMapReduceException("error"), new EmrTaskException()); |
||||||
|
emrAddStepsTask.handle(); |
||||||
|
Assert.assertEquals(EXIT_CODE_FAILURE, emrAddStepsTask.getExitStatusCode()); |
||||||
|
} |
||||||
|
|
||||||
|
private EmrParameters buildEmrTaskParameters() { |
||||||
|
EmrParameters emrParameters = new EmrParameters(); |
||||||
|
String stepsDefineJson; |
||||||
|
try (InputStream i = this.getClass().getResourceAsStream("EmrAddStepsDefine.json")) { |
||||||
|
assert i != null; |
||||||
|
stepsDefineJson = IOUtils.toString(i, StandardCharsets.UTF_8); |
||||||
|
} catch (Exception e) { |
||||||
|
throw new RuntimeException(e); |
||||||
|
} |
||||||
|
emrParameters.setProgramType(ProgramType.ADD_JOB_FLOW_STEPS); |
||||||
|
emrParameters.setStepsDefineJson(stepsDefineJson); |
||||||
|
|
||||||
|
return emrParameters; |
||||||
|
} |
||||||
|
|
||||||
|
private EmrParameters buildErrorEmrTaskParameters() { |
||||||
|
EmrParameters emrParameters = new EmrParameters(); |
||||||
|
String stepsDefineJson; |
||||||
|
try (InputStream i = this.getClass().getResourceAsStream("EmrErrorAddStepsDefine.json")) { |
||||||
|
assert i != null; |
||||||
|
stepsDefineJson = IOUtils.toString(i, StandardCharsets.UTF_8); |
||||||
|
} catch (Exception e) { |
||||||
|
throw new RuntimeException(e); |
||||||
|
} |
||||||
|
emrParameters.setProgramType(ProgramType.ADD_JOB_FLOW_STEPS); |
||||||
|
emrParameters.setStepsDefineJson(stepsDefineJson); |
||||||
|
|
||||||
|
return emrParameters; |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,17 @@ |
|||||||
|
{ |
||||||
|
"JobFlowId": "j-3V628TKAERHP8", |
||||||
|
"Steps": [ |
||||||
|
{ |
||||||
|
"Name": "calculate_pi", |
||||||
|
"ActionOnFailure": "CONTINUE", |
||||||
|
"HadoopJarStep": { |
||||||
|
"Jar": "command-runner.jar", |
||||||
|
"Args": [ |
||||||
|
"/usr/lib/spark/bin/run-example", |
||||||
|
"SparkPi", |
||||||
|
"15" |
||||||
|
] |
||||||
|
} |
||||||
|
} |
||||||
|
] |
||||||
|
} |
||||||
@ -0,0 +1,29 @@ |
|||||||
|
{ |
||||||
|
"JobFlowId": "j-3V628TKAERHP8", |
||||||
|
"Steps": [ |
||||||
|
{ |
||||||
|
"Name": "calculate_pi", |
||||||
|
"ActionOnFailure": "CONTINUE", |
||||||
|
"HadoopJarStep": { |
||||||
|
"Jar": "command-runner.jar", |
||||||
|
"Args": [ |
||||||
|
"/usr/lib/spark/bin/run-example", |
||||||
|
"SparkPi", |
||||||
|
"15" |
||||||
|
] |
||||||
|
} |
||||||
|
}, |
||||||
|
{ |
||||||
|
"Name": "calculate_pi", |
||||||
|
"ActionOnFailure": "CONTINUE", |
||||||
|
"HadoopJarStep": { |
||||||
|
"Jar": "command-runner.jar", |
||||||
|
"Args": [ |
||||||
|
"/usr/lib/spark/bin/run-example", |
||||||
|
"SparkPi", |
||||||
|
"15" |
||||||
|
] |
||||||
|
} |
||||||
|
} |
||||||
|
] |
||||||
|
} |
||||||
Loading…
Reference in new issue