github
/
DolphinScheduler
mirror of https://gitee.com/dolphinscheduler/DolphinScheduler.git
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

[Improvement][Test] Remove Powermock in dolphinscheduler-task-plugin (#12153) 

* Remove the usage of powermock in task-plugin module
3.2.0-release
Stalary 2 years ago committed by GitHub
parent
ad4f344283
commit
aa5526ef5d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 116 additions and 259 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 48
      dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/src/test/java/org/apache/dolphinscheduler/plugin/task/dvc/DvcTaskTest.java
  2. 2
      dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/AbstractEmrTask.java
  3. 2
      dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/EmrAddStepsTask.java
  4. 2
      dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/EmrJobFlowTask.java
  5. 65
      dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/test/java/org/apache/dolphinscheduler/plugin/task/emr/EmrAddStepsTaskTest.java
  6. 61
      dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/test/java/org/apache/dolphinscheduler/plugin/task/emr/EmrJobFlowTaskTest.java
  7. 8
      dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/test/java/org/apache/dolphinscheduler/plugin/task/http/HttpTaskTest.java
  8. 48
      dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/test/java/org/apache/dolphinler/plugin/task/mlflow/MlflowTaskTest.java
  9. 13
      dolphinscheduler-task-plugin/dolphinscheduler-task-openmldb/src/test/java/org/apache/dolphinscheduler/plugin/task/openmldb/OpenmldbTaskTest.java
  10. 6
      dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/test/java/org/apache/dolphinscheduler/plugin/task/python/PythonTaskTest.java
  11. 33
      dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/test/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchTaskTest.java
  12. 47
      dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/test/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerTaskTest.java
  13. 34
      dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/test/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskTest.java

48
dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/src/test/java/org/apache/dolphinscheduler/plugin/task/dvc/DvcTaskTest.java
Unescape View File

@ -17,56 +17,24 @@
package org.apache.dolphinscheduler.plugin.task.dvc;
import java.util.Date;
import java.util.UUID;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContextCacheManager;
import org.apache.dolphinscheduler.spi.utils.JSONUtils;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.dolphinscheduler.spi.utils.PropertyUtils;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.junit.runner.RunWith;
import org.powermock.modules.junit4.PowerMockRunner;
import org.powermock.core.classloader.annotations.SuppressStaticInitializationFor;
import org.apache.dolphinscheduler.spi.utils.JSONUtils;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
@RunWith(PowerMockRunner.class)
@PrepareForTest({
JSONUtils.class,
PropertyUtils.class,
})
@PowerMockIgnore({"javax.*"})
@SuppressStaticInitializationFor("org.apache.dolphinscheduler.spi.utils.PropertyUtils")
@RunWith(MockitoJUnitRunner.class)
public class DvcTaskTest {
@Before
public void before() throws Exception {
PowerMockito.mockStatic(PropertyUtils.class);
}
public TaskExecutionContext createContext(DvcParameters dvcParameters) {
String parameters = JSONUtils.toJsonString(dvcParameters);
TaskExecutionContext taskExecutionContext = Mockito.mock(TaskExecutionContext.class);
Mockito.when(taskExecutionContext.getTaskParams()).thenReturn(parameters);
Mockito.when(taskExecutionContext.getTaskLogName()).thenReturn("DvcTest");
Mockito.when(taskExecutionContext.getExecutePath()).thenReturn("/tmp/dolphinscheduler_dvc_test");
Mockito.when(taskExecutionContext.getTaskAppId()).thenReturn(UUID.randomUUID().toString());
Mockito.when(taskExecutionContext.getStartTime()).thenReturn(new Date());
Mockito.when(taskExecutionContext.getTaskTimeout()).thenReturn(10000);
Mockito.when(taskExecutionContext.getLogPath()).thenReturn("/tmp/dolphinscheduler_dvc_test/log");
Mockito.when(taskExecutionContext.getEnvironmentConfig()).thenReturn("export PATH=$HOME/anaconda3/bin:$PATH");
String userName = System.getenv().get("USER");
Mockito.when(taskExecutionContext.getTenantCode()).thenReturn(userName);
TaskExecutionContextCacheManager.cacheTaskExecutionContext(taskExecutionContext);
return taskExecutionContext;
@ -82,7 +50,7 @@ public class DvcTaskTest {
}
@Test
public void testDvcUpload() throws Exception{
public void testDvcUpload() throws Exception {
DvcTask dvcTask = initTask(createUploadParameters());
Assert.assertEquals(dvcTask.buildCommand(),
"which dvc || { echo \"dvc does not exist\"; exit 1; }; DVC_REPO=git@github.com:<YOUR-NAME-OR-ORG>/dvc-data-repository-example\n" +
@ -101,7 +69,7 @@ public class DvcTaskTest {
}
@Test
public void testDvcDownload() throws Exception{
public void testDvcDownload() throws Exception {
DvcTask dvcTask = initTask(createDownloadParameters());
Assert.assertEquals(dvcTask.buildCommand(),
"which dvc || { echo \"dvc does not exist\"; exit 1; }; DVC_REPO=git@github.com:<YOUR-NAME-OR-ORG>/dvc-data-repository-example\n" +
@ -112,7 +80,7 @@ public class DvcTaskTest {
}
@Test
public void testInitDvc() throws Exception{
public void testInitDvc() throws Exception {
DvcTask dvcTask = initTask(createInitDvcParameters());
Assert.assertEquals(dvcTask.buildCommand(),
"which dvc || { echo \"dvc does not exist\"; exit 1; }; DVC_REPO=git@github.com:<YOUR-NAME-OR-ORG>/dvc-data-repository-example\n" +

2
dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/AbstractEmrTask.java
Unescape View File

@ -98,7 +98,7 @@ public abstract class AbstractEmrTask extends AbstractRemoteTask {
*
* @return AmazonElasticMapReduce
*/
private AmazonElasticMapReduce createEmrClient() {
protected AmazonElasticMapReduce createEmrClient() {
final String awsAccessKeyId = PropertyUtils.getString(TaskConstants.AWS_ACCESS_KEY_ID);
final String awsSecretAccessKey = PropertyUtils.getString(TaskConstants.AWS_SECRET_ACCESS_KEY);

2
dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/EmrAddStepsTask.java
Unescape View File

@ -120,7 +120,7 @@ public class EmrAddStepsTask extends AbstractEmrTask {
*
* @return AddJobFlowStepsRequest
*/
private AddJobFlowStepsRequest createAddJobFlowStepsRequest() {
protected AddJobFlowStepsRequest createAddJobFlowStepsRequest() {
final AddJobFlowStepsRequest addJobFlowStepsRequest;
try {

2
dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/EmrJobFlowTask.java
Unescape View File

@ -114,7 +114,7 @@ public class EmrJobFlowTask extends AbstractEmrTask {
*
* @return RunJobFlowRequest
*/
private RunJobFlowRequest createRunJobFlowRequest() {
protected RunJobFlowRequest createRunJobFlowRequest() {
final RunJobFlowRequest runJobFlowRequest;
try {

65
dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/test/java/org/apache/dolphinscheduler/plugin/task/emr/EmrAddStepsTaskTest.java
Unescape View File

@ -20,13 +20,7 @@ package org.apache.dolphinscheduler.plugin.task.emr;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_FAILURE;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_KILL;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_SUCCESS;
import static org.mockito.ArgumentMatchers.any;
import static org.powermock.api.mockito.PowerMockito.doReturn;
import static org.powermock.api.mockito.PowerMockito.mock;
import static org.powermock.api.mockito.PowerMockito.mockStatic;
import static org.powermock.api.mockito.PowerMockito.spy;
import static org.powermock.api.mockito.PowerMockito.when;
import org.apache.dolphinscheduler.plugin.task.api.TaskCallBack;
import org.apache.dolphinscheduler.plugin.task.api.TaskException;
@ -43,13 +37,10 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
import com.amazonaws.services.elasticmapreduce.AmazonElasticMapReduce;
import com.amazonaws.services.elasticmapreduce.AmazonElasticMapReduceClientBuilder;
import com.amazonaws.services.elasticmapreduce.model.AddJobFlowStepsResult;
import com.amazonaws.services.elasticmapreduce.model.AmazonElasticMapReduceException;
import com.amazonaws.services.elasticmapreduce.model.DescribeStepResult;
@ -62,14 +53,7 @@ import com.amazonaws.services.elasticmapreduce.model.StepStatus;
*
* @since v3.1.0
*/
@RunWith(PowerMockRunner.class)
@PrepareForTest({
AmazonElasticMapReduceClientBuilder.class,
EmrAddStepsTask.class,
AmazonElasticMapReduce.class,
JSONUtils.class
})
@PowerMockIgnore({"javax.*"})
@RunWith(MockitoJUnitRunner.class)
public class EmrAddStepsTaskTest {
private final StepStatus pendingState =
@ -99,32 +83,31 @@ public class EmrAddStepsTaskTest {
// mock EmrParameters and EmrAddStepsTask
EmrParameters emrParameters = buildEmrTaskParameters();
String emrParametersString = JSONUtils.toJsonString(emrParameters);
TaskExecutionContext taskExecutionContext = PowerMockito.mock(TaskExecutionContext.class);
when(taskExecutionContext.getTaskParams()).thenReturn(emrParametersString);
emrAddStepsTask = spy(new EmrAddStepsTask(taskExecutionContext));
TaskExecutionContext taskExecutionContext = Mockito.mock(TaskExecutionContext.class);
Mockito.when(taskExecutionContext.getTaskParams()).thenReturn(emrParametersString);
emrAddStepsTask = Mockito.spy(new EmrAddStepsTask(taskExecutionContext));
// mock emrClient and behavior
emrClient = mock(AmazonElasticMapReduce.class);
emrClient = Mockito.mock(AmazonElasticMapReduce.class);
AddJobFlowStepsResult addJobFlowStepsResult = mock(AddJobFlowStepsResult.class);
when(emrClient.addJobFlowSteps(any())).thenReturn(addJobFlowStepsResult);
when(addJobFlowStepsResult.getStepIds()).thenReturn(Collections.singletonList("step-xx"));
AddJobFlowStepsResult addJobFlowStepsResult = Mockito.mock(AddJobFlowStepsResult.class);
Mockito.when(emrClient.addJobFlowSteps(any())).thenReturn(addJobFlowStepsResult);
Mockito.when(addJobFlowStepsResult.getStepIds()).thenReturn(Collections.singletonList("step-xx"));
doReturn(emrClient).when(emrAddStepsTask, "createEmrClient");
DescribeStepResult describeStepResult = mock(DescribeStepResult.class);
when(emrClient.describeStep(any())).thenReturn(describeStepResult);
Mockito.doReturn(emrClient).when(emrAddStepsTask).createEmrClient();
DescribeStepResult describeStepResult = Mockito.mock(DescribeStepResult.class);
Mockito.when(emrClient.describeStep(any())).thenReturn(describeStepResult);
// mock step
step = mock(Step.class);
when(describeStepResult.getStep()).thenReturn(step);
step = Mockito.mock(Step.class);
Mockito.when(describeStepResult.getStep()).thenReturn(step);
emrAddStepsTask.init();
}
@Test(expected = TaskException.class)
public void testCanNotParseJson() throws Exception {
mockStatic(JSONUtils.class);
when(emrAddStepsTask, "createAddJobFlowStepsRequest").thenThrow(new EmrTaskException("can not parse AddJobFlowStepsRequest from json", new Exception("error")));
Mockito.when(emrAddStepsTask.createAddJobFlowStepsRequest()).thenThrow(new EmrTaskException("can not parse AddJobFlowStepsRequest from json", new Exception("error")));
emrAddStepsTask.handle(taskCallBack);
}
@ -133,17 +116,17 @@ public class EmrAddStepsTaskTest {
// mock EmrParameters and EmrAddStepsTask
EmrParameters emrParameters = buildErrorEmrTaskParameters();
String emrParametersString = JSONUtils.toJsonString(emrParameters);
TaskExecutionContext taskExecutionContext = PowerMockito.mock(TaskExecutionContext.class);
when(taskExecutionContext.getTaskParams()).thenReturn(emrParametersString);
emrAddStepsTask = spy(new EmrAddStepsTask(taskExecutionContext));
doReturn(emrClient).when(emrAddStepsTask, "createEmrClient");
TaskExecutionContext taskExecutionContext = Mockito.mock(TaskExecutionContext.class);
Mockito.when(taskExecutionContext.getTaskParams()).thenReturn(emrParametersString);
emrAddStepsTask = Mockito.spy(new EmrAddStepsTask(taskExecutionContext));
Mockito.doReturn(emrClient).when(emrAddStepsTask).createEmrClient();
emrAddStepsTask.init();
emrAddStepsTask.handle(taskCallBack);
}
@Test
public void testHandle() throws Exception {
when(step.getStatus()).thenReturn(pendingState, runningState, completedState);
Mockito.when(step.getStatus()).thenReturn(pendingState, runningState, completedState);
emrAddStepsTask.handle(taskCallBack);
Assert.assertEquals(EXIT_CODE_SUCCESS, emrAddStepsTask.getExitStatusCode());
@ -151,7 +134,7 @@ public class EmrAddStepsTaskTest {
@Test
public void testHandleUserRequestTerminate() throws Exception {
when(step.getStatus()).thenReturn(pendingState, runningState, cancelledState);
Mockito.when(step.getStatus()).thenReturn(pendingState, runningState, cancelledState);
emrAddStepsTask.handle(taskCallBack);
Assert.assertEquals(EXIT_CODE_KILL, emrAddStepsTask.getExitStatusCode());
@ -159,11 +142,11 @@ public class EmrAddStepsTaskTest {
@Test(expected = TaskException.class)
public void testHandleError() throws Exception {
when(step.getStatus()).thenReturn(pendingState, runningState, failedState);
Mockito.when(step.getStatus()).thenReturn(pendingState, runningState, failedState);
emrAddStepsTask.handle(taskCallBack);
Assert.assertEquals(EXIT_CODE_FAILURE, emrAddStepsTask.getExitStatusCode());
when(emrClient.addJobFlowSteps(any())).thenThrow(new AmazonElasticMapReduceException("error"), new EmrTaskException());
Mockito.when(emrClient.addJobFlowSteps(any())).thenThrow(new AmazonElasticMapReduceException("error"), new EmrTaskException());
emrAddStepsTask.handle(taskCallBack);
}

61
dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/test/java/org/apache/dolphinscheduler/plugin/task/emr/EmrJobFlowTaskTest.java
Unescape View File

@ -20,13 +20,7 @@ package org.apache.dolphinscheduler.plugin.task.emr;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_FAILURE;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_KILL;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_SUCCESS;
import static org.mockito.ArgumentMatchers.any;
import static org.powermock.api.mockito.PowerMockito.doReturn;
import static org.powermock.api.mockito.PowerMockito.mock;
import static org.powermock.api.mockito.PowerMockito.mockStatic;
import static org.powermock.api.mockito.PowerMockito.spy;
import static org.powermock.api.mockito.PowerMockito.when;
import org.apache.dolphinscheduler.plugin.task.api.TaskCallBack;
import org.apache.dolphinscheduler.plugin.task.api.TaskException;
@ -42,13 +36,10 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
import com.amazonaws.services.elasticmapreduce.AmazonElasticMapReduce;
import com.amazonaws.services.elasticmapreduce.AmazonElasticMapReduceClientBuilder;
import com.amazonaws.services.elasticmapreduce.model.AmazonElasticMapReduceException;
import com.amazonaws.services.elasticmapreduce.model.Cluster;
import com.amazonaws.services.elasticmapreduce.model.ClusterState;
@ -58,14 +49,7 @@ import com.amazonaws.services.elasticmapreduce.model.ClusterStatus;
import com.amazonaws.services.elasticmapreduce.model.DescribeClusterResult;
import com.amazonaws.services.elasticmapreduce.model.RunJobFlowResult;
@RunWith(PowerMockRunner.class)
@PrepareForTest({
AmazonElasticMapReduceClientBuilder.class,
EmrJobFlowTask.class,
AmazonElasticMapReduce.class,
JSONUtils.class
})
@PowerMockIgnore({"javax.*"})
@RunWith(MockitoJUnitRunner.class)
public class EmrJobFlowTaskTest {
private final ClusterStatus startingStatus =
@ -126,22 +110,22 @@ public class EmrJobFlowTaskTest {
@Before
public void before() throws Exception {
String emrParameters = buildEmrTaskParameters();
TaskExecutionContext taskExecutionContext = PowerMockito.mock(TaskExecutionContext.class);
when(taskExecutionContext.getTaskParams()).thenReturn(emrParameters);
emrJobFlowTask = spy(new EmrJobFlowTask(taskExecutionContext));
TaskExecutionContext taskExecutionContext = Mockito.mock(TaskExecutionContext.class);
Mockito.when(taskExecutionContext.getTaskParams()).thenReturn(emrParameters);
emrJobFlowTask = Mockito.spy(new EmrJobFlowTask(taskExecutionContext));
// mock emrClient and behavior
emrClient = mock(AmazonElasticMapReduce.class);
RunJobFlowResult runJobFlowResult = mock(RunJobFlowResult.class);
when(emrClient.runJobFlow(any())).thenReturn(runJobFlowResult);
when(runJobFlowResult.getJobFlowId()).thenReturn("xx");
doReturn(emrClient).when(emrJobFlowTask, "createEmrClient");
DescribeClusterResult describeClusterResult = mock(DescribeClusterResult.class);
when(emrClient.describeCluster(any())).thenReturn(describeClusterResult);
emrClient = Mockito.mock(AmazonElasticMapReduce.class);
RunJobFlowResult runJobFlowResult = Mockito.mock(RunJobFlowResult.class);
Mockito.when(emrClient.runJobFlow(any())).thenReturn(runJobFlowResult);
Mockito.when(runJobFlowResult.getJobFlowId()).thenReturn("xx");
Mockito.doReturn(emrClient).when(emrJobFlowTask).createEmrClient();
DescribeClusterResult describeClusterResult = Mockito.mock(DescribeClusterResult.class);
Mockito.when(emrClient.describeCluster(any())).thenReturn(describeClusterResult);
// mock cluster
cluster = mock(Cluster.class);
when(describeClusterResult.getCluster()).thenReturn(cluster);
cluster = Mockito.mock(Cluster.class);
Mockito.when(describeClusterResult.getCluster()).thenReturn(cluster);
emrJobFlowTask.init();
}
@ -149,7 +133,7 @@ public class EmrJobFlowTaskTest {
@Test
public void testHandle() throws Exception {
when(cluster.getStatus()).thenReturn(startingStatus, softwareConfigStatus, runningStatus, terminatingStatus);
Mockito.when(cluster.getStatus()).thenReturn(startingStatus, softwareConfigStatus, runningStatus, terminatingStatus);
emrJobFlowTask.handle(taskCallBack);
Assert.assertEquals(EXIT_CODE_SUCCESS, emrJobFlowTask.getExitStatusCode());
@ -158,7 +142,7 @@ public class EmrJobFlowTaskTest {
@Test
public void testHandleAliveWhenNoSteps() throws Exception {
when(cluster.getStatus()).thenReturn(startingStatus, softwareConfigStatus, runningStatus, waitingStatus);
Mockito.when(cluster.getStatus()).thenReturn(startingStatus, softwareConfigStatus, runningStatus, waitingStatus);
emrJobFlowTask.handle(taskCallBack);
Assert.assertEquals(EXIT_CODE_SUCCESS, emrJobFlowTask.getExitStatusCode());
@ -166,7 +150,7 @@ public class EmrJobFlowTaskTest {
@Test
public void testHandleUserRequestTerminate() throws Exception {
when(cluster.getStatus()).thenReturn(startingStatus, userRequestTerminateStatus);
Mockito.when(cluster.getStatus()).thenReturn(startingStatus, userRequestTerminateStatus);
emrJobFlowTask.handle(taskCallBack);
Assert.assertEquals(EXIT_CODE_KILL, emrJobFlowTask.getExitStatusCode());
@ -174,7 +158,7 @@ public class EmrJobFlowTaskTest {
@Test
public void testHandleTerminatedWithError() throws Exception {
when(cluster.getStatus()).thenReturn(startingStatus, softwareConfigStatus, runningStatus, terminatedWithErrorsStatus);
Mockito.when(cluster.getStatus()).thenReturn(startingStatus, softwareConfigStatus, runningStatus, terminatedWithErrorsStatus);
emrJobFlowTask.handle(taskCallBack);
Assert.assertEquals(EXIT_CODE_FAILURE, emrJobFlowTask.getExitStatusCode());
@ -182,21 +166,20 @@ public class EmrJobFlowTaskTest {
@Test(expected = TaskException.class)
public void testCanNotParseJson() throws Exception {
mockStatic(JSONUtils.class);
when(emrJobFlowTask, "createRunJobFlowRequest").thenThrow(new EmrTaskException("can not parse RunJobFlowRequest from json", new Exception("error")));
Mockito.when(emrJobFlowTask.createRunJobFlowRequest()).thenThrow(new EmrTaskException("can not parse RunJobFlowRequest from json", new Exception("error")));
emrJobFlowTask.handle(taskCallBack);
}
@Test(expected = TaskException.class)
public void testClusterStatusNull() throws Exception {
when(emrClient.describeCluster(any())).thenReturn(null);
Mockito.when(emrClient.describeCluster(any())).thenReturn(null);
emrJobFlowTask.handle(taskCallBack);
}
@Test(expected = TaskException.class)
public void testRunJobFlowError() throws Exception {
when(emrClient.runJobFlow(any())).thenThrow(new AmazonElasticMapReduceException("error"), new EmrTaskException());
Mockito.when(emrClient.runJobFlow(any())).thenThrow(new AmazonElasticMapReduceException("error"), new EmrTaskException());
emrJobFlowTask.handle(taskCallBack);
}

8
dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/test/java/org/apache/dolphinscheduler/plugin/task/http/HttpTaskTest.java
Unescape View File

@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.plugin.task.http;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_FAILURE;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_SUCCESS;
import static org.powermock.api.mockito.PowerMockito.when;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.model.Property;
@ -43,7 +42,9 @@ import okhttp3.mockwebserver.RecordedRequest;
import org.junit.After;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
@ -51,6 +52,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
/**
* Test HttpTask
*/
@RunWith(MockitoJUnitRunner.class)
public class HttpTaskTest {
private static final String CONTENT_TYPE = "Content-Type";
@ -203,7 +205,7 @@ public class HttpTaskTest {
private HttpTask generateHttpTaskFromParamData(String paramData, Map<String, String> prepareParamsMap) {
TaskExecutionContext taskExecutionContext = Mockito.mock(TaskExecutionContext.class);
when(taskExecutionContext.getTaskParams()).thenReturn(paramData);
Mockito.when(taskExecutionContext.getTaskParams()).thenReturn(paramData);
if (prepareParamsMap != null) {
Map<String, Property> propertyParamsMap = new HashMap<>();
prepareParamsMap.forEach((k, v) -> {
@ -212,7 +214,7 @@ public class HttpTaskTest {
property.setValue(v);
propertyParamsMap.put(k, property);
});
when(taskExecutionContext.getPrepareParamsMap()).thenReturn(propertyParamsMap);
Mockito.when(taskExecutionContext.getPrepareParamsMap()).thenReturn(propertyParamsMap);
}
HttpTask httpTask = new HttpTask(taskExecutionContext);
httpTask.init();

48
dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/test/java/org/apache/dolphinler/plugin/task/mlflow/MlflowTaskTest.java
Unescape View File

@ -17,8 +17,6 @@
package org.apache.dolphinler.plugin.task.mlflow;
import static org.powermock.api.mockito.PowerMockito.when;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContextCacheManager;
import org.apache.dolphinscheduler.plugin.task.mlflow.MlflowConstants;
@ -27,53 +25,37 @@ import org.apache.dolphinscheduler.plugin.task.mlflow.MlflowTask;
import org.apache.dolphinscheduler.spi.utils.JSONUtils;
import org.apache.dolphinscheduler.spi.utils.PropertyUtils;
import java.util.Date;
import java.util.UUID;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.core.classloader.annotations.SuppressStaticInitializationFor;
import org.powermock.modules.junit4.PowerMockRunner;
import org.mockito.junit.MockitoJUnitRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@RunWith(PowerMockRunner.class)
@PrepareForTest({
JSONUtils.class,
PropertyUtils.class,
})
@PowerMockIgnore({"javax.*"})
@SuppressStaticInitializationFor("org.apache.dolphinscheduler.spi.utils.PropertyUtils")
@RunWith(MockitoJUnitRunner.class)
public class MlflowTaskTest {
private static final Logger logger = LoggerFactory.getLogger(MlflowTask.class);
private MockedStatic<PropertyUtils> propertyUtilsMockedStatic;
@Before
public void before() throws Exception {
PowerMockito.mockStatic(PropertyUtils.class);
public void init() {
propertyUtilsMockedStatic = Mockito.mockStatic(PropertyUtils.class);
propertyUtilsMockedStatic.when(() -> PropertyUtils.getString(MlflowConstants.PRESET_REPOSITORY_VERSION_KEY)).thenReturn("main");
}
@After
public void clean() {
propertyUtilsMockedStatic.close();
}
public TaskExecutionContext createContext(MlflowParameters mlflowParameters) {
String parameters = JSONUtils.toJsonString(mlflowParameters);
TaskExecutionContext taskExecutionContext = Mockito.mock(TaskExecutionContext.class);
Mockito.when(taskExecutionContext.getTaskParams()).thenReturn(parameters);
Mockito.when(taskExecutionContext.getTaskLogName()).thenReturn("MLflowTest");
Mockito.when(taskExecutionContext.getExecutePath()).thenReturn("/tmp/dolphinscheduler_test");
Mockito.when(taskExecutionContext.getTaskAppId()).thenReturn(UUID.randomUUID().toString());
Mockito.when(taskExecutionContext.getTenantCode()).thenReturn("root");
Mockito.when(taskExecutionContext.getStartTime()).thenReturn(new Date());
Mockito.when(taskExecutionContext.getTaskTimeout()).thenReturn(10000);
Mockito.when(taskExecutionContext.getLogPath()).thenReturn("/tmp/dolphinscheduler_test/log");
Mockito.when(taskExecutionContext.getEnvironmentConfig()).thenReturn("export PATH=$HOME/anaconda3/bin:$PATH");
String userName = System.getenv().get("USER");
Mockito.when(taskExecutionContext.getTenantCode()).thenReturn(userName);
TaskExecutionContextCacheManager.cacheTaskExecutionContext(taskExecutionContext);
return taskExecutionContext;
}
@ -85,11 +67,11 @@ public class MlflowTaskTest {
Assert.assertEquals("main", MlflowTask.getPresetRepositoryVersion());
String definedRepository = "https://github.com/<MY-ID>/dolphinscheduler-mlflow";
when(PropertyUtils.getString(MlflowConstants.PRESET_REPOSITORY_KEY)).thenAnswer(invocation -> definedRepository);
Mockito.when(PropertyUtils.getString(MlflowConstants.PRESET_REPOSITORY_KEY)).thenAnswer(invocation -> definedRepository);
Assert.assertEquals(definedRepository, MlflowTask.getPresetRepository());
String definedRepositoryVersion = "dev";
when(PropertyUtils.getString(MlflowConstants.PRESET_REPOSITORY_VERSION_KEY)).thenAnswer(invocation -> definedRepositoryVersion);
Mockito.when(PropertyUtils.getString(MlflowConstants.PRESET_REPOSITORY_VERSION_KEY)).thenAnswer(invocation -> definedRepositoryVersion);
Assert.assertEquals(definedRepositoryVersion, MlflowTask.getPresetRepositoryVersion());
}

13
dolphinscheduler-task-plugin/dolphinscheduler-task-openmldb/src/test/java/org/apache/dolphinscheduler/plugin/task/openmldb/OpenmldbTaskTest.java
Unescape View File

@ -20,13 +20,14 @@ package org.apache.dolphinscheduler.plugin.task.openmldb;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.model.Property;
import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
import org.apache.dolphinscheduler.spi.utils.JSONUtils;
import java.util.HashMap;
import java.util.Map;
import org.junit.Assert;
import org.junit.Test;
import org.powermock.reflect.Whitebox;
import org.mockito.Mockito;
public class OpenmldbTaskTest {
static class MockOpenmldbTask extends OpenmldbTask {
@ -59,15 +60,19 @@ public class OpenmldbTaskTest {
@Test
public void buildSQLWithComment() throws Exception {
OpenmldbTask openmldbTask = createOpenmldbTask();
TaskExecutionContext taskExecutionContext = Mockito.mock(TaskExecutionContext.class);
OpenmldbParameters openmldbParameters = new OpenmldbParameters();
openmldbParameters.setExecuteMode("offline");
openmldbParameters.setZk("localhost:2181");
openmldbParameters.setZkPath("dolphinscheduler");
String rawSQLScript = "select * from users\r\n"
+ "-- some comment\n"
+ "inner join order on users.order_id = order.id; \n\n;"
+ "select * from users;";
openmldbParameters.setSql(rawSQLScript);
Whitebox.setInternalState(openmldbTask, "openmldbParameters", openmldbParameters);
Mockito.when(taskExecutionContext.getTaskParams()).thenReturn(JSONUtils.toJsonString(openmldbParameters));
OpenmldbTask openmldbTask = new OpenmldbTask(taskExecutionContext);
openmldbTask.init();
OpenmldbParameters internal = (OpenmldbParameters) openmldbTask.getParameters();
Assert.assertNotNull(internal);
Assert.assertEquals(internal.getExecuteMode(), "offline");
@ -75,7 +80,7 @@ public class OpenmldbTaskTest {
String result1 = openmldbTask.buildPythonScriptContent();
Assert.assertEquals("import openmldb\n"
+ "import sqlalchemy as db\n"
+ "engine = db.create_engine('openmldb:///?zk=null&zkPath=null')\n"
+ "engine = db.create_engine('openmldb:///?zk=localhost:2181&zkPath=dolphinscheduler')\n"
+ "con = engine.connect()\n"
+ "con.execute(\"set @@execute_mode='offline';\")\n"
+ "con.execute(\"set @@sync_job=true\")\n"

6
dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/test/java/org/apache/dolphinscheduler/plugin/task/python/PythonTaskTest.java
Unescape View File

@ -19,17 +19,13 @@ package org.apache.dolphinscheduler.plugin.task.python;
import org.junit.Assert;
import org.junit.Test;
import org.powermock.reflect.Whitebox;
public class PythonTaskTest {
@Test
public void buildPythonExecuteCommand() throws Exception {
PythonTask pythonTask = createPythonTask();
String methodName = "buildPythonExecuteCommand";
String pythonFile = "test.py";
String result1 = Whitebox.invokeMethod(pythonTask, methodName, pythonFile);
Assert.assertEquals("${PYTHON_HOME} test.py", result1);
Assert.assertEquals("${PYTHON_HOME} test.py", pythonTask.buildPythonExecuteCommand("test.py"));
}
private PythonTask createPythonTask() {

33
dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/test/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchTaskTest.java
Unescape View File

@ -45,26 +45,14 @@ import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.core.classloader.annotations.SuppressStaticInitializationFor;
import org.powermock.modules.junit4.PowerMockRunner;
@RunWith(PowerMockRunner.class)
@PrepareForTest({JSONUtils.class, PropertyUtils.class,})
@PowerMockIgnore({"javax.*"})
@SuppressStaticInitializationFor("org.apache.dolphinscheduler.spi.utils.PropertyUtils")
import org.mockito.junit.MockitoJUnitRunner;
@RunWith(MockitoJUnitRunner.class)
public class PytorchTaskTest {
private final String pythonPath = ".";
private final String requirementPath = "requirements.txt";
@Before
public void before() {
PowerMockito.mockStatic(PropertyUtils.class);
}
@Test
public void testPythonEnvManager() {
PythonEnvManager envManager = new PythonEnvManager();
@ -207,22 +195,7 @@ public class PytorchTaskTest {
public TaskExecutionContext createContext(PytorchParameters pytorchParameters) {
String parameters = JSONUtils.toJsonString(pytorchParameters);
TaskExecutionContext taskExecutionContext = Mockito.mock(TaskExecutionContext.class);
Mockito.when(taskExecutionContext.getTaskLogName()).thenReturn("PytorchTest");
String APP_ID = UUID.randomUUID().toString();
String folder = String.format("/tmp/dolphinscheduler_PytorchTest_%s", APP_ID);
Mockito.when(taskExecutionContext.getExecutePath()).thenReturn(folder);
Mockito.when(taskExecutionContext.getTaskAppId()).thenReturn(APP_ID);
Mockito.when(taskExecutionContext.getTenantCode()).thenReturn("root");
Mockito.when(taskExecutionContext.getStartTime()).thenReturn(new Date());
Mockito.when(taskExecutionContext.getTaskTimeout()).thenReturn(10000);
Mockito.when(taskExecutionContext.getLogPath()).thenReturn(folder + "/log");
Mockito.when(taskExecutionContext.getTaskParams()).thenReturn(parameters);
String envirementConfig = "export PATH=$HOME/anaconda3/bin:$PATH\n" + "export PYTHON_HOME=/bin/python";
Mockito.when(taskExecutionContext.getEnvironmentConfig()).thenReturn(envirementConfig);
String userName = System.getenv().get("USER");
Mockito.when(taskExecutionContext.getTenantCode()).thenReturn(userName);
TaskExecutionContextCacheManager.cacheTaskExecutionContext(taskExecutionContext);
return taskExecutionContext;
}

47
dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/test/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerTaskTest.java
Unescape View File

@ -18,40 +18,29 @@
package org.apache.dolphinscheduler.plugin.task.sagemaker;
import static org.mockito.ArgumentMatchers.any;
import static org.powermock.api.mockito.PowerMockito.mock;
import static org.powermock.api.mockito.PowerMockito.when;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.spi.utils.JSONUtils;
import org.apache.dolphinscheduler.spi.utils.PropertyUtils;
import org.apache.commons.io.IOUtils;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import org.mockito.junit.MockitoJUnitRunner;
import com.amazonaws.services.sagemaker.AmazonSageMaker;
import com.amazonaws.services.sagemaker.model.DescribePipelineExecutionResult;
import com.amazonaws.services.sagemaker.model.ListPipelineExecutionStepsResult;
import com.amazonaws.services.sagemaker.model.PipelineExecutionStep;
import com.amazonaws.services.sagemaker.model.StartPipelineExecutionRequest;
import com.amazonaws.services.sagemaker.model.StartPipelineExecutionResult;
import com.amazonaws.services.sagemaker.model.StopPipelineExecutionResult;
@RunWith(PowerMockRunner.class)
@PrepareForTest({JSONUtils.class, PropertyUtils.class,})
@PowerMockIgnore({"javax.*"})
@RunWith(MockitoJUnitRunner.class)
public class SagemakerTaskTest {
private final String pipelineExecutionArn = "test-pipeline-arn";
@ -66,34 +55,22 @@ public class SagemakerTaskTest {
TaskExecutionContext taskExecutionContext = Mockito.mock(TaskExecutionContext.class);
Mockito.when(taskExecutionContext.getTaskParams()).thenReturn(parameters);
client = mock(AmazonSageMaker.class);
client = Mockito.mock(AmazonSageMaker.class);
sagemakerTask = new SagemakerTask(taskExecutionContext);
sagemakerTask.init();
StartPipelineExecutionResult startPipelineExecutionResult = mock(StartPipelineExecutionResult.class);
when(startPipelineExecutionResult.getPipelineExecutionArn()).thenReturn(pipelineExecutionArn);
StartPipelineExecutionResult startPipelineExecutionResult = Mockito.mock(StartPipelineExecutionResult.class);
Mockito.when(startPipelineExecutionResult.getPipelineExecutionArn()).thenReturn(pipelineExecutionArn);
StopPipelineExecutionResult stopPipelineExecutionResult = mock(StopPipelineExecutionResult.class);
when(stopPipelineExecutionResult.getPipelineExecutionArn()).thenReturn(pipelineExecutionArn);
StopPipelineExecutionResult stopPipelineExecutionResult = Mockito.mock(StopPipelineExecutionResult.class);
Mockito.when(stopPipelineExecutionResult.getPipelineExecutionArn()).thenReturn(pipelineExecutionArn);
DescribePipelineExecutionResult describePipelineExecutionResult = mock(DescribePipelineExecutionResult.class);
when(describePipelineExecutionResult.getPipelineExecutionStatus()).thenReturn("Executing", "Succeeded");
ListPipelineExecutionStepsResult listPipelineExecutionStepsResult =
mock(ListPipelineExecutionStepsResult.class);
PipelineExecutionStep pipelineExecutionStep = mock(PipelineExecutionStep.class);
List<PipelineExecutionStep> pipelineExecutionSteps = new ArrayList<>();
pipelineExecutionSteps.add(pipelineExecutionStep);
pipelineExecutionSteps.add(pipelineExecutionStep);
when(pipelineExecutionStep.toString()).thenReturn("Test Step1", "Test Step2");
when(listPipelineExecutionStepsResult.getPipelineExecutionSteps()).thenReturn(pipelineExecutionSteps);
when(client.startPipelineExecution(any())).thenReturn(startPipelineExecutionResult);
when(client.stopPipelineExecution(any())).thenReturn(stopPipelineExecutionResult);
when(client.describePipelineExecution(any())).thenReturn(describePipelineExecutionResult);
when(client.listPipelineExecutionSteps(any())).thenReturn(listPipelineExecutionStepsResult);
DescribePipelineExecutionResult describePipelineExecutionResult = Mockito.mock(DescribePipelineExecutionResult.class);
Mockito.when(describePipelineExecutionResult.getPipelineExecutionStatus()).thenReturn("Executing", "Succeeded");
Mockito.when(client.startPipelineExecution(any())).thenReturn(startPipelineExecutionResult);
Mockito.when(client.stopPipelineExecution(any())).thenReturn(stopPipelineExecutionResult);
Mockito.when(client.describePipelineExecution(any())).thenReturn(describePipelineExecutionResult);
}
@Test

34
dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/test/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskTest.java
Unescape View File

@ -17,9 +17,6 @@
package org.apache.dolphinscheduler.plugin.task.spark;
import static org.powermock.api.mockito.PowerMockito.spy;
import static org.powermock.api.mockito.PowerMockito.when;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo;
import org.apache.dolphinscheduler.spi.utils.JSONUtils;
@ -29,28 +26,21 @@ import java.util.Collections;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@RunWith(PowerMockRunner.class)
@PrepareForTest({
JSONUtils.class
})
@PowerMockIgnore({"javax.*"})
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
@RunWith(MockitoJUnitRunner.class)
public class SparkTaskTest {
@Test
public void testBuildCommandWithSparkSql() throws Exception {
String parameters = buildSparkParametersWithSparkSql();
TaskExecutionContext taskExecutionContext = PowerMockito.mock(TaskExecutionContext.class);
when(taskExecutionContext.getTaskParams()).thenReturn(parameters);
when(taskExecutionContext.getExecutePath()).thenReturn("/tmp");
when(taskExecutionContext.getTaskAppId()).thenReturn("5536");
TaskExecutionContext taskExecutionContext = Mockito.mock(TaskExecutionContext.class);
Mockito.when(taskExecutionContext.getTaskParams()).thenReturn(parameters);
Mockito.when(taskExecutionContext.getExecutePath()).thenReturn("/tmp");
Mockito.when(taskExecutionContext.getTaskAppId()).thenReturn("5536");
SparkTask sparkTask = spy(new SparkTask(taskExecutionContext));
SparkTask sparkTask = Mockito.spy(new SparkTask(taskExecutionContext));
sparkTask.init();
Assert.assertEquals(sparkTask.buildCommand(),
"${SPARK_HOME}/bin/spark-sql " +
@ -68,11 +58,9 @@ public class SparkTaskTest {
@Test
public void testBuildCommandWithSparkSubmit() {
String parameters = buildSparkParametersWithSparkSubmit();
TaskExecutionContext taskExecutionContext = PowerMockito.mock(TaskExecutionContext.class);
when(taskExecutionContext.getTaskParams()).thenReturn(parameters);
when(taskExecutionContext.getExecutePath()).thenReturn("/tmp");
when(taskExecutionContext.getTaskAppId()).thenReturn("5536");
SparkTask sparkTask = spy(new SparkTask(taskExecutionContext));
TaskExecutionContext taskExecutionContext = Mockito.mock(TaskExecutionContext.class);
Mockito.when(taskExecutionContext.getTaskParams()).thenReturn(parameters);
SparkTask sparkTask = Mockito.spy(new SparkTask(taskExecutionContext));
sparkTask.init();
Assert.assertEquals(sparkTask.buildCommand(),
"${SPARK_HOME}/bin/spark-submit " +

Write Preview
Loading…
Cancel
Save