Browse Source
* Supplementary data by schedule * fix sonar check bug * fix code duplicated blocks * ut * loop by day * MasterExecThread test * test add licene Co-authored-by: dailidong <dailidong66@gmail.com>pull/2/head
老佛爷
5 years ago
committed by
qiaozhanwei
14 changed files with 636 additions and 22 deletions
@ -0,0 +1,229 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
package org.apache.dolphinscheduler.api.service; |
||||
|
||||
import org.apache.dolphinscheduler.api.enums.Status; |
||||
import org.apache.dolphinscheduler.common.Constants; |
||||
import org.apache.dolphinscheduler.common.enums.CommandType; |
||||
import org.apache.dolphinscheduler.common.enums.Priority; |
||||
import org.apache.dolphinscheduler.common.enums.ReleaseState; |
||||
import org.apache.dolphinscheduler.common.enums.RunMode; |
||||
import org.apache.dolphinscheduler.dao.ProcessDao; |
||||
import org.apache.dolphinscheduler.dao.entity.*; |
||||
import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; |
||||
import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; |
||||
import org.junit.Assert; |
||||
import org.junit.Before; |
||||
import org.junit.Test; |
||||
import org.junit.runner.RunWith; |
||||
import org.mockito.InjectMocks; |
||||
import org.mockito.Mock; |
||||
import org.mockito.Mockito; |
||||
import org.mockito.junit.MockitoJUnitRunner; |
||||
|
||||
import java.text.ParseException; |
||||
import java.util.*; |
||||
|
||||
import static org.mockito.ArgumentMatchers.any; |
||||
import static org.mockito.Mockito.verify; |
||||
import static org.mockito.Mockito.times; |
||||
|
||||
/** |
||||
* test for ExecutorService |
||||
*/ |
||||
@RunWith(MockitoJUnitRunner.Silent.class) |
||||
public class ExecutorService2Test { |
||||
|
||||
@InjectMocks |
||||
private ExecutorService executorService; |
||||
|
||||
@Mock |
||||
private ProcessDao processDao; |
||||
|
||||
@Mock |
||||
private ProcessDefinitionMapper processDefinitionMapper; |
||||
|
||||
@Mock |
||||
private ProjectMapper projectMapper; |
||||
|
||||
@Mock |
||||
private ProjectService projectService; |
||||
|
||||
private int processDefinitionId = 1; |
||||
|
||||
private int tenantId = 1; |
||||
|
||||
private int userId = 1; |
||||
|
||||
private ProcessDefinition processDefinition = new ProcessDefinition(); |
||||
|
||||
private User loginUser = new User(); |
||||
|
||||
private String projectName = "projectName"; |
||||
|
||||
private Project project = new Project(); |
||||
|
||||
private String cronTime; |
||||
|
||||
@Before |
||||
public void init(){ |
||||
// user
|
||||
loginUser.setId(userId); |
||||
|
||||
// processDefinition
|
||||
processDefinition.setId(processDefinitionId); |
||||
processDefinition.setReleaseState(ReleaseState.ONLINE); |
||||
processDefinition.setTenantId(tenantId); |
||||
processDefinition.setUserId(userId); |
||||
|
||||
// project
|
||||
project.setName(projectName); |
||||
|
||||
// cronRangeTime
|
||||
cronTime = "2020-01-01 00:00:00,2020-01-31 23:00:00"; |
||||
|
||||
// mock
|
||||
Mockito.when(projectMapper.queryByName(projectName)).thenReturn(project); |
||||
Mockito.when(projectService.checkProjectAndAuth(loginUser, project, projectName)).thenReturn(checkProjectAndAuth()); |
||||
Mockito.when(processDefinitionMapper.selectById(processDefinitionId)).thenReturn(processDefinition); |
||||
Mockito.when(processDao.getTenantForProcess(tenantId, userId)).thenReturn(new Tenant()); |
||||
Mockito.when(processDao.createCommand(any(Command.class))).thenReturn(1); |
||||
} |
||||
|
||||
/** |
||||
* not complement |
||||
* @throws ParseException |
||||
*/ |
||||
@Test |
||||
public void testNoComplement() throws ParseException { |
||||
try { |
||||
Mockito.when(processDao.queryReleaseSchedulerListByProcessDefinitionId(processDefinitionId)).thenReturn(zeroSchedulerList()); |
||||
Map<String, Object> result = executorService.execProcessInstance(loginUser, projectName, |
||||
processDefinitionId, cronTime, CommandType.START_PROCESS, |
||||
null, null, |
||||
null, null, 0, |
||||
"", "", RunMode.RUN_MODE_SERIAL, |
||||
Priority.LOW, 0, 110); |
||||
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); |
||||
verify(processDao, times(1)).createCommand(any(Command.class)); |
||||
}catch (Exception e){ |
||||
Assert.assertTrue(false); |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* date error |
||||
* @throws ParseException |
||||
*/ |
||||
@Test |
||||
public void testDateError() throws ParseException { |
||||
try { |
||||
Mockito.when(processDao.queryReleaseSchedulerListByProcessDefinitionId(processDefinitionId)).thenReturn(zeroSchedulerList()); |
||||
Map<String, Object> result = executorService.execProcessInstance(loginUser, projectName, |
||||
processDefinitionId, "2020-01-31 23:00:00,2020-01-01 00:00:00", CommandType.COMPLEMENT_DATA, |
||||
null, null, |
||||
null, null, 0, |
||||
"", "", RunMode.RUN_MODE_SERIAL, |
||||
Priority.LOW, 0, 110); |
||||
Assert.assertEquals(Status.START_PROCESS_INSTANCE_ERROR, result.get(Constants.STATUS)); |
||||
verify(processDao, times(0)).createCommand(any(Command.class)); |
||||
}catch (Exception e){ |
||||
Assert.assertTrue(false); |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* serial |
||||
* @throws ParseException |
||||
*/ |
||||
@Test |
||||
public void testSerial() throws ParseException { |
||||
try { |
||||
Mockito.when(processDao.queryReleaseSchedulerListByProcessDefinitionId(processDefinitionId)).thenReturn(zeroSchedulerList()); |
||||
Map<String, Object> result = executorService.execProcessInstance(loginUser, projectName, |
||||
processDefinitionId, cronTime, CommandType.COMPLEMENT_DATA, |
||||
null, null, |
||||
null, null, 0, |
||||
"", "", RunMode.RUN_MODE_SERIAL, |
||||
Priority.LOW, 0, 110); |
||||
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); |
||||
verify(processDao, times(1)).createCommand(any(Command.class)); |
||||
}catch (Exception e){ |
||||
Assert.assertTrue(false); |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* without schedule |
||||
* @throws ParseException |
||||
*/ |
||||
@Test |
||||
public void testParallelWithOutSchedule() throws ParseException { |
||||
try{ |
||||
Mockito.when(processDao.queryReleaseSchedulerListByProcessDefinitionId(processDefinitionId)).thenReturn(zeroSchedulerList()); |
||||
Map<String, Object> result = executorService.execProcessInstance(loginUser, projectName, |
||||
processDefinitionId, cronTime, CommandType.COMPLEMENT_DATA, |
||||
null, null, |
||||
null, null, 0, |
||||
"", "", RunMode.RUN_MODE_PARALLEL, |
||||
Priority.LOW, 0, 110); |
||||
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); |
||||
verify(processDao, times(31)).createCommand(any(Command.class)); |
||||
}catch (Exception e){ |
||||
Assert.assertTrue(false); |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* with schedule |
||||
* @throws ParseException |
||||
*/ |
||||
@Test |
||||
public void testParallelWithSchedule() throws ParseException { |
||||
try{ |
||||
Mockito.when(processDao.queryReleaseSchedulerListByProcessDefinitionId(processDefinitionId)).thenReturn(oneSchedulerList()); |
||||
Map<String, Object> result = executorService.execProcessInstance(loginUser, projectName, |
||||
processDefinitionId, cronTime, CommandType.COMPLEMENT_DATA, |
||||
null, null, |
||||
null, null, 0, |
||||
"", "", RunMode.RUN_MODE_PARALLEL, |
||||
Priority.LOW, 0, 110); |
||||
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); |
||||
verify(processDao, times(16)).createCommand(any(Command.class)); |
||||
}catch (Exception e){ |
||||
Assert.assertTrue(false); |
||||
} |
||||
} |
||||
|
||||
private List<Schedule> zeroSchedulerList(){ |
||||
return Collections.EMPTY_LIST; |
||||
} |
||||
|
||||
private List<Schedule> oneSchedulerList(){ |
||||
List<Schedule> schedulerList = new LinkedList<>(); |
||||
Schedule schedule = new Schedule(); |
||||
schedule.setCrontab("0 0 0 1/2 * ?"); |
||||
schedulerList.add(schedule); |
||||
return schedulerList; |
||||
} |
||||
|
||||
private Map<String, Object> checkProjectAndAuth(){ |
||||
Map<String, Object> result = new HashMap<>(); |
||||
result.put(Constants.STATUS, Status.SUCCESS); |
||||
return result; |
||||
} |
||||
} |
||||
@ -0,0 +1,79 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
package org.apache.dolphinscheduler.server.utils; |
||||
|
||||
import org.quartz.impl.triggers.CronTriggerImpl; |
||||
import org.slf4j.Logger; |
||||
import org.slf4j.LoggerFactory; |
||||
|
||||
import java.text.ParseException; |
||||
import java.util.Date; |
||||
import java.util.LinkedList; |
||||
import java.util.List; |
||||
|
||||
/** |
||||
* ScheduleUtils |
||||
*/ |
||||
public class ScheduleUtils { |
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(ScheduleUtils.class); |
||||
|
||||
/** |
||||
* Get the execution time of the time interval |
||||
* @param cron |
||||
* @param from |
||||
* @param to |
||||
* @return |
||||
*/ |
||||
public static List<Date> getRecentTriggerTime(String cron, Date from, Date to) { |
||||
return getRecentTriggerTime(cron, Integer.MAX_VALUE, from, to); |
||||
} |
||||
|
||||
/** |
||||
* Get the execution time of the time interval |
||||
* @param cron |
||||
* @param size |
||||
* @param from |
||||
* @param to |
||||
* @return |
||||
*/ |
||||
public static List<Date> getRecentTriggerTime(String cron, int size, Date from, Date to) { |
||||
List list = new LinkedList<Date>(); |
||||
if(to.before(from)){ |
||||
logger.error("schedule date from:{} must before date to:{}!", from, to); |
||||
return list; |
||||
} |
||||
try { |
||||
CronTriggerImpl trigger = new CronTriggerImpl(); |
||||
trigger.setCronExpression(cron); |
||||
trigger.setStartTime(from); |
||||
trigger.setEndTime(to); |
||||
trigger.computeFirstFireTime(null); |
||||
for (int i = 0; i < size; i++) { |
||||
Date schedule = trigger.getNextFireTime(); |
||||
if(null == schedule){ |
||||
break; |
||||
} |
||||
list.add(schedule); |
||||
trigger.triggered(null); |
||||
} |
||||
} catch (ParseException e) { |
||||
logger.error("cron:{} error:{}", cron, e.getMessage()); |
||||
} |
||||
return java.util.Collections.unmodifiableList(list); |
||||
} |
||||
} |
||||
@ -0,0 +1,154 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
package org.apache.dolphinscheduler.server.master; |
||||
|
||||
import com.alibaba.fastjson.JSONObject; |
||||
import org.apache.dolphinscheduler.common.enums.*; |
||||
import org.apache.dolphinscheduler.common.graph.DAG; |
||||
import org.apache.dolphinscheduler.common.utils.DateUtils; |
||||
import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; |
||||
import org.apache.dolphinscheduler.dao.ProcessDao; |
||||
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; |
||||
import org.apache.dolphinscheduler.dao.entity.ProcessInstance; |
||||
import org.apache.dolphinscheduler.dao.entity.Schedule; |
||||
import org.apache.dolphinscheduler.server.master.config.MasterConfig; |
||||
import org.apache.dolphinscheduler.server.master.runner.MasterExecThread; |
||||
import org.junit.Assert; |
||||
import org.junit.Before; |
||||
import org.junit.Test; |
||||
import org.junit.runner.RunWith; |
||||
import org.mockito.Mockito; |
||||
import org.powermock.api.mockito.PowerMockito; |
||||
import org.powermock.core.classloader.annotations.PrepareForTest; |
||||
import org.powermock.modules.junit4.PowerMockRunner; |
||||
import org.springframework.context.ApplicationContext; |
||||
import java.lang.reflect.Field; |
||||
import java.lang.reflect.Method; |
||||
import java.text.ParseException; |
||||
import java.util.*; |
||||
import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_END_DATE; |
||||
import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_START_DATE; |
||||
import static org.mockito.Mockito.times; |
||||
import static org.mockito.Mockito.verify; |
||||
import static org.powermock.api.mockito.PowerMockito.mock; |
||||
|
||||
/** |
||||
* test for MasterExecThread |
||||
*/ |
||||
@RunWith(PowerMockRunner.class) |
||||
@PrepareForTest({MasterExecThread.class}) |
||||
public class MasterExecThreadTest { |
||||
|
||||
private MasterExecThread masterExecThread; |
||||
|
||||
private ProcessInstance processInstance; |
||||
|
||||
private ProcessDao processDao; |
||||
|
||||
private int processDefinitionId = 1; |
||||
|
||||
private MasterConfig config; |
||||
|
||||
private ApplicationContext applicationContext; |
||||
|
||||
@Before |
||||
public void init() throws Exception{ |
||||
processDao = mock(ProcessDao.class); |
||||
|
||||
applicationContext = mock(ApplicationContext.class); |
||||
config = new MasterConfig(); |
||||
config.setMasterExecTaskNum(1); |
||||
SpringApplicationContext springApplicationContext = new SpringApplicationContext(); |
||||
springApplicationContext.setApplicationContext(applicationContext); |
||||
Mockito.when(applicationContext.getBean(MasterConfig.class)).thenReturn(config); |
||||
|
||||
processInstance = mock(ProcessInstance.class); |
||||
Mockito.when(processInstance.getProcessDefinitionId()).thenReturn(processDefinitionId); |
||||
Mockito.when(processInstance.getState()).thenReturn(ExecutionStatus.SUCCESS); |
||||
Mockito.when(processInstance.getHistoryCmd()).thenReturn(CommandType.COMPLEMENT_DATA.toString()); |
||||
Mockito.when(processInstance.getIsSubProcess()).thenReturn(Flag.NO); |
||||
Mockito.when(processInstance.getScheduleTime()).thenReturn(DateUtils.stringToDate("2020-01-01 00:00:00")); |
||||
Map<String, String> cmdParam = new HashMap<>(); |
||||
cmdParam.put(CMDPARAM_COMPLEMENT_DATA_START_DATE, "2020-01-01 00:00:00"); |
||||
cmdParam.put(CMDPARAM_COMPLEMENT_DATA_END_DATE, "2020-01-31 23:00:00"); |
||||
Mockito.when(processInstance.getCommandParam()).thenReturn(JSONObject.toJSONString(cmdParam)); |
||||
ProcessDefinition processDefinition = new ProcessDefinition(); |
||||
processDefinition.setGlobalParamMap(Collections.EMPTY_MAP); |
||||
processDefinition.setGlobalParamList(Collections.EMPTY_LIST); |
||||
Mockito.when(processInstance.getProcessDefinition()).thenReturn(processDefinition); |
||||
|
||||
masterExecThread = PowerMockito.spy(new MasterExecThread(processInstance, processDao)); |
||||
// prepareProcess init dag
|
||||
Field dag = MasterExecThread.class.getDeclaredField("dag"); |
||||
dag.setAccessible(true); |
||||
dag.set(masterExecThread, new DAG()); |
||||
PowerMockito.doNothing().when(masterExecThread, "executeProcess"); |
||||
PowerMockito.doNothing().when(masterExecThread, "postHandle"); |
||||
PowerMockito.doNothing().when(masterExecThread, "prepareProcess"); |
||||
PowerMockito.doNothing().when(masterExecThread, "runProcess"); |
||||
PowerMockito.doNothing().when(masterExecThread, "endProcess"); |
||||
} |
||||
|
||||
/** |
||||
* without schedule |
||||
* @throws ParseException |
||||
*/ |
||||
@Test |
||||
public void testParallelWithOutSchedule() throws ParseException { |
||||
try{ |
||||
Mockito.when(processDao.queryReleaseSchedulerListByProcessDefinitionId(processDefinitionId)).thenReturn(zeroSchedulerList()); |
||||
Method method = MasterExecThread.class.getDeclaredMethod("executeComplementProcess"); |
||||
method.setAccessible(true); |
||||
method.invoke(masterExecThread); |
||||
// one create save, and 1-30 for next save, and last day 31 no save
|
||||
verify(processDao, times(31)).saveProcessInstance(processInstance); |
||||
}catch (Exception e){ |
||||
e.printStackTrace(); |
||||
Assert.assertTrue(false); |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* with schedule |
||||
* @throws ParseException |
||||
*/ |
||||
@Test |
||||
public void testParallelWithSchedule() throws ParseException { |
||||
try{ |
||||
Mockito.when(processDao.queryReleaseSchedulerListByProcessDefinitionId(processDefinitionId)).thenReturn(oneSchedulerList()); |
||||
Method method = MasterExecThread.class.getDeclaredMethod("executeComplementProcess"); |
||||
method.setAccessible(true); |
||||
method.invoke(masterExecThread); |
||||
// one create save, and 15(1 to 31 step 2) for next save, and last day 31 no save
|
||||
verify(processDao, times(16)).saveProcessInstance(processInstance); |
||||
}catch (Exception e){ |
||||
Assert.assertTrue(false); |
||||
} |
||||
} |
||||
|
||||
private List<Schedule> zeroSchedulerList(){ |
||||
return Collections.EMPTY_LIST; |
||||
} |
||||
|
||||
private List<Schedule> oneSchedulerList(){ |
||||
List<Schedule> schedulerList = new LinkedList<>(); |
||||
Schedule schedule = new Schedule(); |
||||
schedule.setCrontab("0 0 0 1/2 * ?"); |
||||
schedulerList.add(schedule); |
||||
return schedulerList; |
||||
} |
||||
} |
||||
@ -0,0 +1,44 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.server.utils; |
||||
|
||||
import org.apache.dolphinscheduler.common.utils.DateUtils; |
||||
import org.junit.Test; |
||||
import java.util.Date; |
||||
import static org.junit.Assert.assertEquals; |
||||
|
||||
/** |
||||
* Test ScheduleUtils |
||||
*/ |
||||
public class ScheduleUtilsTest { |
||||
|
||||
/** |
||||
* Test the getRecentTriggerTime method |
||||
*/ |
||||
@Test |
||||
public void testGetRecentTriggerTime() { |
||||
Date from = DateUtils.stringToDate("2020-01-01 00:00:00"); |
||||
Date to = DateUtils.stringToDate("2020-01-31 01:00:00"); |
||||
// test date
|
||||
assertEquals(0, ScheduleUtils.getRecentTriggerTime("0 0 0 * * ? ", to, from).size()); |
||||
// test error cron
|
||||
assertEquals(0, ScheduleUtils.getRecentTriggerTime("0 0 0 * *", from, to).size()); |
||||
// test cron
|
||||
assertEquals(31, ScheduleUtils.getRecentTriggerTime("0 0 0 * * ? ", from, to).size()); |
||||
} |
||||
} |
||||
Loading…
Reference in new issue