diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index bac74da0b0..c10f443384 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -43,10 +43,6 @@ org.apache.dolphinscheduler dolphinscheduler-dao - - org.apache.dolphinscheduler - dolphinscheduler-server - diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java index c1689c5bec..257f15d580 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java @@ -30,7 +30,7 @@ import org.apache.dolphinscheduler.dao.entity.*; import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; -import org.apache.dolphinscheduler.server.utils.ScheduleUtils; +import org.apache.dolphinscheduler.dao.utils.cron.CronUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -512,8 +512,7 @@ public class ExecutorService extends BaseService{ List listDate = new LinkedList<>(); if(!CollectionUtils.isEmpty(schedules)){ for (Schedule item : schedules) { - List list = ScheduleUtils.getRecentTriggerTime(item.getCrontab(), start, end); - listDate.addAll(list); + listDate.addAll(CronUtils.getSelfFireDateList(start, end, item.getCrontab())); } } if(!CollectionUtils.isEmpty(listDate)){ diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecutorService2Test.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecutorService2Test.java index b4f3e7e31f..66c7a3ebab 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecutorService2Test.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecutorService2Test.java @@ -203,7 +203,7 @@ public class ExecutorService2Test { "", "", RunMode.RUN_MODE_PARALLEL, Priority.LOW, 0, 110); Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); - verify(processDao, times(16)).createCommand(any(Command.class)); + verify(processDao, times(15)).createCommand(any(Command.class)); }catch (Exception e){ Assert.assertTrue(false); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/cron/CronUtils.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/cron/CronUtils.java index 8649462110..8a9087a33c 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/cron/CronUtils.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/cron/CronUtils.java @@ -156,6 +156,23 @@ public class CronUtils { return dateList; } + /** + * gets all scheduled times for a period of time based on self dependency + * @param startTime startTime + * @param endTime endTime + * @param cron cron + * @return date list + */ + public static List getSelfFireDateList(Date startTime, Date endTime, String cron) { + CronExpression cronExpression = null; + try { + cronExpression = parse2CronExpression(cron); + }catch (ParseException e){ + logger.error(e.getMessage(), e); + return Collections.EMPTY_LIST; + } + return getSelfFireDateList(startTime, endTime, cronExpression); + } /** * get expiration time diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/cron/CronUtilsTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/cron/CronUtilsTest.java index c375143d7b..1135cf20f5 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/cron/CronUtilsTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/cron/CronUtilsTest.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.dao.cron; import org.apache.dolphinscheduler.common.enums.CycleEnum; +import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.dao.utils.cron.CronUtils; import com.cronutils.builder.CronBuilder; import com.cronutils.model.Cron; @@ -31,10 +32,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.text.ParseException; +import java.util.Date; import static com.cronutils.model.field.expression.FieldExpressionFactory.*; /** + * CronUtilsTest */ public class CronUtilsTest { @@ -55,8 +58,9 @@ public class CronUtilsTest { .withSecond(on(0)) .instance(); // Obtain the string expression - String cronAsString = cron.asString(); // 0 */5 * * * ? * Every five minutes(once every 5 minutes) + String cronAsString = cron.asString(); + // 0 */5 * * * ? * Every five minutes(once every 5 minutes) Assert.assertEquals(cronAsString, "0 */5 * * * ? *"); } @@ -68,9 +72,6 @@ public class CronUtilsTest { @Test public void testCronParse() throws ParseException { String strCrontab = "0 1 2 3 * ? *"; - strCrontab = "0/50 0/59 * * * ? *"; - strCrontab = "3/5 * 0/5 * * ? *"; - strCrontab = "1/5 3/5 1/5 3/30 * ? *"; Cron depCron = CronUtils.parse2Cron(strCrontab); Assert.assertEquals(depCron.retrieve(CronFieldName.SECOND).getExpression().asString(), "0"); @@ -87,12 +88,14 @@ public class CronUtilsTest { */ @Test public void testScheduleType() throws ParseException { - - CycleEnum cycleEnum = CronUtils.getMaxCycle("0 */1 * * * ? *"); + CycleEnum cycleEnum = CronUtils.getMaxCycle(CronUtils.parse2Cron("0 */1 * * * ? *")); Assert.assertEquals(cycleEnum.name(), "MINUTE"); CycleEnum cycleEnum2 = CronUtils.getMaxCycle("0 * * * * ? *"); Assert.assertEquals(cycleEnum2.name(), "MINUTE"); + + CycleEnum cycleEnum3 = CronUtils.getMiniCycle(CronUtils.parse2Cron("0 * * * * ? *")); + Assert.assertEquals(cycleEnum3.name(), "MINUTE"); } /** @@ -109,26 +112,9 @@ public class CronUtilsTest { .withMinute(every(5)) .withSecond(on(0)) .instance(); - - String cronAsString = cron1.asString(); // 0 */5 * * * ? * once every 5 minutes - //logger.info(cronAsString); - // Obtain the string expression - //String minCrontab = "0 0 * * * ? *"; - //String minCrontab = "0 0 10,14,16 * * ?"; - //String minCrontab = "0 0-5 14 * * ? *"; - //String minCrontab = "0 0 2 ? * SUN *"; - //String minCrontab = "* 0,3 2 SUN * 1#1 *"; - //String minCrontab = "* 0,3 * 1W * ? *"; - //cron = CronUtils.parse2Cron("0 * * * * ? *"); - // month cycle - /*String[] cronArayy = new String[]{"* 0,3 * 1W * ? *","* 0 0 1W * ? *", - "0 0 0 L 3/5 ? *","0 0 0 ? 3/5 2/2 *"};*/ // minute cycle String[] cronArayy = new String[]{"* * * * * ? *","* 0 * * * ? *", "* 5 * * 3/5 ? *","0 0 * * * ? *"}; - // week cycle - /*String[] cronArayy = new String[]{"* * * ? * 2/1 *","0 *//*5 * ? * 2/1 *", - "* * *//*5 ? * 2/1 *"};*/ for(String minCrontab:cronArayy){ if (!org.quartz.CronExpression.isValidExpression(minCrontab)) { throw new RuntimeException(minCrontab+" verify failure, cron expression not valid"); @@ -171,7 +157,6 @@ public class CronUtilsTest { logger.info("dayOfWeekField instanceof And:"+(dayOfWeekField.getExpression() instanceof And)); logger.info("dayOfWeekField instanceof QuestionMark:"+(dayOfWeekField.getExpression() instanceof QuestionMark)); - CycleEnum cycleEnum = CronUtils.getMaxCycle(minCrontab); if(cycleEnum !=null){ logger.info(cycleEnum.name()); @@ -180,4 +165,34 @@ public class CronUtilsTest { } } } -} + + @Test + public void getSelfFireDateList() throws ParseException{ + Date from = DateUtils.stringToDate("2020-01-01 00:00:00"); + Date to = DateUtils.stringToDate("2020-01-31 00:00:00"); + // test date + Assert.assertEquals(0, CronUtils.getSelfFireDateList(to, from, "0 0 0 * * ? ").size()); + // test error cron + Assert.assertEquals(0, CronUtils.getSelfFireDateList(from, to, "0 0 0 * *").size()); + // test cron + Assert.assertEquals(29, CronUtils.getSelfFireDateList(from, to, "0 0 0 * * ? ").size()); + // test other + Assert.assertEquals(30, CronUtils.getFireDateList(from, to, CronUtils.parse2CronExpression("0 0 0 * * ? ")).size()); + Assert.assertEquals(5, CronUtils.getSelfFireDateList(from, to, CronUtils.parse2CronExpression("0 0 0 * * ? "), 5).size()); + } + + @Test + public void getExpirationTime(){ + Date startTime = DateUtils.stringToDate("2020-02-07 18:30:00"); + Date expirationTime = CronUtils.getExpirationTime(startTime, CycleEnum.HOUR); + Assert.assertEquals("2020-02-07 19:30:00", DateUtils.dateToString(expirationTime)); + expirationTime = CronUtils.getExpirationTime(startTime, CycleEnum.DAY); + Assert.assertEquals("2020-02-07 23:59:59", DateUtils.dateToString(expirationTime)); + expirationTime = CronUtils.getExpirationTime(startTime, CycleEnum.WEEK); + Assert.assertEquals("2020-02-07 23:59:59", DateUtils.dateToString(expirationTime)); + expirationTime = CronUtils.getExpirationTime(startTime, CycleEnum.MONTH); + Assert.assertEquals("2020-02-07 23:59:59", DateUtils.dateToString(expirationTime)); + expirationTime = CronUtils.getExpirationTime(startTime, CycleEnum.YEAR); + Assert.assertEquals("2020-02-07 18:30:00", DateUtils.dateToString(expirationTime)); + } +} \ No newline at end of file diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java index 2d0e99a867..2b1ff4d23f 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java @@ -33,9 +33,9 @@ import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.Schedule; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.utils.DagHelper; +import org.apache.dolphinscheduler.dao.utils.cron.CronUtils; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.utils.AlertManager; -import org.apache.dolphinscheduler.server.utils.ScheduleUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -213,8 +213,7 @@ public class MasterExecThread implements Runnable { List listDate = Lists.newLinkedList(); if(!CollectionUtils.isEmpty(schedules)){ for (Schedule schedule : schedules) { - List list = ScheduleUtils.getRecentTriggerTime(schedule.getCrontab(), startDate, endDate); - listDate.addAll(list); + listDate.addAll(CronUtils.getSelfFireDateList(startDate, endDate, schedule.getCrontab())); } } // get first fire date diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ScheduleUtils.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ScheduleUtils.java deleted file mode 100644 index 11730b9545..0000000000 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ScheduleUtils.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.dolphinscheduler.server.utils; - -import org.quartz.impl.triggers.CronTriggerImpl; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.text.ParseException; -import java.util.Date; -import java.util.LinkedList; -import java.util.List; - -/** - * ScheduleUtils - */ -public class ScheduleUtils { - - private static final Logger logger = LoggerFactory.getLogger(ScheduleUtils.class); - - /** - * Get the execution time of the time interval - * @param cron - * @param from - * @param to - * @return - */ - public static List getRecentTriggerTime(String cron, Date from, Date to) { - return getRecentTriggerTime(cron, Integer.MAX_VALUE, from, to); - } - - /** - * Get the execution time of the time interval - * @param cron - * @param size - * @param from - * @param to - * @return - */ - public static List getRecentTriggerTime(String cron, int size, Date from, Date to) { - List list = new LinkedList(); - if(to.before(from)){ - logger.error("schedule date from:{} must before date to:{}!", from, to); - return list; - } - try { - CronTriggerImpl trigger = new CronTriggerImpl(); - trigger.setCronExpression(cron); - trigger.setStartTime(from); - trigger.setEndTime(to); - trigger.computeFirstFireTime(null); - for (int i = 0; i < size; i++) { - Date schedule = trigger.getNextFireTime(); - if(null == schedule){ - break; - } - list.add(schedule); - trigger.triggered(null); - } - } catch (ParseException e) { - logger.error("cron:{} error:{}", cron, e.getMessage()); - } - return java.util.Collections.unmodifiableList(list); - } -} \ No newline at end of file diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java index 6f31e66213..d7c3de13a5 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java @@ -134,7 +134,7 @@ public class MasterExecThreadTest { method.setAccessible(true); method.invoke(masterExecThread); // one create save, and 15(1 to 31 step 2) for next save, and last day 31 no save - verify(processDao, times(16)).saveProcessInstance(processInstance); + verify(processDao, times(15)).saveProcessInstance(processInstance); }catch (Exception e){ Assert.assertTrue(false); } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ScheduleUtilsTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ScheduleUtilsTest.java deleted file mode 100644 index 4fbbdab70f..0000000000 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ScheduleUtilsTest.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.server.utils; - -import org.apache.dolphinscheduler.common.utils.DateUtils; -import org.junit.Test; -import java.util.Date; -import static org.junit.Assert.assertEquals; - -/** - * Test ScheduleUtils - */ -public class ScheduleUtilsTest { - - /** - * Test the getRecentTriggerTime method - */ - @Test - public void testGetRecentTriggerTime() { - Date from = DateUtils.stringToDate("2020-01-01 00:00:00"); - Date to = DateUtils.stringToDate("2020-01-31 01:00:00"); - // test date - assertEquals(0, ScheduleUtils.getRecentTriggerTime("0 0 0 * * ? ", to, from).size()); - // test error cron - assertEquals(0, ScheduleUtils.getRecentTriggerTime("0 0 0 * *", from, to).size()); - // test cron - assertEquals(31, ScheduleUtils.getRecentTriggerTime("0 0 0 * * ? ", from, to).size()); - } -} \ No newline at end of file diff --git a/pom.xml b/pom.xml index 59278f73cd..875577c672 100644 --- a/pom.xml +++ b/pom.xml @@ -706,12 +706,12 @@ **/server/utils/SparkArgsUtilsTest.java **/server/utils/FlinkArgsUtilsTest.java **/server/utils/ParamUtilsTest.java - **/server/utils/ScheduleUtilsTest.java **/server/master/MasterExecThreadTest.java **/dao/mapper/AccessTokenMapperTest.java **/dao/mapper/AlertGroupMapperTest.java **/dao/mapper/AlertMapperTest.java **/dao/mapper/CommandMapperTest.java + **/dao/cron/CronUtilsTest.java **/alert/template/AlertTemplateFactoryTest.java **/alert/template/impl/DefaultHTMLTemplateTest.java **/server/worker/task/datax/DataxTaskTest.java