Browse Source

[Improvement-#11608][task-plugin] New time variables are added to facilitate business development (#11667)

Add new time expressions for users' convenience

Co-authored-by: 冯剑 Jian <jian.feng@jiduauto.com>
Co-authored-by: David <dailidong66@gmail.com>
3.2.0-release
冯剑 2 years ago committed by GitHub
parent
commit
779a0b9c75
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 9
      docs/docs/en/guide/parameter/built-in.md
  2. 49
      docs/docs/zh/guide/parameter/built-in.md
  3. 39
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskConstants.java
  4. 194
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/TimePlaceholderUtils.java
  5. 128
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/parser/TimePlaceholderUtilsTest.java

9
docs/docs/en/guide/parameter/built-in.md

@ -31,4 +31,13 @@
- First N hours:`$[HHmmss-N/24]`
- Next N minutes:`$[HHmmss+N/24/60]`
- First N minutes:`$[HHmmss-N/24/60]`
3. Business Attribute Method
- Today:`$[this_day(yyyy-MM-dd)]`, Such as 2022-08-26 => 2022-08-26
- Yesterday:`$[last_day(yyyy-MM-dd)]`, Such as 2022-08-26 => 2022-08-25
- Week N of the year,start the week on Monday: `$[year_week(yyyy-MM-dd)]`,Such as 2022-08-26 => 2022-34
- Week N of the year,start the week on N:`$[year_week(yyyy-MM-dd,N)]` Such as when N=5, 2022-08-26 => 2022-35
- Before(-)/After(+) The first day of Month(The unit of N is month): `$[month_first_day(yyyy-MM-dd,-N)]`,Such as when N=1, 2022-08-26 => 2022-07-01
- Before(-)/After(+) The last day of Month(The unit of N is month): `$[month_last_day(yyyy-MM-dd,-N)]`,Such as when N=1, 2022-08-28 => 2022-07-31
- Before(-)/After(+) The first day of Week(The unit of N is week): `$[week_first_day(yyyy-MM-dd,-N)]`,Such as when N=1, 2022-08-26 => 2022-08-15
- Before(-)/After(+) The last day of Week(The unit of N is week): `$[week_last_day(yyyy-MM-dd,-N)]`, Such as when N=1, 2022-08-26 => 2022-08-21

49
docs/docs/zh/guide/parameter/built-in.md

@ -29,22 +29,33 @@
- 也可以通过以下两种方式:
1.使用add_months()函数,该函数用于加减月份,
第一个入口参数为[yyyyMMdd],表示返回时间的格式
第二个入口参数为月份偏移量,表示加减多少个月
* 后 N 年:$[add_months(yyyyMMdd,12*N)]
* 前 N 年:$[add_months(yyyyMMdd,-12*N)]
* 后 N 月:$[add_months(yyyyMMdd,N)]
* 前 N 月:$[add_months(yyyyMMdd,-N)]
*******************************************
2.直接加减数字
在自定义格式后直接“+/-”数字
* 后 N 周:$[yyyyMMdd+7*N]
* 前 N 周:$[yyyyMMdd-7*N]
* 后 N 天:$[yyyyMMdd+N]
* 前 N 天:$[yyyyMMdd-N]
* 后 N 小时:$[HHmmss+N/24]
* 前 N 小时:$[HHmmss-N/24]
* 后 N 分钟:$[HHmmss+N/24/60]
* 前 N 分钟:$[HHmmss-N/24/60]
1.使用add_months()函数,该函数用于加减月份,
第一个入口参数为[yyyyMMdd],表示返回时间的格式
第二个入口参数为月份偏移量,表示加减多少个月
* 后 N 年:$[add_months(yyyyMMdd,12*N)]
* 前 N 年:$[add_months(yyyyMMdd,-12*N)]
* 后 N 月:$[add_months(yyyyMMdd,N)]
* 前 N 月:$[add_months(yyyyMMdd,-N)]
*******************************************
2.直接加减数字
在自定义格式后直接“+/-”数字
* 后 N 周:$[yyyyMMdd+7*N]
* 前 N 周:$[yyyyMMdd-7*N]
* 后 N 天:$[yyyyMMdd+N]
* 前 N 天:$[yyyyMMdd-N]
* 后 N 小时:$[HHmmss+N/24]
* 前 N 小时:$[HHmmss-N/24]
* 后 N 分钟:$[HHmmss+N/24/60]
* 前 N 分钟:$[HHmmss-N/24/60]
*******************************************
3.业务属性方式
在自定义格式后直接“+/-”数字
支持日志格式:所有日期表达式,例如:yyyy-MM-dd/yyyyMMddHHmmss
* 当天:$[this_day(yyyy-MM-dd)],如:2022-08-26 => 2022-08-26
* 昨天:$[last_day(yyyy-MM-dd)],如:2022-08-26 => 2022-08-25
* 年的第N周,以周一为起点:$[year_week(yyyy-MM-dd)],如:2022-08-26 => 2022-34
* 年的第N周,以周N为起点:$[year_week(yyyy-MM-dd,N)],如:N=5时 2022-08-26 => 2022-35
* 前(-)/后(+) N 月第一天:$[month_first_day(yyyy-MM-dd,-N)],如:N=1时 2022-08-26 => 2022-07-01
* 前(-)/后(+) N 月最后一天:$[month_last_day(yyyy-MM-dd,-N)],如:N=1时 2022-08-28 => 2022-07-31
* 前(-)/后(+) N 周的周一:$[week_first_day(yyyy-MM-dd,-N)],如:N=1 2022-08-26 => 2022-08-15
* 前(-)/后(+) N 周的周日:$[week_last_day(yyyy-MM-dd,-N)],如:N=1 2022-08-26 => 2022-08-21

39
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskConstants.java

@ -54,6 +54,11 @@ public class TaskConstants {
*/
public static final String COMMA = ",";
/**
* hyphen
*/
public static final String HYPHEN = "-";
/**
* slash /
*/
@ -230,6 +235,40 @@ public class TaskConstants {
* week_end
*/
public static final String WEEK_END = "week_end";
/**
* this_day
*/
public static final String THIS_DAY = "this_day";
/**
* last_day
*/
public static final String LAST_DAY = "last_day";
/**
* month_first_day
*/
public static final String MONTH_FIRST_DAY = "month_first_day";
/**
* month_last_day
*/
public static final String MONTH_LAST_DAY = "month_last_day";
/**
* week_first_day
*/
public static final String WEEK_FIRST_DAY = "week_first_day";
/**
* week_last_day
*/
public static final String WEEK_LAST_DAY = "week_last_day";
/**
* year_week
*
*/
public static final String YEAR_WEEK = "year_week";
/**
* timestamp
*/

194
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/TimePlaceholderUtils.java

@ -17,16 +17,21 @@
package org.apache.dolphinscheduler.plugin.task.api.parser;
import static org.apache.commons.lang3.time.DateUtils.addWeeks;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.ADD_CHAR;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.ADD_MONTHS;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.ADD_STRING;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.COMMA;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.DIVISION_CHAR;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.DIVISION_STRING;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.HYPHEN;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.LAST_DAY;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.LEFT_BRACE_CHAR;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.LEFT_BRACE_STRING;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.MONTH_BEGIN;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.MONTH_END;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.MONTH_FIRST_DAY;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.MONTH_LAST_DAY;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.MULTIPLY_CHAR;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.MULTIPLY_STRING;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.N;
@ -35,9 +40,13 @@ import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.PARAMETE
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.RIGHT_BRACE_CHAR;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.SUBTRACT_CHAR;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.SUBTRACT_STRING;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.THIS_DAY;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TIMESTAMP;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.WEEK_BEGIN;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.WEEK_END;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.WEEK_FIRST_DAY;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.WEEK_LAST_DAY;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.YEAR_WEEK;
import static org.apache.dolphinscheduler.spi.utils.DateUtils.addDays;
import static org.apache.dolphinscheduler.spi.utils.DateUtils.addMinutes;
import static org.apache.dolphinscheduler.spi.utils.DateUtils.addMonths;
@ -47,6 +56,7 @@ import org.apache.dolphinscheduler.spi.utils.StringUtils;
import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import java.util.Map;
@ -351,6 +361,8 @@ public class TimePlaceholderUtils {
Date timestamp = DateUtils.parse(dateStr, PARAMETER_FORMAT_TIME);
value = String.valueOf(timestamp.getTime() / 1000);
} else if (expression.startsWith(YEAR_WEEK)) {
value = calculateYearWeek(expression,date);
} else {
Map.Entry<Date, String> entry = calcTimeExpression(expression, date);
value = DateUtils.format(entry.getKey(), entry.getValue());
@ -363,6 +375,92 @@ public class TimePlaceholderUtils {
return value;
}
/**
* get week of year
* @param expression expression
* @param date date
* @return week of year
*/
public static String calculateYearWeek(String expression, Date date) {
String dataFormat = expression.substring(YEAR_WEEK.length() + 1, expression.length() - 1);
String targetDate = "";
try {
if(dataFormat.contains(COMMA)) {
String param1 = dataFormat.split(COMMA)[0];
String param2 = dataFormat.split(COMMA)[1];
dataFormat = param1;
targetDate = transformYearWeek(date, dataFormat, calculate(param2));
} else {
targetDate = transformYearWeek(date, dataFormat, 1);
}
} catch (Exception e) {
throw new RuntimeException("expression not valid");
}
return targetDate;
}
/**
* transform week of year
* @param date date
* @param format date_format,for example: yyyy-MM-dd / yyyyMMdd
* @param weekDay day of week
* @return date_string
*/
private static String transformYearWeek(Date date, String format, int weekDay) {
Calendar calendar = Calendar.getInstance();
//Minimum number of days required for the first week of the year
calendar.setMinimalDaysInFirstWeek(4);
//By default ,Set Monday as the first day of the week
switch (weekDay) {
case 2:
calendar.setFirstDayOfWeek(Calendar.TUESDAY);
break;
case 3:
calendar.setFirstDayOfWeek(Calendar.WEDNESDAY);
break;
case 4:
calendar.setFirstDayOfWeek(Calendar.THURSDAY);
break;
case 5:
calendar.setFirstDayOfWeek(Calendar.FRIDAY);
break;
case 6:
calendar.setFirstDayOfWeek(Calendar.SATURDAY);
break;
case 7:
calendar.setFirstDayOfWeek(Calendar.SUNDAY);
break;
default:
calendar.setFirstDayOfWeek(Calendar.MONDAY);
break;
}
calendar.setTimeInMillis(date.getTime());
int weekOfYear = calendar.get(Calendar.WEEK_OF_YEAR);
int year = calendar.get(Calendar.YEAR);
String weekYearStr = "";
if (weekOfYear < 10 && format.contains(HYPHEN)) {
weekYearStr = String.format("%d%s0%d",year,HYPHEN,weekOfYear);
} else if (weekOfYear >= 10 && format.contains(HYPHEN)){
weekYearStr = String.format("%d%s%d",year,HYPHEN,weekOfYear);
} else if (weekOfYear < 10){
weekYearStr = String.format("%d0%d",year,weekOfYear);
} else {
weekYearStr = String.format("%d%d",year,weekOfYear);
}
return weekYearStr;
}
/**
* calculate time expresstion
*
@ -383,6 +481,18 @@ public class TimePlaceholderUtils {
resultEntry = calcWeekStart(expression, date);
} else if (expression.startsWith(WEEK_END)) {
resultEntry = calcWeekEnd(expression, date);
} else if (expression.startsWith(MONTH_FIRST_DAY)) {
resultEntry = calcCustomDay(expression,MONTH_FIRST_DAY, date);
} else if (expression.startsWith(MONTH_LAST_DAY)) {
resultEntry = calcCustomDay(expression,MONTH_LAST_DAY, date);
} else if (expression.startsWith(THIS_DAY)) {
resultEntry = calcCustomDay(expression,THIS_DAY, date);
} else if (expression.startsWith(LAST_DAY)) {
resultEntry = calcCustomDay(expression,LAST_DAY, date);
} else if (expression.startsWith(WEEK_FIRST_DAY)) {
resultEntry = calcCustomDay(expression,WEEK_FIRST_DAY, date);
} else if (expression.startsWith(WEEK_LAST_DAY)) {
resultEntry = calcCustomDay(expression,WEEK_LAST_DAY, date);
} else {
resultEntry = calcMinutes(expression, date);
}
@ -438,6 +548,90 @@ public class TimePlaceholderUtils {
throw new RuntimeException("expression not valid");
}
/**
* calculate time expression
* month first day month last day
* @param expression expression
* @param date date
* @return calculate time expression with date,format
*/
public static Map.Entry<Date, String> calcCustomDay(String expression,String keyDate, Date date) {
String dataFormat = "yyyy-MM-dd";
Date targetDate = new Date();
switch (keyDate) {
case MONTH_FIRST_DAY:
dataFormat = expression.substring(MONTH_FIRST_DAY.length() + 1, expression.length() - 1);
if(dataFormat.contains(COMMA)) {
String param1 = dataFormat.split(COMMA)[0];
String param2 = dataFormat.split(COMMA)[1];
dataFormat = param1;
targetDate = addMonths(DateUtils.getFirstDayOfMonth(date),calculate(param2));
} else {
targetDate = DateUtils.getFirstDayOfMonth(date);
}
break;
case MONTH_LAST_DAY:
dataFormat = expression.substring(MONTH_LAST_DAY.length() + 1, expression.length() - 1);
if(dataFormat.contains(COMMA)) {
String param1 = dataFormat.split(COMMA)[0];
String param2 = dataFormat.split(COMMA)[1];
dataFormat = param1;
Date lastMonthDay = addMonths(date,calculate(param2));
targetDate = DateUtils.getLastDayOfMonth(lastMonthDay);
} else {
targetDate = DateUtils.getLastDayOfMonth(date);
}
break;
case THIS_DAY:
dataFormat = expression.substring(THIS_DAY.length() + 1, expression.length() - 1);
targetDate = addDays(date, 0);
break;
case LAST_DAY:
dataFormat = expression.substring(LAST_DAY.length() + 1, expression.length() - 1);
targetDate = addDays(date, -1);
break;
case WEEK_FIRST_DAY:
dataFormat = expression.substring(WEEK_FIRST_DAY.length() + 1, expression.length() - 1);
if(dataFormat.contains(COMMA)) {
String param1 = dataFormat.split(COMMA)[0];
String param2 = dataFormat.split(COMMA)[1];
dataFormat = param1;
targetDate = addWeeks(DateUtils.getMonday(date), calculate(param2));
} else {
targetDate = addWeeks(DateUtils.getMonday(date), 0);
}
break;
case WEEK_LAST_DAY:
dataFormat = expression.substring(WEEK_LAST_DAY.length() + 1, expression.length() - 1);
if(dataFormat.contains(COMMA)) {
String param1 = dataFormat.split(COMMA)[0];
String param2 = dataFormat.split(COMMA)[1];
dataFormat = param1;
targetDate = addWeeks(DateUtils.getSunday(date), calculate(param2));
} else {
targetDate = addWeeks(DateUtils.getSunday(date), 0);
}
break;
default:
break;
}
return new AbstractMap.SimpleImmutableEntry<>(targetDate, dataFormat);
}
/**
* get first day of week
*

128
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/parser/TimePlaceholderUtilsTest.java

@ -0,0 +1,128 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.api.parser;
import org.apache.dolphinscheduler.spi.enums.CommandType;
import org.apache.dolphinscheduler.spi.utils.DateUtils;
import java.util.Date;
import java.util.Map;
import org.junit.Assert;
import org.junit.Test;
public class TimePlaceholderUtilsTest {
Date date = DateUtils.parse("2022-08-26 00:00:00", "yyyy-MM-dd HH:mm:ss");
Map<String, String> timeParams = BusinessTimeUtils.getBusinessTime(CommandType.COMPLEMENT_DATA, date);
@Test
public void timePlaceHolderForThisDay() {
String thisDay = "$[this_day(yyyy-MM-dd)]";
String thisDate = "$[this_day(yyyyMMdd)]";
String thisDayTime = ParameterUtils.convertParameterPlaceholders(thisDay, timeParams);
Assert.assertEquals(thisDayTime, "2022-08-26");
String thisDateTime = ParameterUtils.convertParameterPlaceholders(thisDate, timeParams);
Assert.assertEquals(thisDateTime, "20220826");
}
@Test
public void timePlaceHolderForLastDay() {
String lastDay = "$[last_day(yyyy-MM-dd)]";
String lastDate = "$[last_day(yyyyMMdd)]";
String lastDayTime = ParameterUtils.convertParameterPlaceholders(lastDay, timeParams);
Assert.assertEquals(lastDayTime, "2022-08-25");
String lastDateTime = ParameterUtils.convertParameterPlaceholders(lastDate, timeParams);
Assert.assertEquals(lastDateTime, "20220825");
}
@Test
public void timePlaceHolderForYearWeekDay() {
// Start the week on Monday
String yearWeekDate = "$[year_week(yyyy-MM-dd)]";
String yearWeekDay = "$[year_week(yyyyMMdd)]";
String yearWeekDateTime = ParameterUtils.convertParameterPlaceholders(yearWeekDate, timeParams);
Assert.assertEquals(yearWeekDateTime, "2022-34");
String yearWeekDayTime = ParameterUtils.convertParameterPlaceholders(yearWeekDay, timeParams);
Assert.assertEquals(yearWeekDayTime, "202234");
// Start the week on Friday
String yearWeekDateAny = "$[year_week(yyyy-MM-dd,5)]";
String yearWeekDayAny = "$[year_week(yyyyMMdd,5)]";
String yearWeekDateAnyTime = ParameterUtils.convertParameterPlaceholders(yearWeekDateAny, timeParams);
Assert.assertEquals(yearWeekDateAnyTime, "2022-35");
String yearWeekDayAnyTime = ParameterUtils.convertParameterPlaceholders(yearWeekDayAny, timeParams);
Assert.assertEquals(yearWeekDayAnyTime, "202235");
}
@Test
public void timePlaceHolderForMonthFirstDay() {
String monthFirstDate = "$[month_first_day(yyyy-MM-dd,-1)]";
String monthFirstDay = "$[month_first_day(yyyyMMdd,-1)]";
String monthFirstDateTime = ParameterUtils.convertParameterPlaceholders(monthFirstDate, timeParams);
Assert.assertEquals(monthFirstDateTime, "2022-07-01");
String monthFirstDayTime = ParameterUtils.convertParameterPlaceholders(monthFirstDay, timeParams);
Assert.assertEquals(monthFirstDayTime, "20220701");
}
@Test
public void timePlaceHolderForMonthLastDay() {
String monthLastDate = "$[month_last_day(yyyy-MM-dd,-1)]";
String monthLastDay = "$[month_last_day(yyyyMMdd,-1)]";
String monthLastDateTime = ParameterUtils.convertParameterPlaceholders(monthLastDate, timeParams);
Assert.assertEquals(monthLastDateTime, "2022-07-31");
String monthLastDayTime = ParameterUtils.convertParameterPlaceholders(monthLastDay, timeParams);
Assert.assertEquals(monthLastDayTime, "20220731");
}
@Test
public void timePlaceHolderForWeekFirstDay() {
String weekFirstDate = "$[week_first_day(yyyy-MM-dd,0)]";
String weekFirstDay = "$[week_first_day(yyyyMMdd,0)]";
String weekFirstDateTime = ParameterUtils.convertParameterPlaceholders(weekFirstDate, timeParams);
Assert.assertEquals(weekFirstDateTime, "2022-08-22");
String weekFirstDayTime = ParameterUtils.convertParameterPlaceholders(weekFirstDay, timeParams);
Assert.assertEquals(weekFirstDayTime, "20220822");
}
@Test
public void timePlaceHolderForWeekLastDay() {
String weekLastDate = "$[week_last_day(yyyy-MM-dd,0)]";
String weekLastDay = "$[week_last_day(yyyyMMdd,0)]";
String weekLastDateTime = ParameterUtils.convertParameterPlaceholders(weekLastDate, timeParams);
Assert.assertEquals(weekLastDateTime,"2022-08-28");
String weekLastDayTime = ParameterUtils.convertParameterPlaceholders(weekLastDay, timeParams);
Assert.assertEquals(weekLastDayTime,"20220828");
}
}
Loading…
Cancel
Save