From fef1a7ef21a811771565aee3e7cba9bd77cfa868 Mon Sep 17 00:00:00 2001 From: Shiwen Cheng Date: Mon, 15 Feb 2021 23:38:42 +0800 Subject: [PATCH 1/5] [Fix-4760][api] Repair dolphinscheduler-postgre.sql error sql (#4774) --- sql/dolphinscheduler-postgre.sql | 7 ++++--- sql/dolphinscheduler_mysql.sql | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/sql/dolphinscheduler-postgre.sql b/sql/dolphinscheduler-postgre.sql index b242205d3f..6d6d60b3b1 100644 --- a/sql/dolphinscheduler-postgre.sql +++ b/sql/dolphinscheduler-postgre.sql @@ -767,9 +767,10 @@ INSERT INTO t_ds_user(user_name, user_password, user_type, email, phone, tenant_ VALUES ('admin', '7ad2410b2f4c074479a8937a28a22b8f', '0', 'xxx@qq.com', '', '0', 1, '2018-03-27 15:48:50', '2018-10-24 17:40:22'); --- Records of t_ds_alertgroup，dolphinscheduler warning group -INSERT INTO t_ds_alertgroup(id,alert_instance_ids, create_user_id, group_name, description, create_time, update_time) -VALUES (1,'1,2', 1, 'default admin warning group', 'default admin warning group', '2018-11-29 10:20:39', '2018-11-29 10:20:39'); +-- Records of t_ds_alertgroup, default admin warning group +INSERT INTO t_ds_alertgroup(alert_instance_ids, create_user_id, group_name, description, create_time, update_time) +VALUES ('1,2', 1, 'default admin warning group', 'default admin warning group', '2018-11-29 10:20:39', + '2018-11-29 10:20:39'); -- Records of t_ds_queue,default queue name : default INSERT INTO t_ds_queue(queue_name, queue, create_time, update_time) diff --git a/sql/dolphinscheduler_mysql.sql b/sql/dolphinscheduler_mysql.sql index bb6588dbec..79964b7eba 100644 --- a/sql/dolphinscheduler_mysql.sql +++ b/sql/dolphinscheduler_mysql.sql @@ -812,7 +812,7 @@ VALUES ('1', '1.3.0'); -- Records of t_ds_alertgroup -- ---------------------------- INSERT INTO `t_ds_alertgroup` -VALUES (1,"1,2", 1, 'default admin warning group', 'default admin warning group', '2018-11-29 10:20:39', +VALUES ("1,2", 1, 'default admin warning group', 'default admin warning group', '2018-11-29 10:20:39', '2018-11-29 10:20:39'); -- ---------------------------- From 5025b5015f6fc6c9918d04434b2ce76832d1b627 Mon Sep 17 00:00:00 2001 From: zhuangchong <37063904+zhuangchong@users.noreply.github.com> Date: Tue, 16 Feb 2021 10:19:27 +0800 Subject: [PATCH 2/5] Repair dolphinscheduler-mysql.sql error sql. (#4780) --- sql/dolphinscheduler_mysql.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/dolphinscheduler_mysql.sql b/sql/dolphinscheduler_mysql.sql index 79964b7eba..2ce4156475 100644 --- a/sql/dolphinscheduler_mysql.sql +++ b/sql/dolphinscheduler_mysql.sql @@ -811,7 +811,7 @@ VALUES ('1', '1.3.0'); -- ---------------------------- -- Records of t_ds_alertgroup -- ---------------------------- -INSERT INTO `t_ds_alertgroup` +INSERT INTO `t_ds_alertgroup`(alert_instance_ids, create_user_id, group_name, description, create_time, update_time) VALUES ("1,2", 1, 'default admin warning group', 'default admin warning group', '2018-11-29 10:20:39', '2018-11-29 10:20:39'); From 18e4087c7e9f995e86a28dcbd7d3ce16ced590f5 Mon Sep 17 00:00:00 2001 From: zhuangchong <37063904+zhuangchong@users.noreply.github.com> Date: Tue, 16 Feb 2021 20:22:01 +0800 Subject: [PATCH 3/5] [Improvement][JSONUtils] Improvement JSONUtils parse object method (#4781) * Improvement JSONUtils parse object method. * update JSONUtils toMap method. * rerun ut test. --- .../common/utils/JSONUtils.java | 29 +++++++++---------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java index 4ab04997d8..73af57929b 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java @@ -211,25 +211,13 @@ public class JSONUtils { /** * json to map - *

* {@link #toMap(String, Class, Class)} * * @param json json * @return json to map */ public static Map toMap(String json) { - if (StringUtils.isEmpty(json)) { - return null; - } - - try { - return objectMapper.readValue(json, new TypeReference>() { - }); - } catch (Exception e) { - logger.error("json to map exception!", e); - } - - return null; + return parseObject(json, new TypeReference>() {}); } /** @@ -243,13 +231,24 @@ public class JSONUtils { * @return to map */ public static Map toMap(String json, Class classK, Class classV) { + return parseObject(json, new TypeReference>() {}); + } + + /** + * json to object + * + * @param json json string + * @param type type reference + * @param + * @return return parse object + */ + public static T parseObject(String json, TypeReference type) { if (StringUtils.isEmpty(json)) { return null; } try { - return objectMapper.readValue(json, new TypeReference>() { - }); + return objectMapper.readValue(json, type); } catch (Exception e) { logger.error("json to map exception!", e); } From d41dd10a1649ee56c362e115354d287bfa09af66 Mon Sep 17 00:00:00 2001 From: Shiwen Cheng Date: Wed, 17 Feb 2021 11:48:47 +0800 Subject: [PATCH 4/5] [Fix][CI] Fix error: unable to resolve action apache/skywalking-eyes@9bd5feb (#4784) --- .github/workflows/ci_backend.yml | 2 +- .github/workflows/ci_e2e.yml | 2 +- .github/workflows/ci_ut.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_backend.yml b/.github/workflows/ci_backend.yml index 7b245d77ef..e19336eab3 100644 --- a/.github/workflows/ci_backend.yml +++ b/.github/workflows/ci_backend.yml @@ -49,7 +49,7 @@ jobs: with: submodule: true - name: Check License Header - uses: apache/skywalking-eyes@9bd5feb + uses: apache/skywalking-eyes@ec88b7d850018c8983f87729ea88549e100c5c82 - name: Set up JDK 1.8 uses: actions/setup-java@v1 with: diff --git a/.github/workflows/ci_e2e.yml b/.github/workflows/ci_e2e.yml index 3e781056d0..cab1eeda61 100644 --- a/.github/workflows/ci_e2e.yml +++ b/.github/workflows/ci_e2e.yml @@ -33,7 +33,7 @@ jobs: with: submodule: true - name: Check License Header - uses: apache/skywalking-eyes@9bd5feb + uses: apache/skywalking-eyes@ec88b7d850018c8983f87729ea88549e100c5c82 - uses: actions/cache@v1 with: path: ~/.m2/repository diff --git a/.github/workflows/ci_ut.yml b/.github/workflows/ci_ut.yml index 09542d97f1..bb1e8c83a9 100644 --- a/.github/workflows/ci_ut.yml +++ b/.github/workflows/ci_ut.yml @@ -36,7 +36,7 @@ jobs: with: submodule: true - name: Check License Header - uses: apache/skywalking-eyes@9bd5feb + uses: apache/skywalking-eyes@ec88b7d850018c8983f87729ea88549e100c5c82 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Only enable review / suggestion here - uses: actions/cache@v1 From 23591b71a2859021e9280899b1f1b6ff3b679fbc Mon Sep 17 00:00:00 2001 From: zhuangchong <37063904+zhuangchong@users.noreply.github.com> Date: Thu, 18 Feb 2021 10:08:23 +0800 Subject: [PATCH 5/5] [Improvement-4751][API] Improvement the duration field modify the display to a time-based minute-second format (e.g. 1d 10h 20m 1s) (#4752) * imp ui instance duration field display content. * update DateUtils format2Duration method. --- .../api/service/ProcessInstanceService.java | 2 +- .../api/service/TaskInstanceService.java | 2 +- .../impl/ProcessDefinitionServiceImpl.java | 2 +- .../common/utils/DateUtils.java | 86 ++++++++++++++++++- .../common/utils/DateUtilsTest.java | 39 +++++++++ .../dao/entity/ProcessInstance.java | 6 +- .../dao/entity/TaskInstance.java | 8 +- 7 files changed, 131 insertions(+), 14 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java index b41ff71f5d..6458a768d8 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java @@ -256,7 +256,7 @@ public class ProcessInstanceService extends BaseService { List processInstances = processInstanceList.getRecords(); for (ProcessInstance processInstance : processInstances) { - processInstance.setDuration(DateUtils.differSec(processInstance.getStartTime(), processInstance.getEndTime())); + processInstance.setDuration(DateUtils.format2Duration(processInstance.getStartTime(), processInstance.getEndTime())); User executor = usersService.queryUser(processInstance.getExecutorId()); if (null != executor) { processInstance.setExecutorName(executor.getUserName()); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskInstanceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskInstanceService.java index dd2caff3b6..6c68202313 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskInstanceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskInstanceService.java @@ -131,7 +131,7 @@ public class TaskInstanceService extends BaseService { List taskInstanceList = taskInstanceIPage.getRecords(); for (TaskInstance taskInstance : taskInstanceList) { - taskInstance.setDuration(DateUtils.differSec(taskInstance.getStartTime(), taskInstance.getEndTime())); + taskInstance.setDuration(DateUtils.format2Duration(taskInstance.getStartTime(), taskInstance.getEndTime())); User executor = usersService.queryUser(taskInstance.getExecutorId()); if (null != executor) { taskInstance.setExecutorName(executor.getUserName()); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java index af91ed2674..2a2ae78618 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java @@ -1327,7 +1327,7 @@ public class ProcessDefinitionServiceImpl extends BaseService implements List processInstanceList = processInstanceService.queryByProcessDefineId(processId, limit); for (ProcessInstance processInstance : processInstanceList) { - processInstance.setDuration(DateUtils.differSec(processInstance.getStartTime(), processInstance.getEndTime())); + processInstance.setDuration(DateUtils.format2Duration(processInstance.getStartTime(), processInstance.getEndTime())); } if (limit > processInstanceList.size()) { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java index 80c0ed411c..a5312995ac 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java @@ -241,15 +241,50 @@ public class DateUtils { */ public static String format2Readable(long ms) { - long days = ms / (1000 * 60 * 60 * 24); - long hours = (ms % (1000 * 60 * 60 * 24)) / (1000 * 60 * 60); - long minutes = (ms % (1000 * 60 * 60)) / (1000 * 60); - long seconds = (ms % (1000 * 60)) / 1000; + long days = MILLISECONDS.toDays(ms); + long hours = MILLISECONDS.toDurationHours(ms); + long minutes = MILLISECONDS.toDurationMinutes(ms); + long seconds = MILLISECONDS.toDurationSeconds(ms); return String.format("%02d %02d:%02d:%02d", days, hours, minutes, seconds); } + /** + * + * format time to duration + * + * @param d1 d1 + * @param d2 d2 + * @return format time + */ + public static String format2Duration(Date d1, Date d2) { + return format2Duration(differMs(d1, d2)); + } + + /** + * format time to duration + * + * @param ms ms + * @return format time + */ + public static String format2Duration(long ms) { + + long days = MILLISECONDS.toDays(ms); + long hours = MILLISECONDS.toDurationHours(ms); + long minutes = MILLISECONDS.toDurationMinutes(ms); + long seconds = MILLISECONDS.toDurationSeconds(ms); + + StringBuilder strBuilder = new StringBuilder(); + strBuilder = days > 0 ? strBuilder.append(days).append("d").append(" ") : strBuilder; + strBuilder = hours > 0 ? strBuilder.append(hours).append("h").append(" ") : strBuilder; + strBuilder = minutes > 0 ? strBuilder.append(minutes).append("m").append(" ") : strBuilder; + strBuilder = seconds > 0 ? strBuilder.append(seconds).append("s") : strBuilder; + + return strBuilder.toString(); + + } + /** * get monday *

@@ -454,4 +489,47 @@ public class DateUtils { return getCurrentTime(Constants.YYYYMMDDHHMMSSSSS); } + static final long C0 = 1L; + static final long C1 = C0 * 1000L; + static final long C2 = C1 * 1000L; + static final long C3 = C2 * 1000L; + static final long C4 = C3 * 60L; + static final long C5 = C4 * 60L; + static final long C6 = C5 * 24L; + + /** + * Time unit representing one thousandth of a second + */ + public static class MILLISECONDS { + + public static long toSeconds(long d) { + return d / (C3 / C2); + } + + public static long toMinutes(long d) { + return d / (C4 / C2); + } + + public static long toHours(long d) { + return d / (C5 / C2); + } + + public static long toDays(long d) { + return d / (C6 / C2); + } + + public static long toDurationSeconds(long d) { + return (d % (C4 / C2)) / (C3 / C2); + } + + public static long toDurationMinutes(long d) { + return (d % (C5 / C2)) / (C4 / C2); + } + + public static long toDurationHours(long d) { + return (d % (C6 / C2)) / (C5 / C2); + } + + } + } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/DateUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/DateUtilsTest.java index fa16446cd8..63f0be5906 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/DateUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/DateUtilsTest.java @@ -157,4 +157,43 @@ public class DateUtilsTest { Assert.assertNotNull(timeStamp); } + @Test + public void testFormat2Duration() { + + // days hours minutes seconds + Date d1 = DateUtils.stringToDate("2020-01-20 11:00:00"); + Date d2 = DateUtils.stringToDate("2020-01-21 12:10:10"); + String duration = DateUtils.format2Duration(d2, d1); + Assert.assertEquals("1d 1h 10m 10s", duration); + + // hours minutes seconds + d1 = DateUtils.stringToDate("2020-01-20 11:00:00"); + d2 = DateUtils.stringToDate("2020-01-20 12:10:10"); + duration = DateUtils.format2Duration(d2, d1); + Assert.assertEquals("1h 10m 10s", duration); + + // minutes seconds + d1 = DateUtils.stringToDate("2020-01-20 11:00:00"); + d2 = DateUtils.stringToDate("2020-01-20 11:10:10"); + duration = DateUtils.format2Duration(d2, d1); + Assert.assertEquals("10m 10s", duration); + + // minutes seconds + d1 = DateUtils.stringToDate("2020-01-20 11:10:00"); + d2 = DateUtils.stringToDate("2020-01-20 11:10:10"); + duration = DateUtils.format2Duration(d2, d1); + Assert.assertEquals("10s", duration); + + d1 = DateUtils.stringToDate("2020-01-20 11:10:00"); + d2 = DateUtils.stringToDate("2020-01-21 11:10:10"); + duration = DateUtils.format2Duration(d2, d1); + Assert.assertEquals("1d 10s", duration); + + d1 = DateUtils.stringToDate("2020-01-20 11:10:00"); + d2 = DateUtils.stringToDate("2020-01-20 16:10:10"); + duration = DateUtils.format2Duration(d2, d1); + Assert.assertEquals("5h 10s", duration); + + } + } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java index f1d43a353b..03e81dc92d 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java @@ -202,7 +202,7 @@ public class ProcessInstance { * @return */ @TableField(exist = false) - private Long duration; + private String duration; /** * process instance priority @@ -547,11 +547,11 @@ public class ProcessInstance { this.dependenceScheduleTimes = dependenceScheduleTimes; } - public Long getDuration() { + public String getDuration() { return duration; } - public void setDuration(Long duration) { + public void setDuration(String duration) { this.duration = duration; } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java index b13ca87e38..ce8d6d58f5 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.dao.entity; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; @@ -170,7 +171,7 @@ public class TaskInstance implements Serializable { * duration */ @TableField(exist = false) - private Long duration; + private String duration; /** * max retry times @@ -437,11 +438,11 @@ public class TaskInstance implements Serializable { this.processInstanceName = processInstanceName; } - public Long getDuration() { + public String getDuration() { return duration; } - public void setDuration(Long duration) { + public void setDuration(String duration) { this.duration = duration; } @@ -505,7 +506,6 @@ public class TaskInstance implements Serializable { return TaskType.CONDITIONS.equals(TaskType.valueOf(this.taskType)); } - /** * determine if you can try again *