From 4bec7921602cc7bc3e9a189427d127853d196121 Mon Sep 17 00:00:00 2001 From: JinYong Li <42576980+JinyLeeChina@users.noreply.github.com> Date: Fri, 19 Nov 2021 16:04:05 +0800 Subject: [PATCH] [Fix][Common] rewrite code generate,fix bit shift (#6914) * rewrite code generate,fix bit shift * fix ut * add algorithm from licenses file * add algorithm from licenses file * add algorithm from licenses file * add algorithm from licenses file * add algorithm from licenses file * fix ut --- .licenserc.yaml | 1 + LICENSE | 2 +- .../service/impl/EnvironmentServiceImpl.java | 8 +- .../impl/ProcessDefinitionServiceImpl.java | 20 ++-- .../api/service/impl/ProjectServiceImpl.java | 8 +- .../impl/TaskDefinitionServiceImpl.java | 8 +- .../common/utils/CodeGenerateUtils.java | 74 +++++++++++++++ .../common/utils/SnowFlakeUtils.java | 94 ------------------- ...lsTest.java => CodeGenerateUtilsTest.java} | 12 +-- .../dao/upgrade/ProcessDefinitionDao.java | 4 +- .../dao/upgrade/ProjectDao.java | 4 +- .../dao/upgrade/UpgradeDao.java | 4 +- dolphinscheduler-dist/release-docs/LICENSE | 1 + .../licenses/LICENSE-snowflake.txt | 11 +++ .../server/PythonGatewayServer.java | 8 +- .../service/process/ProcessService.java | 11 +-- licenses/LICENSE-snowflake.txt | 11 +++ 17 files changed, 142 insertions(+), 139 deletions(-) create mode 100644 dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtils.java delete mode 100644 dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SnowFlakeUtils.java rename dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/{SnowFlakeUtilsTest.java => CodeGenerateUtilsTest.java} (73%) create mode 100644 dolphinscheduler-dist/release-docs/licenses/LICENSE-snowflake.txt create mode 100644 licenses/LICENSE-snowflake.txt diff --git a/.licenserc.yaml b/.licenserc.yaml index 4fdd1d3ac7..baf8cf604b 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -26,6 +26,7 @@ header: - LICENSE - DISCLAIMER - dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ScriptRunner.java + - dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtils.java - mvnw.cmd - dolphinscheduler-dao/src/main/resources/sql/soft_version - .mvn diff --git a/LICENSE b/LICENSE index 8c9beb882f..20191b3d53 100644 --- a/LICENSE +++ b/LICENSE @@ -219,4 +219,4 @@ The text of each license is the standard Apache 2.0 license. DolphinPluginClassLoader from https://github.com/prestosql/presto Apache 2.0 DolphinPluginDiscovery from https://github.com/prestosql/presto Apache 2.0 DolphinPluginLoader from https://github.com/prestosql/presto Apache 2.0 - + CodeGenerateUtils from https://github.com/twitter-archive/snowflake/tree/snowflake-2010 Apache 2.0 diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceImpl.java index a4392902c0..76897775ce 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceImpl.java @@ -23,9 +23,9 @@ import org.apache.dolphinscheduler.api.service.EnvironmentService; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils; +import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils.CodeGenerateException; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils; -import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils.SnowFlakeException; import org.apache.dolphinscheduler.dao.entity.Environment; import org.apache.dolphinscheduler.dao.entity.EnvironmentWorkerGroupRelation; import org.apache.dolphinscheduler.dao.entity.TaskDefinition; @@ -115,9 +115,9 @@ public class EnvironmentServiceImpl extends BaseServiceImpl implements Environme env.setUpdateTime(new Date()); long code = 0L; try { - code = SnowFlakeUtils.getInstance().nextId(); + code = CodeGenerateUtils.getInstance().genCode(); env.setCode(code); - } catch (SnowFlakeException e) { + } catch (CodeGenerateException e) { logger.error("Environment code get error, ", e); } if (code == 0L) { diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java index ae1faf2757..17ca1dd10b 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java @@ -40,10 +40,10 @@ import org.apache.dolphinscheduler.common.graph.DAG; import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.model.TaskNodeRelation; import org.apache.dolphinscheduler.common.thread.Stopper; +import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils; +import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils.CodeGenerateException; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils; -import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils.SnowFlakeException; import org.apache.dolphinscheduler.dao.entity.DagData; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog; @@ -222,8 +222,8 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro } long processDefinitionCode; try { - processDefinitionCode = SnowFlakeUtils.getInstance().nextId(); - } catch (SnowFlakeException e) { + processDefinitionCode = CodeGenerateUtils.getInstance().genCode(); + } catch (CodeGenerateException e) { putMsg(result, Status.INTERNAL_SERVER_ERROR_ARGS); return result; } @@ -874,8 +874,8 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro processDefinition.setProjectCode(projectCode); processDefinition.setUserId(loginUser.getId()); try { - processDefinition.setCode(SnowFlakeUtils.getInstance().nextId()); - } catch (SnowFlakeException e) { + processDefinition.setCode(CodeGenerateUtils.getInstance().genCode()); + } catch (CodeGenerateException e) { putMsg(result, Status.CREATE_PROCESS_DEFINITION_ERROR); return false; } @@ -894,10 +894,10 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro taskDefinitionLog.setOperator(loginUser.getId()); taskDefinitionLog.setOperateTime(now); try { - long code = SnowFlakeUtils.getInstance().nextId(); + long code = CodeGenerateUtils.getInstance().genCode(); taskCodeMap.put(taskDefinitionLog.getCode(), code); taskDefinitionLog.setCode(code); - } catch (SnowFlakeException e) { + } catch (CodeGenerateException e) { logger.error("Task code get error, ", e); putMsg(result, Status.INTERNAL_SERVER_ERROR_ARGS, "Error generating task definition code"); return false; @@ -1363,8 +1363,8 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro processDefinition.setProjectCode(targetProjectCode); if (isCopy) { try { - processDefinition.setCode(SnowFlakeUtils.getInstance().nextId()); - } catch (SnowFlakeException e) { + processDefinition.setCode(CodeGenerateUtils.getInstance().genCode()); + } catch (CodeGenerateException e) { putMsg(result, Status.INTERNAL_SERVER_ERROR_ARGS); throw new ServiceException(Status.INTERNAL_SERVER_ERROR_ARGS); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java index 9fc5aab84d..b4ed942686 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java @@ -25,8 +25,8 @@ import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.UserType; -import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils; -import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils.SnowFlakeException; +import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils; +import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils.CodeGenerateException; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.Project; import org.apache.dolphinscheduler.dao.entity.ProjectUser; @@ -97,14 +97,14 @@ public class ProjectServiceImpl extends BaseServiceImpl implements ProjectServic project = Project .newBuilder() .name(name) - .code(SnowFlakeUtils.getInstance().nextId()) + .code(CodeGenerateUtils.getInstance().genCode()) .description(desc) .userId(loginUser.getId()) .userName(loginUser.getUserName()) .createTime(now) .updateTime(now) .build(); - } catch (SnowFlakeException e) { + } catch (CodeGenerateException e) { putMsg(result, Status.CREATE_PROJECT_ERROR); return result; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java index ee10467001..2ce1c384e3 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java @@ -27,9 +27,9 @@ import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.common.enums.ReleaseState; +import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils; +import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils.CodeGenerateException; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils; -import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils.SnowFlakeException; import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelation; import org.apache.dolphinscheduler.dao.entity.Project; import org.apache.dolphinscheduler.dao.entity.TaskDefinition; @@ -431,9 +431,9 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe List taskCodes = new ArrayList<>(); try { for (int i = 0; i < genNum; i++) { - taskCodes.add(SnowFlakeUtils.getInstance().nextId()); + taskCodes.add(CodeGenerateUtils.getInstance().genCode()); } - } catch (SnowFlakeException e) { + } catch (CodeGenerateException e) { logger.error("Task code get error, ", e); putMsg(result, Status.INTERNAL_SERVER_ERROR_ARGS, "Error generating task definition code"); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtils.java new file mode 100644 index 0000000000..ffea87be51 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtils.java @@ -0,0 +1,74 @@ +/** Copyright 2010-2012 Twitter, Inc.*/ + +package org.apache.dolphinscheduler.common.utils; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Objects; + +/** + * Rewriting based on Twitter snowflake algorithm + */ +public class CodeGenerateUtils { + // start timestamp + private static final long START_TIMESTAMP = 1609430400000L; //2021-01-01 00:00:00 + // Each machine generates 32 in the same millisecond + private static final long LOW_DIGIT_BIT = 5L; + private static final long MIDDLE_BIT = 2L; + private static final long MAX_LOW_DIGIT = ~(-1L << LOW_DIGIT_BIT); + // The displacement to the left + private static final long MIDDLE_LEFT = LOW_DIGIT_BIT; + private static final long HIGH_DIGIT_LEFT = LOW_DIGIT_BIT + MIDDLE_BIT; + private final long machineHash; + private long lowDigit = 0L; + private long recordMillisecond = -1L; + + private static final long SYSTEM_TIMESTAMP = System.currentTimeMillis(); + private static final long SYSTEM_NANOTIME = System.nanoTime(); + + private CodeGenerateUtils() throws CodeGenerateException { + try { + this.machineHash = Math.abs(Objects.hash(InetAddress.getLocalHost().getHostName())) % (2 << (MIDDLE_BIT - 1)); + } catch (UnknownHostException e) { + throw new CodeGenerateException(e.getMessage()); + } + } + + private static CodeGenerateUtils instance = null; + + public static synchronized CodeGenerateUtils getInstance() throws CodeGenerateException { + if (instance == null) { + instance = new CodeGenerateUtils(); + } + return instance; + } + + public synchronized long genCode() throws CodeGenerateException { + long nowtMillisecond = systemMillisecond(); + if (nowtMillisecond < recordMillisecond) { + throw new CodeGenerateException("New code exception because time is set back."); + } + if (nowtMillisecond == recordMillisecond) { + lowDigit = (lowDigit + 1) & MAX_LOW_DIGIT; + if (lowDigit == 0L) { + while (nowtMillisecond <= recordMillisecond) { + nowtMillisecond = systemMillisecond(); + } + } + } else { + lowDigit = 0L; + } + recordMillisecond = nowtMillisecond; + return (nowtMillisecond - START_TIMESTAMP) << HIGH_DIGIT_LEFT | machineHash << MIDDLE_LEFT | lowDigit; + } + + private long systemMillisecond() { + return SYSTEM_TIMESTAMP + (System.nanoTime() - SYSTEM_NANOTIME) / 1000000; + } + + public static class CodeGenerateException extends Exception { + public CodeGenerateException(String message) { + super(message); + } + } +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SnowFlakeUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SnowFlakeUtils.java deleted file mode 100644 index 6393e19b8c..0000000000 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SnowFlakeUtils.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.common.utils; - -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.util.Objects; - -public class SnowFlakeUtils { - // start timestamp - private static final long START_TIMESTAMP = 1609430400000L; //2021-01-01 00:00:00 - // Each machine generates 32 in the same millisecond - private static final long SEQUENCE_BIT = 5; - private static final long MACHINE_BIT = 2; - private static final long MAX_SEQUENCE = ~(-1L << SEQUENCE_BIT); - // The displacement to the left - private static final long MACHINE_LEFT = SEQUENCE_BIT + MACHINE_BIT; - private static final long TIMESTAMP_LEFT = SEQUENCE_BIT + MACHINE_BIT + MACHINE_LEFT; - private final int machineId; - private long sequence = 0L; - private long lastTimestamp = -1L; - - private static final long SYSTEM_TIMESTAMP = System.currentTimeMillis(); - private static final long SYSTEM_NANOTIME = System.nanoTime(); - - private SnowFlakeUtils() throws SnowFlakeException { - try { - this.machineId = Math.abs(Objects.hash(InetAddress.getLocalHost().getHostName())) % 4; - } catch (UnknownHostException e) { - throw new SnowFlakeException(e.getMessage()); - } - } - - private static SnowFlakeUtils instance = null; - - public static synchronized SnowFlakeUtils getInstance() throws SnowFlakeException { - if (instance == null) { - instance = new SnowFlakeUtils(); - } - return instance; - } - - public synchronized long nextId() throws SnowFlakeException { - long currStmp = nowTimestamp(); - if (currStmp < lastTimestamp) { - throw new SnowFlakeException("Clock moved backwards. Refusing to generate id"); - } - if (currStmp == lastTimestamp) { - sequence = (sequence + 1) & MAX_SEQUENCE; - if (sequence == 0L) { - currStmp = getNextMill(); - } - } else { - sequence = 0L; - } - lastTimestamp = currStmp; - return (currStmp - START_TIMESTAMP) << TIMESTAMP_LEFT - | machineId << MACHINE_LEFT - | sequence; - } - - private long getNextMill() { - long mill = nowTimestamp(); - while (mill <= lastTimestamp) { - mill = nowTimestamp(); - } - return mill; - } - - private long nowTimestamp() { - return SYSTEM_TIMESTAMP + (System.nanoTime() - SYSTEM_NANOTIME) / 1000000; - } - - public static class SnowFlakeException extends Exception { - public SnowFlakeException(String message) { - super(message); - } - } -} diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SnowFlakeUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtilsTest.java similarity index 73% rename from dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SnowFlakeUtilsTest.java rename to dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtilsTest.java index e72e87f2c6..d949bd82b8 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SnowFlakeUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtilsTest.java @@ -22,14 +22,14 @@ import java.util.HashSet; import org.junit.Assert; import org.junit.Test; -public class SnowFlakeUtilsTest { +public class CodeGenerateUtilsTest { @Test - public void testNoGenerateDuplicateId() throws SnowFlakeUtils.SnowFlakeException { - HashSet existsSnowFlakeId = new HashSet<>(); + public void testNoGenerateDuplicateCode() throws CodeGenerateUtils.CodeGenerateException { + HashSet existsCode = new HashSet<>(); for (int i = 0; i < 100; i++) { - Long currentId = SnowFlakeUtils.getInstance().nextId(); - Assert.assertFalse(existsSnowFlakeId.contains(currentId)); - existsSnowFlakeId.add(currentId); + Long currentCode = CodeGenerateUtils.getInstance().genCode(); + Assert.assertFalse(existsCode.contains(currentCode)); + existsCode.add(currentCode); } } } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ProcessDefinitionDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ProcessDefinitionDao.java index c41359aa35..f4d198e0a1 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ProcessDefinitionDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ProcessDefinitionDao.java @@ -20,8 +20,8 @@ package org.apache.dolphinscheduler.dao.upgrade; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.common.enums.ReleaseState; +import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils; import org.apache.dolphinscheduler.common.utils.ConnectionUtils; -import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import java.sql.Connection; @@ -110,7 +110,7 @@ public class ProcessDefinitionDao { processDefinition.setId(rs.getInt(1)); long code = rs.getLong(2); if (code == 0L) { - code = SnowFlakeUtils.getInstance().nextId(); + code = CodeGenerateUtils.getInstance().genCode(); } processDefinition.setCode(code); processDefinition.setVersion(Constants.VERSION_FIRST); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ProjectDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ProjectDao.java index 794d71a292..2906902f5a 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ProjectDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ProjectDao.java @@ -17,8 +17,8 @@ package org.apache.dolphinscheduler.dao.upgrade; +import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils; import org.apache.dolphinscheduler.common.utils.ConnectionUtils; -import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils; import java.sql.Connection; import java.sql.PreparedStatement; @@ -51,7 +51,7 @@ public class ProjectDao { Integer id = rs.getInt(1); long code = rs.getLong(2); if (code == 0L) { - code = SnowFlakeUtils.getInstance().nextId(); + code = CodeGenerateUtils.getInstance().genCode(); } projectMap.put(id, code); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java index f4b097f4ec..7d7446978a 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java @@ -25,11 +25,11 @@ import org.apache.dolphinscheduler.common.enums.TaskType; import org.apache.dolphinscheduler.common.enums.TimeoutFlag; import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.TaskTimeoutParameter; +import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils; import org.apache.dolphinscheduler.common.utils.ConnectionUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.SchemaUtils; import org.apache.dolphinscheduler.common.utils.ScriptRunner; -import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils; import org.apache.dolphinscheduler.dao.datasource.ConnectionFactory; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog; @@ -673,7 +673,7 @@ public abstract class UpgradeDao { String name = task.get("name").asText(); taskDefinitionLog.setName(name); taskDefinitionLog.setWorkerGroup(task.get("workerGroup").asText()); - long taskCode = SnowFlakeUtils.getInstance().nextId(); + long taskCode = CodeGenerateUtils.getInstance().genCode(); taskDefinitionLog.setCode(taskCode); taskDefinitionLog.setVersion(Constants.VERSION_FIRST); taskDefinitionLog.setProjectCode(processDefinition.getProjectCode()); diff --git a/dolphinscheduler-dist/release-docs/LICENSE b/dolphinscheduler-dist/release-docs/LICENSE index c008388752..a9abb80840 100644 --- a/dolphinscheduler-dist/release-docs/LICENSE +++ b/dolphinscheduler-dist/release-docs/LICENSE @@ -402,6 +402,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt. protostuff-api 1.7.2: https://github.com/protostuff/protostuff/protostuff-api Apache-2.0 protostuff-collectionschema 1.7.2: https://github.com/protostuff/protostuff/protostuff-collectionschema Apache-2.0 prometheus client_java(simpleclient) 0.12.0: https://github.com/prometheus/client_java, Apache 2.0 + snowflake snowflake-2010: https://github.com/twitter-archive/snowflake/tree/snowflake-2010, Apache 2.0 ======================================================================== BSD licenses diff --git a/dolphinscheduler-dist/release-docs/licenses/LICENSE-snowflake.txt b/dolphinscheduler-dist/release-docs/licenses/LICENSE-snowflake.txt new file mode 100644 index 0000000000..e257174b05 --- /dev/null +++ b/dolphinscheduler-dist/release-docs/licenses/LICENSE-snowflake.txt @@ -0,0 +1,11 @@ +Copyright 2010-2012 Twitter, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +file except in compliance with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed +under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. \ No newline at end of file diff --git a/dolphinscheduler-python/src/main/java/org/apache/dolphinscheduler/server/PythonGatewayServer.java b/dolphinscheduler-python/src/main/java/org/apache/dolphinscheduler/server/PythonGatewayServer.java index 64a94e0170..9ae966fe4f 100644 --- a/dolphinscheduler-python/src/main/java/org/apache/dolphinscheduler/server/PythonGatewayServer.java +++ b/dolphinscheduler-python/src/main/java/org/apache/dolphinscheduler/server/PythonGatewayServer.java @@ -36,7 +36,7 @@ import org.apache.dolphinscheduler.common.enums.RunMode; import org.apache.dolphinscheduler.common.enums.TaskDependType; import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.enums.WarningType; -import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils; +import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.Project; import org.apache.dolphinscheduler.dao.entity.Queue; @@ -150,18 +150,18 @@ public class PythonGatewayServer extends SpringBootServletInitializer { return taskDefinitionService.genTaskCodeList(genNum); } - public Map getCodeAndVersion(String projectName, String taskName) throws SnowFlakeUtils.SnowFlakeException { + public Map getCodeAndVersion(String projectName, String taskName) throws CodeGenerateUtils.CodeGenerateException { Project project = projectMapper.queryByName(projectName); Map result = new HashMap<>(); // project do not exists, mean task not exists too, so we should directly return init value if (project == null) { - result.put("code", SnowFlakeUtils.getInstance().nextId()); + result.put("code", CodeGenerateUtils.getInstance().genCode()); result.put("version", 0L); return result; } TaskDefinition taskDefinition = taskDefinitionMapper.queryByName(project.getCode(), taskName); if (taskDefinition == null) { - result.put("code", SnowFlakeUtils.getInstance().nextId()); + result.put("code", CodeGenerateUtils.getInstance().genCode()); result.put("version", 0L); } else { result.put("code", taskDefinition.getCode()); diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java index e8f7cd8f88..bf0706c40a 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.service.process; +import static java.util.stream.Collectors.toSet; import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_END_DATE; import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_START_DATE; import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_EMPTY_SUB_PROCESS; @@ -27,8 +28,6 @@ import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_SUB_PROCESS import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_SUB_PROCESS_PARENT_INSTANCE_ID; import static org.apache.dolphinscheduler.common.Constants.LOCAL_PARAMS; -import static java.util.stream.Collectors.toSet; - import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.AuthorizationType; import org.apache.dolphinscheduler.common.enums.CommandType; @@ -50,11 +49,11 @@ import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.TaskTimeoutParameter; import org.apache.dolphinscheduler.common.task.subprocess.SubProcessParameters; +import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils; +import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils.CodeGenerateException; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; -import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils; -import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils.SnowFlakeException; import org.apache.dolphinscheduler.common.utils.TaskParametersUtils; import org.apache.dolphinscheduler.dao.entity.Command; import org.apache.dolphinscheduler.dao.entity.DagData; @@ -2290,8 +2289,8 @@ public class ProcessService { taskDefinitionLog.setCreateTime(now); if (taskDefinitionLog.getCode() == 0) { try { - taskDefinitionLog.setCode(SnowFlakeUtils.getInstance().nextId()); - } catch (SnowFlakeException e) { + taskDefinitionLog.setCode(CodeGenerateUtils.getInstance().genCode()); + } catch (CodeGenerateException e) { logger.error("Task code get error, ", e); return Constants.DEFINITION_FAILURE; } diff --git a/licenses/LICENSE-snowflake.txt b/licenses/LICENSE-snowflake.txt new file mode 100644 index 0000000000..e257174b05 --- /dev/null +++ b/licenses/LICENSE-snowflake.txt @@ -0,0 +1,11 @@ +Copyright 2010-2012 Twitter, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +file except in compliance with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed +under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. \ No newline at end of file