Browse Source

Bind processId to constructor CodeGenerator (#15848)

3.2.2-prepare
Wenjun Ruan 9 months ago committed by GitHub
parent
commit
27d0563fe4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
  1. 6
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/python/PythonGateway.java
  2. 2
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ClusterServiceImpl.java
  3. 2
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceImpl.java
  4. 2
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java
  5. 2
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/K8SNamespaceServiceImpl.java
  6. 16
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java
  7. 2
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectParameterServiceImpl.java
  8. 2
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectPreferenceServiceImpl.java
  9. 2
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java
  10. 6
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java
  11. 6
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceTest.java
  12. 93
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtils.java
  13. 53
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtilsTest.java
  14. 2
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
  15. 2
      dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ProcessDefinitionDao.java
  16. 2
      dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ProjectDao.java
  17. 2
      dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/upgrader/v200/V200DolphinSchedulerUpgrader.java
  18. 16
      dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/demo/ProcessDefinitionDemo.java

6
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/python/PythonGateway.java

@ -184,7 +184,7 @@ public class PythonGateway {
Map<String, Long> result = new HashMap<>(); Map<String, Long> result = new HashMap<>();
// project do not exists, mean task not exists too, so we should directly return init value // project do not exists, mean task not exists too, so we should directly return init value
if (project == null) { if (project == null) {
result.put("code", CodeGenerateUtils.getInstance().genCode()); result.put("code", CodeGenerateUtils.genCode());
result.put("version", 0L); result.put("version", 0L);
return result; return result;
} }
@ -194,7 +194,7 @@ public class PythonGateway {
// In the case project exists, but current workflow still not created, we should also return the init // In the case project exists, but current workflow still not created, we should also return the init
// version of it // version of it
if (processDefinition == null) { if (processDefinition == null) {
result.put("code", CodeGenerateUtils.getInstance().genCode()); result.put("code", CodeGenerateUtils.genCode());
result.put("version", 0L); result.put("version", 0L);
return result; return result;
} }
@ -202,7 +202,7 @@ public class PythonGateway {
TaskDefinition taskDefinition = TaskDefinition taskDefinition =
taskDefinitionMapper.queryByName(project.getCode(), processDefinition.getCode(), taskName); taskDefinitionMapper.queryByName(project.getCode(), processDefinition.getCode(), taskName);
if (taskDefinition == null) { if (taskDefinition == null) {
result.put("code", CodeGenerateUtils.getInstance().genCode()); result.put("code", CodeGenerateUtils.genCode());
result.put("version", 0L); result.put("version", 0L);
} else { } else {
result.put("code", taskDefinition.getCode()); result.put("code", taskDefinition.getCode());

2
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ClusterServiceImpl.java

@ -96,7 +96,7 @@ public class ClusterServiceImpl extends BaseServiceImpl implements ClusterServic
cluster.setOperator(loginUser.getId()); cluster.setOperator(loginUser.getId());
cluster.setCreateTime(new Date()); cluster.setCreateTime(new Date());
cluster.setUpdateTime(new Date()); cluster.setUpdateTime(new Date());
cluster.setCode(CodeGenerateUtils.getInstance().genCode()); cluster.setCode(CodeGenerateUtils.genCode());
if (clusterMapper.insert(cluster) > 0) { if (clusterMapper.insert(cluster) > 0) {
return cluster.getCode(); return cluster.getCode();

2
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceImpl.java

@ -123,7 +123,7 @@ public class EnvironmentServiceImpl extends BaseServiceImpl implements Environme
env.setUpdateTime(new Date()); env.setUpdateTime(new Date());
long code = 0L; long code = 0L;
try { try {
code = CodeGenerateUtils.getInstance().genCode(); code = CodeGenerateUtils.genCode();
env.setCode(code); env.setCode(code);
} catch (CodeGenerateException e) { } catch (CodeGenerateException e) {
log.error("Generate environment code error.", e); log.error("Generate environment code error.", e);

2
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java

@ -258,7 +258,7 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ
checkScheduleTimeNumExceed(commandType, cronTime); checkScheduleTimeNumExceed(commandType, cronTime);
checkMasterExists(); checkMasterExists();
long triggerCode = CodeGenerateUtils.getInstance().genCode(); long triggerCode = CodeGenerateUtils.genCode();
/** /**
* create command * create command

2
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/K8SNamespaceServiceImpl.java

@ -141,7 +141,7 @@ public class K8SNamespaceServiceImpl extends BaseServiceImpl implements K8sNames
long code = 0L; long code = 0L;
try { try {
code = CodeGenerateUtils.getInstance().genCode(); code = CodeGenerateUtils.genCode();
cluster.setCode(code); cluster.setCode(code);
} catch (CodeGenerateUtils.CodeGenerateException e) { } catch (CodeGenerateUtils.CodeGenerateException e) {
log.error("Generate cluster code error.", e); log.error("Generate cluster code error.", e);

16
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java

@ -299,7 +299,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
List<TaskDefinitionLog> taskDefinitionLogs = generateTaskDefinitionList(taskDefinitionJson); List<TaskDefinitionLog> taskDefinitionLogs = generateTaskDefinitionList(taskDefinitionJson);
List<ProcessTaskRelationLog> taskRelationList = generateTaskRelationList(taskRelationJson, taskDefinitionLogs); List<ProcessTaskRelationLog> taskRelationList = generateTaskRelationList(taskRelationJson, taskDefinitionLogs);
long processDefinitionCode = CodeGenerateUtils.getInstance().genCode(); long processDefinitionCode = CodeGenerateUtils.genCode();
ProcessDefinition processDefinition = ProcessDefinition processDefinition =
new ProcessDefinition(projectCode, name, processDefinitionCode, description, new ProcessDefinition(projectCode, name, processDefinitionCode, description,
globalParams, locations, timeout, loginUser.getId()); globalParams, locations, timeout, loginUser.getId());
@ -360,7 +360,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
long processDefinitionCode; long processDefinitionCode;
try { try {
processDefinitionCode = CodeGenerateUtils.getInstance().genCode(); processDefinitionCode = CodeGenerateUtils.genCode();
} catch (CodeGenerateException e) { } catch (CodeGenerateException e) {
throw new ServiceException(Status.INTERNAL_SERVER_ERROR_ARGS); throw new ServiceException(Status.INTERNAL_SERVER_ERROR_ARGS);
} }
@ -1233,7 +1233,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
// build process definition // build process definition
processDefinition = new ProcessDefinition(projectCode, processDefinition = new ProcessDefinition(projectCode,
processDefinitionName, processDefinitionName,
CodeGenerateUtils.getInstance().genCode(), CodeGenerateUtils.genCode(),
"", "",
"[]", null, "[]", null,
0, loginUser.getId()); 0, loginUser.getId());
@ -1388,7 +1388,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
sqlParameters.setSqlType(SqlType.NON_QUERY.ordinal()); sqlParameters.setSqlType(SqlType.NON_QUERY.ordinal());
sqlParameters.setLocalParams(Collections.emptyList()); sqlParameters.setLocalParams(Collections.emptyList());
taskDefinition.setTaskParams(JSONUtils.toJsonString(sqlParameters)); taskDefinition.setTaskParams(JSONUtils.toJsonString(sqlParameters));
taskDefinition.setCode(CodeGenerateUtils.getInstance().genCode()); taskDefinition.setCode(CodeGenerateUtils.genCode());
taskDefinition.setTaskType(TASK_TYPE_SQL); taskDefinition.setTaskType(TASK_TYPE_SQL);
taskDefinition.setFailRetryTimes(0); taskDefinition.setFailRetryTimes(0);
taskDefinition.setFailRetryInterval(0); taskDefinition.setFailRetryInterval(0);
@ -1433,7 +1433,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
processDefinition.setProjectCode(projectCode); processDefinition.setProjectCode(projectCode);
processDefinition.setUserId(loginUser.getId()); processDefinition.setUserId(loginUser.getId());
try { try {
processDefinition.setCode(CodeGenerateUtils.getInstance().genCode()); processDefinition.setCode(CodeGenerateUtils.genCode());
} catch (CodeGenerateException e) { } catch (CodeGenerateException e) {
log.error( log.error(
"Save process definition error because generate process definition code error, projectCode:{}.", "Save process definition error because generate process definition code error, projectCode:{}.",
@ -1456,7 +1456,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
taskDefinitionLog.setOperator(loginUser.getId()); taskDefinitionLog.setOperator(loginUser.getId());
taskDefinitionLog.setOperateTime(now); taskDefinitionLog.setOperateTime(now);
try { try {
long code = CodeGenerateUtils.getInstance().genCode(); long code = CodeGenerateUtils.genCode();
taskCodeMap.put(taskDefinitionLog.getCode(), code); taskCodeMap.put(taskDefinitionLog.getCode(), code);
taskDefinitionLog.setCode(code); taskDefinitionLog.setCode(code);
} catch (CodeGenerateException e) { } catch (CodeGenerateException e) {
@ -2074,7 +2074,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
Map<Long, Long> taskCodeMap = new HashMap<>(); Map<Long, Long> taskCodeMap = new HashMap<>();
for (TaskDefinitionLog taskDefinitionLog : taskDefinitionLogs) { for (TaskDefinitionLog taskDefinitionLog : taskDefinitionLogs) {
try { try {
long taskCode = CodeGenerateUtils.getInstance().genCode(); long taskCode = CodeGenerateUtils.genCode();
taskCodeMap.put(taskDefinitionLog.getCode(), taskCode); taskCodeMap.put(taskDefinitionLog.getCode(), taskCode);
taskDefinitionLog.setCode(taskCode); taskDefinitionLog.setCode(taskCode);
} catch (CodeGenerateException e) { } catch (CodeGenerateException e) {
@ -2097,7 +2097,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
} }
final long oldProcessDefinitionCode = processDefinition.getCode(); final long oldProcessDefinitionCode = processDefinition.getCode();
try { try {
processDefinition.setCode(CodeGenerateUtils.getInstance().genCode()); processDefinition.setCode(CodeGenerateUtils.genCode());
} catch (CodeGenerateException e) { } catch (CodeGenerateException e) {
log.error("Generate process definition code error, projectCode:{}.", targetProjectCode, e); log.error("Generate process definition code error, projectCode:{}.", targetProjectCode, e);
putMsg(result, Status.INTERNAL_SERVER_ERROR_ARGS); putMsg(result, Status.INTERNAL_SERVER_ERROR_ARGS);

2
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectParameterServiceImpl.java

@ -97,7 +97,7 @@ public class ProjectParameterServiceImpl extends BaseServiceImpl implements Proj
.builder() .builder()
.paramName(projectParameterName) .paramName(projectParameterName)
.paramValue(projectParameterValue) .paramValue(projectParameterValue)
.code(CodeGenerateUtils.getInstance().genCode()) .code(CodeGenerateUtils.genCode())
.projectCode(projectCode) .projectCode(projectCode)
.userId(loginUser.getId()) .userId(loginUser.getId())
.createTime(now) .createTime(now)

2
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectPreferenceServiceImpl.java

@ -76,7 +76,7 @@ public class ProjectPreferenceServiceImpl extends BaseServiceImpl
projectPreference.setProjectCode(projectCode); projectPreference.setProjectCode(projectCode);
projectPreference.setPreferences(preferences); projectPreference.setPreferences(preferences);
projectPreference.setUserId(loginUser.getId()); projectPreference.setUserId(loginUser.getId());
projectPreference.setCode(CodeGenerateUtils.getInstance().genCode()); projectPreference.setCode(CodeGenerateUtils.genCode());
projectPreference.setState(1); projectPreference.setState(1);
projectPreference.setCreateTime(now); projectPreference.setCreateTime(now);
projectPreference.setUpdateTime(now); projectPreference.setUpdateTime(now);

2
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java

@ -126,7 +126,7 @@ public class ProjectServiceImpl extends BaseServiceImpl implements ProjectServic
project = Project project = Project
.builder() .builder()
.name(name) .name(name)
.code(CodeGenerateUtils.getInstance().genCode()) .code(CodeGenerateUtils.genCode())
.description(desc) .description(desc)
.userId(loginUser.getId()) .userId(loginUser.getId())
.userName(loginUser.getUserName()) .userName(loginUser.getUserName())

6
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java

@ -265,7 +265,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe
long taskDefinitionCode; long taskDefinitionCode;
try { try {
taskDefinitionCode = CodeGenerateUtils.getInstance().genCode(); taskDefinitionCode = CodeGenerateUtils.genCode();
} catch (CodeGenerateException e) { } catch (CodeGenerateException e) {
throw new ServiceException(Status.INTERNAL_SERVER_ERROR_ARGS); throw new ServiceException(Status.INTERNAL_SERVER_ERROR_ARGS);
} }
@ -338,7 +338,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe
} }
long taskCode = taskDefinition.getCode(); long taskCode = taskDefinition.getCode();
if (taskCode == 0) { if (taskCode == 0) {
taskDefinition.setCode(CodeGenerateUtils.getInstance().genCode()); taskDefinition.setCode(CodeGenerateUtils.genCode());
} }
List<ProcessTaskRelationLog> processTaskRelationLogList = List<ProcessTaskRelationLog> processTaskRelationLogList =
processTaskRelationMapper.queryByProcessCode(processDefinitionCode) processTaskRelationMapper.queryByProcessCode(processDefinitionCode)
@ -1264,7 +1264,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe
List<Long> taskCodes = new ArrayList<>(); List<Long> taskCodes = new ArrayList<>();
try { try {
for (int i = 0; i < genNum; i++) { for (int i = 0; i < genNum; i++) {
taskCodes.add(CodeGenerateUtils.getInstance().genCode()); taskCodes.add(CodeGenerateUtils.genCode());
} }
} catch (CodeGenerateException e) { } catch (CodeGenerateException e) {
log.error("Generate task definition code error.", e); log.error("Generate task definition code error.", e);

6
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceTest.java

@ -96,9 +96,6 @@ public class EnvironmentServiceTest {
@Mock @Mock
private ResourcePermissionCheckService resourcePermissionCheckService; private ResourcePermissionCheckService resourcePermissionCheckService;
@Mock
private CodeGenerateUtils codeGenerateUtils;
public static final String testUserName = "environmentServerTest"; public static final String testUserName = "environmentServerTest";
public static final String environmentName = "Env1"; public static final String environmentName = "Env1";
@ -141,8 +138,7 @@ public class EnvironmentServiceTest {
() -> environmentService.createEnvironment(adminUser, "testName", "test", "test", workerGroups)); () -> environmentService.createEnvironment(adminUser, "testName", "test", "test", workerGroups));
try (MockedStatic<CodeGenerateUtils> ignored = Mockito.mockStatic(CodeGenerateUtils.class)) { try (MockedStatic<CodeGenerateUtils> ignored = Mockito.mockStatic(CodeGenerateUtils.class)) {
when(CodeGenerateUtils.getInstance()).thenReturn(codeGenerateUtils); when(CodeGenerateUtils.genCode()).thenThrow(CodeGenerateUtils.CodeGenerateException.class);
when(codeGenerateUtils.genCode()).thenThrow(CodeGenerateUtils.CodeGenerateException.class);
assertThrowsServiceException(Status.INTERNAL_SERVER_ERROR_ARGS, assertThrowsServiceException(Status.INTERNAL_SERVER_ERROR_ARGS,
() -> environmentService.createEnvironment(adminUser, "testName", "test", "test", workerGroups)); () -> environmentService.createEnvironment(adminUser, "testName", "test", "test", workerGroups));

93
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtils.java

@ -1,4 +1,6 @@
/** Copyright 2010-2012 Twitter, Inc.*/ /**
* Copyright 2010-2012 Twitter, Inc.
*/
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
@ -6,66 +8,71 @@ import java.net.InetAddress;
import java.net.UnknownHostException; import java.net.UnknownHostException;
import java.util.Objects; import java.util.Objects;
import lombok.extern.slf4j.Slf4j;
/** /**
* Rewriting based on Twitter snowflake algorithm * Rewriting based on Twitter snowflake algorithm
*/ */
@Slf4j
public class CodeGenerateUtils { public class CodeGenerateUtils {
// start timestamp private static final CodeGenerator codeGenerator;
private static final long START_TIMESTAMP = 1609430400000L; // 2021-01-01 00:00:00
// Each machine generates 32 in the same millisecond
private static final long LOW_DIGIT_BIT = 5L;
private static final long MIDDLE_BIT = 2L;
private static final long MAX_LOW_DIGIT = ~(-1L << LOW_DIGIT_BIT);
// The displacement to the left
private static final long MIDDLE_LEFT = LOW_DIGIT_BIT;
private static final long HIGH_DIGIT_LEFT = LOW_DIGIT_BIT + MIDDLE_BIT;
private final long machineHash;
private long lowDigit = 0L;
private long recordMillisecond = -1L;
private static final long SYSTEM_TIMESTAMP = System.currentTimeMillis();
private static final long SYSTEM_NANOTIME = System.nanoTime();
private CodeGenerateUtils() throws CodeGenerateException { static {
try { try {
this.machineHash = codeGenerator = new CodeGenerator(InetAddress.getLocalHost().getHostName() + "-" + OSUtils.getProcessID());
Math.abs(Objects.hash(InetAddress.getLocalHost().getHostName())) % (2 << (MIDDLE_BIT - 1));
} catch (UnknownHostException e) { } catch (UnknownHostException e) {
throw new CodeGenerateException(e.getMessage()); throw new CodeGenerateException(e.getMessage());
} }
} }
private static CodeGenerateUtils instance = null; public static long genCode() throws CodeGenerateException {
return codeGenerator.genCode();
public static synchronized CodeGenerateUtils getInstance() throws CodeGenerateException {
if (instance == null) {
instance = new CodeGenerateUtils();
}
return instance;
} }
public synchronized long genCode() throws CodeGenerateException { public static class CodeGenerator {
long nowtMillisecond = systemMillisecond();
if (nowtMillisecond < recordMillisecond) { // start timestamp
throw new CodeGenerateException("New code exception because time is set back."); private static final long START_TIMESTAMP = 1609430400000L; // 2021-01-01 00:00:00
// Each machine generates 32 in the same millisecond
private static final long LOW_DIGIT_BIT = 5L;
private static final long MACHINE_BIT = 5L;
private static final long MAX_LOW_DIGIT = ~(-1L << LOW_DIGIT_BIT);
// The displacement to the left
private static final long HIGH_DIGIT_LEFT = LOW_DIGIT_BIT + MACHINE_BIT;
public final long machineHash;
private long lowDigit = 0L;
private long recordMillisecond = -1L;
private static final long SYSTEM_TIMESTAMP = System.currentTimeMillis();
private static final long SYSTEM_NANOTIME = System.nanoTime();
public CodeGenerator(String appName) {
this.machineHash = Math.abs(Objects.hash(appName)) % (1 << MACHINE_BIT);
} }
if (nowtMillisecond == recordMillisecond) {
lowDigit = (lowDigit + 1) & MAX_LOW_DIGIT; public synchronized long genCode() throws CodeGenerateException {
if (lowDigit == 0L) { long nowtMillisecond = systemMillisecond();
while (nowtMillisecond <= recordMillisecond) { if (nowtMillisecond < recordMillisecond) {
nowtMillisecond = systemMillisecond(); throw new CodeGenerateException("New code exception because time is set back.");
}
if (nowtMillisecond == recordMillisecond) {
lowDigit = (lowDigit + 1) & MAX_LOW_DIGIT;
if (lowDigit == 0L) {
while (nowtMillisecond <= recordMillisecond) {
nowtMillisecond = systemMillisecond();
}
} }
} else {
lowDigit = 0L;
} }
} else { recordMillisecond = nowtMillisecond;
lowDigit = 0L; return (nowtMillisecond - START_TIMESTAMP) << HIGH_DIGIT_LEFT | machineHash << LOW_DIGIT_BIT | lowDigit;
} }
recordMillisecond = nowtMillisecond;
return (nowtMillisecond - START_TIMESTAMP) << HIGH_DIGIT_LEFT | machineHash << MIDDLE_LEFT | lowDigit;
}
private long systemMillisecond() { private long systemMillisecond() {
return SYSTEM_TIMESTAMP + (System.nanoTime() - SYSTEM_NANOTIME) / 1000000; return SYSTEM_TIMESTAMP + (System.nanoTime() - SYSTEM_NANOTIME) / 1000000;
}
} }
public static class CodeGenerateException extends RuntimeException { public static class CodeGenerateException extends RuntimeException {

53
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtilsTest.java

@ -17,20 +17,59 @@
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
import static org.junit.jupiter.api.Assertions.assertEquals;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.HashSet; import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
public class CodeGenerateUtilsTest { class CodeGenerateUtilsTest {
@Test @Test
public void testNoGenerateDuplicateCode() throws CodeGenerateUtils.CodeGenerateException { void testNoGenerateDuplicateCode() {
HashSet<Long> existsCode = new HashSet<>(); int codeNum = 10000000;
for (int i = 0; i < 100; i++) { List<Long> existsCode = new ArrayList<>();
Long currentCode = CodeGenerateUtils.getInstance().genCode(); for (int i = 0; i < codeNum; i++) {
Assertions.assertFalse(existsCode.contains(currentCode)); Long currentCode = CodeGenerateUtils.genCode();
existsCode.add(currentCode); existsCode.add(currentCode);
} }
Set<Long> existsCodeSet = new HashSet<>(existsCode);
// Disallow duplicate code
assertEquals(existsCode.size(), existsCodeSet.size());
}
@Test
void testNoGenerateDuplicateCodeWithDifferentAppName() throws UnknownHostException, InterruptedException {
int threadNum = 10;
int codeNum = 1000000;
final String hostName = InetAddress.getLocalHost().getHostName();
Map<String, List<Long>> machineCodes = new ConcurrentHashMap<>();
CountDownLatch countDownLatch = new CountDownLatch(threadNum);
for (int i = 0; i < threadNum; i++) {
final int c = i;
new Thread(() -> {
List<Long> codes = new ArrayList<>(codeNum);
CodeGenerateUtils.CodeGenerator codeGenerator = new CodeGenerateUtils.CodeGenerator(hostName + "-" + c);
for (int j = 0; j < codeNum; j++) {
codes.add(codeGenerator.genCode());
}
machineCodes.put(Thread.currentThread().getName(), codes);
countDownLatch.countDown();
}).start();
}
countDownLatch.await();
Set<Long> totalCodes = new HashSet<>();
machineCodes.values().forEach(totalCodes::addAll);
assertEquals(codeNum * threadNum, totalCodes.size());
} }
} }

2
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java

@ -1679,7 +1679,7 @@ public class ProcessServiceImpl implements ProcessService {
taskDefinitionLog.setOperateTime(now); taskDefinitionLog.setOperateTime(now);
taskDefinitionLog.setOperator(operator.getId()); taskDefinitionLog.setOperator(operator.getId());
if (taskDefinitionLog.getCode() == 0) { if (taskDefinitionLog.getCode() == 0) {
taskDefinitionLog.setCode(CodeGenerateUtils.getInstance().genCode()); taskDefinitionLog.setCode(CodeGenerateUtils.genCode());
} }
if (taskDefinitionLog.getVersion() == 0) { if (taskDefinitionLog.getVersion() == 0) {
// init first version // init first version

2
dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ProcessDefinitionDao.java

@ -100,7 +100,7 @@ public class ProcessDefinitionDao {
processDefinition.setId(rs.getInt(1)); processDefinition.setId(rs.getInt(1));
long code = rs.getLong(2); long code = rs.getLong(2);
if (code == 0L) { if (code == 0L) {
code = CodeGenerateUtils.getInstance().genCode(); code = CodeGenerateUtils.genCode();
} }
processDefinition.setCode(code); processDefinition.setCode(code);
processDefinition.setVersion(Constants.VERSION_FIRST); processDefinition.setVersion(Constants.VERSION_FIRST);

2
dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ProjectDao.java

@ -47,7 +47,7 @@ public class ProjectDao {
Integer id = rs.getInt(1); Integer id = rs.getInt(1);
long code = rs.getLong(2); long code = rs.getLong(2);
if (code == 0L) { if (code == 0L) {
code = CodeGenerateUtils.getInstance().genCode(); code = CodeGenerateUtils.genCode();
} }
projectMap.put(id, code); projectMap.put(id, code);
} }

2
dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/upgrader/v200/V200DolphinSchedulerUpgrader.java

@ -206,7 +206,7 @@ public class V200DolphinSchedulerUpgrader implements DolphinSchedulerUpgrader {
taskDefinitionLog.setName(name); taskDefinitionLog.setName(name);
taskDefinitionLog taskDefinitionLog
.setWorkerGroup(task.get("workerGroup") == null ? "default" : task.get("workerGroup").asText()); .setWorkerGroup(task.get("workerGroup") == null ? "default" : task.get("workerGroup").asText());
long taskCode = CodeGenerateUtils.getInstance().genCode(); long taskCode = CodeGenerateUtils.genCode();
taskDefinitionLog.setCode(taskCode); taskDefinitionLog.setCode(taskCode);
taskDefinitionLog.setVersion(Constants.VERSION_FIRST); taskDefinitionLog.setVersion(Constants.VERSION_FIRST);
taskDefinitionLog.setProjectCode(processDefinition.getProjectCode()); taskDefinitionLog.setProjectCode(processDefinition.getProjectCode());

16
dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/demo/ProcessDefinitionDemo.java

@ -85,7 +85,7 @@ public class ProcessDefinitionDemo {
project = Project project = Project
.builder() .builder()
.name("demo") .name("demo")
.code(CodeGenerateUtils.getInstance().genCode()) .code(CodeGenerateUtils.genCode())
.description("") .description("")
.userId(loginUser.getId()) .userId(loginUser.getId())
.userName(loginUser.getUserName()) .userName(loginUser.getUserName())
@ -167,7 +167,7 @@ public class ProcessDefinitionDemo {
List<Long> taskCodes = new ArrayList<>(); List<Long> taskCodes = new ArrayList<>();
try { try {
for (int i = 0; i < 1; i++) { for (int i = 0; i < 1; i++) {
taskCodes.add(CodeGenerateUtils.getInstance().genCode()); taskCodes.add(CodeGenerateUtils.genCode());
} }
} catch (CodeGenerateUtils.CodeGenerateException e) { } catch (CodeGenerateUtils.CodeGenerateException e) {
log.error("task code get error, ", e); log.error("task code get error, ", e);
@ -242,7 +242,7 @@ public class ProcessDefinitionDemo {
List<Long> taskCodes = new ArrayList<>(); List<Long> taskCodes = new ArrayList<>();
try { try {
for (int i = 0; i < 2; i++) { for (int i = 0; i < 2; i++) {
taskCodes.add(CodeGenerateUtils.getInstance().genCode()); taskCodes.add(CodeGenerateUtils.genCode());
} }
} catch (CodeGenerateUtils.CodeGenerateException e) { } catch (CodeGenerateUtils.CodeGenerateException e) {
log.error("task code get error, ", e); log.error("task code get error, ", e);
@ -334,7 +334,7 @@ public class ProcessDefinitionDemo {
List<Long> taskCodes = new ArrayList<>(); List<Long> taskCodes = new ArrayList<>();
try { try {
for (int i = 0; i < 2; i++) { for (int i = 0; i < 2; i++) {
taskCodes.add(CodeGenerateUtils.getInstance().genCode()); taskCodes.add(CodeGenerateUtils.genCode());
} }
} catch (CodeGenerateUtils.CodeGenerateException e) { } catch (CodeGenerateUtils.CodeGenerateException e) {
log.error("task code get error, ", e); log.error("task code get error, ", e);
@ -420,7 +420,7 @@ public class ProcessDefinitionDemo {
List<Long> taskCodes = new ArrayList<>(); List<Long> taskCodes = new ArrayList<>();
try { try {
for (int i = 0; i < 4; i++) { for (int i = 0; i < 4; i++) {
taskCodes.add(CodeGenerateUtils.getInstance().genCode()); taskCodes.add(CodeGenerateUtils.genCode());
} }
} catch (CodeGenerateUtils.CodeGenerateException e) { } catch (CodeGenerateUtils.CodeGenerateException e) {
log.error("task code get error, ", e); log.error("task code get error, ", e);
@ -537,7 +537,7 @@ public class ProcessDefinitionDemo {
List<Long> taskCodes = new ArrayList<>(); List<Long> taskCodes = new ArrayList<>();
try { try {
for (int i = 0; i < 4; i++) { for (int i = 0; i < 4; i++) {
taskCodes.add(CodeGenerateUtils.getInstance().genCode()); taskCodes.add(CodeGenerateUtils.genCode());
} }
} catch (CodeGenerateUtils.CodeGenerateException e) { } catch (CodeGenerateUtils.CodeGenerateException e) {
log.error("task code get error, ", e); log.error("task code get error, ", e);
@ -656,7 +656,7 @@ public class ProcessDefinitionDemo {
List<Long> taskCodes = new ArrayList<>(); List<Long> taskCodes = new ArrayList<>();
try { try {
for (int i = 0; i < 3; i++) { for (int i = 0; i < 3; i++) {
taskCodes.add(CodeGenerateUtils.getInstance().genCode()); taskCodes.add(CodeGenerateUtils.genCode());
} }
} catch (CodeGenerateUtils.CodeGenerateException e) { } catch (CodeGenerateUtils.CodeGenerateException e) {
log.error("task code get error, ", e); log.error("task code get error, ", e);
@ -755,7 +755,7 @@ public class ProcessDefinitionDemo {
List<Long> taskCodes = new ArrayList<>(); List<Long> taskCodes = new ArrayList<>();
try { try {
for (int i = 0; i < 1; i++) { for (int i = 0; i < 1; i++) {
taskCodes.add(CodeGenerateUtils.getInstance().genCode()); taskCodes.add(CodeGenerateUtils.genCode());
} }
} catch (CodeGenerateUtils.CodeGenerateException e) { } catch (CodeGenerateUtils.CodeGenerateException e) {
log.error("task code get error, ", e); log.error("task code get error, ", e);

Loading…
Cancel
Save