Browse Source

Merge pull request #20 from apache/dev

update from apache
pull/3/MERGE
BoYiZhang 4 years ago committed by GitHub
parent
commit
65216016ca
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 4
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java
  2. 4
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionControllerTest.java
  3. 2
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java
  4. 190
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/threadutils/ThreadUtilsTest.java
  5. 40
      dolphinscheduler-dist/release-docs/LICENSE
  6. 4
      dolphinscheduler-dist/release-docs/NOTICE
  7. 2
      dolphinscheduler-dist/src/main/assembly/dolphinscheduler-binary.xml
  8. 2
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java
  9. 29
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
  10. 13
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/CuratorZookeeperClient.java
  11. 3
      dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/CuratorZookeeperClientTest.java
  12. 4
      dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.js
  13. 6
      pom.xml
  14. 90
      tools/dependencies/known-dependencies.txt

4
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java

@ -223,7 +223,7 @@ public class ProcessDefinitionServiceImpl extends BaseService implements
// return processDefinition object with ID
result.put(Constants.DATA_LIST, processDefineMapper.selectById(processDefine.getId()));
putMsg(result, Status.SUCCESS);
result.put("processDefinitionId", processDefine.getId());
result.put(PROCESSDEFINITIONID, processDefine.getId());
return result;
}
@ -438,7 +438,7 @@ public class ProcessDefinitionServiceImpl extends BaseService implements
if (processDefinition == null) {
putMsg(result, Status.SUCCESS);
} else {
putMsg(result, Status.PROCESS_INSTANCE_EXIST, name);
putMsg(result, Status.VERIFY_PROCESS_DEFINITION_NAME_UNIQUE_ERROR, name);
}
return result;
}

4
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionControllerTest.java

@ -117,14 +117,14 @@ public class ProcessDefinitionControllerTest {
public void testVerifyProcessDefinitionName() throws Exception {
Map<String, Object> result = new HashMap<>();
putMsg(result, Status.PROCESS_INSTANCE_EXIST);
putMsg(result, Status.VERIFY_PROCESS_DEFINITION_NAME_UNIQUE_ERROR);
String projectName = "test";
String name = "dag_test";
Mockito.when(processDefinitionService.verifyProcessDefinitionName(user, projectName, name)).thenReturn(result);
Result response = processDefinitionController.verifyProcessDefinitionName(user, projectName, name);
Assert.assertEquals(Status.PROCESS_INSTANCE_EXIST.getCode(), response.getCode().intValue());
Assert.assertEquals(Status.VERIFY_PROCESS_DEFINITION_NAME_UNIQUE_ERROR.getCode(), response.getCode().intValue());
}

2
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java

@ -627,7 +627,7 @@ public class ProcessDefinitionServiceTest {
Mockito.when(processDefineMapper.verifyByDefineName(project.getId(), "test_pdf")).thenReturn(getProcessDefinition());
Map<String, Object> processExistRes = processDefinitionService.verifyProcessDefinitionName(loginUser,
"project_test1", "test_pdf");
Assert.assertEquals(Status.PROCESS_INSTANCE_EXIST, processExistRes.get(Constants.STATUS));
Assert.assertEquals(Status.VERIFY_PROCESS_DEFINITION_NAME_UNIQUE_ERROR, processExistRes.get(Constants.STATUS));
}
@Test

190
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/threadutils/ThreadUtilsTest.java

@ -1,190 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.threadutils;
import org.apache.dolphinscheduler.common.thread.Stopper;
import org.apache.dolphinscheduler.common.thread.ThreadPoolExecutors;
import org.apache.dolphinscheduler.common.thread.ThreadUtils;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Calendar;
import java.util.concurrent.*;
import static org.junit.Assert.*;
public class ThreadUtilsTest {
private static final Logger logger = LoggerFactory.getLogger(ThreadUtilsTest.class);
/**
* create a naming thread
*/
@Test
public void testNewDaemonFixedThreadExecutor() {
// create core size and max size are all 3
ExecutorService testExec = ThreadUtils.newDaemonFixedThreadExecutor("test-exec-thread",10);
for (int i = 0; i < 19; i++) {
final int index = i;
testExec.submit(() -> {
System.out.println("do some work index " + index);
});
}
assertFalse(testExec.isShutdown());
testExec.shutdownNow();
assertTrue(testExec.isShutdown());
}
/**
* test schedulerThreadExecutor as for print time in scheduler
* default check thread is 1
*/
@Test
public void testNewDaemonScheduleThreadExecutor() {
ScheduledExecutorService scheduleService = ThreadUtils.newDaemonThreadScheduledExecutor("scheduler-thread", 1);
Calendar start = Calendar.getInstance();
Calendar globalTimer = Calendar.getInstance();
globalTimer.set(2019, Calendar.DECEMBER, 1, 0, 0, 0);
// current
Calendar end = Calendar.getInstance();
end.set(2019, Calendar.DECEMBER, 1, 0, 0, 3);
Runnable schedulerTask = new Runnable() {
@Override
public void run() {
start.set(2019, Calendar.DECEMBER, 1, 0, 0, 0);
int index = 0;
// send heart beat work
while (start.getTime().getTime() <= end.getTime().getTime()) {
System.out.println("worker here");
System.out.println(index ++);
start.add(Calendar.SECOND, 1);
globalTimer.add(Calendar.SECOND, 1);
}
System.out.println("time is " + System.currentTimeMillis());
}
};
scheduleService.scheduleAtFixedRate(schedulerTask, 2, 10, TimeUnit.SECONDS);
assertFalse(scheduleService.isShutdown());
try {
Thread.sleep(60000);
} catch (InterruptedException e) {
e.printStackTrace();
}
scheduleService.shutdownNow();
assertTrue(scheduleService.isShutdown());
}
/**
* test stopper is working normal
*/
@Test
public void testStopper() {
assertTrue(Stopper.isRunning());
Stopper.stop();
assertTrue(Stopper.isStopped());
}
/**
* test threadPoolExecutors with 3 workers and current each 5 tasks
* @throws InterruptedException
*/
@Test
public void testThreadInfo() throws InterruptedException {
ThreadPoolExecutors workers = ThreadPoolExecutors.getInstance("worker", 3);
for (int i = 0; i < 5; ++i ) {
int index = i;
workers.execute(() -> {
for (int j = 0; j < 10; ++j) {
try {
Thread.sleep(1000);
System.out.printf("worker %d is doing the task", index);
System.out.println();
workers.printStatus();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
});
workers.submit(() -> {
for (int j = 0; j < 10; ++j) {
try {
Thread.sleep(1000);
System.out.printf("worker_2 %d is doing the task", index);
System.out.println();
workers.printStatus();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
});
}
Thread.sleep(50001);
workers.shutdown();
}
/**
* test a single daemon thread pool
*/
@Test
public void testNewDaemonSingleThreadExecutor() {
ExecutorService threadTest = ThreadUtils.newDaemonSingleThreadExecutor("thread_test");
threadTest.execute(() -> {
for (int i = 0; i < 100; ++i) {
System.out.println("daemon working ");
}
});
assertFalse(threadTest.isShutdown());
threadTest.shutdownNow();
assertTrue(threadTest.isShutdown());
}
@Test
public void testNewDaemonCachedThreadPool() {
ThreadPoolExecutor threadPoolExecutor = ThreadUtils.newDaemonCachedThreadPool("threadTest-");
Thread thread1 = threadPoolExecutor.getThreadFactory().newThread(() -> {
for (int i = 0; i < 10; ++i) {
System.out.println("this task is with index " + i );
}
});
assertTrue(thread1.getName().startsWith("threadTest-"));
assertFalse(threadPoolExecutor.isShutdown());
threadPoolExecutor.shutdown();
assertTrue(threadPoolExecutor.isShutdown());
}
@Test
public void testNewDaemonCachedThreadPoolWithThreadNumber() {
ThreadPoolExecutor threadPoolExecutor = ThreadUtils.newDaemonCachedThreadPool("threadTest--", 3, 10);
for (int i = 0; i < 10; ++ i) {
threadPoolExecutor.getThreadFactory().newThread(() -> {
assertEquals(3, threadPoolExecutor.getActiveCount());
System.out.println("this task is first work to do");
});
}
assertFalse(threadPoolExecutor.isShutdown());
threadPoolExecutor.shutdown();
assertTrue(threadPoolExecutor.isShutdown());
}
}

40
dolphinscheduler-dist/release-docs/LICENSE vendored

@ -230,7 +230,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
clickhouse-jdbc 0.1.52: https://mvnrepository.com/artifact/ru.yandex.clickhouse/clickhouse-jdbc/0.1.52, Apache 2.0
commons-beanutils 1.7.0 https://mvnrepository.com/artifact/commons-beanutils/commons-beanutils/1.7.0, Apache 2.0
commons-cli 1.2: https://mvnrepository.com/artifact/commons-cli/commons-cli/1.2, Apache 2.0
commons-codec 1.6: https://mvnrepository.com/artifact/commons-codec/commons-codec/1.6, Apache 2.0
commons-codec 1.11: https://mvnrepository.com/artifact/commons-codec/commons-codec/1.11, Apache 2.0
commons-collections 3.2.2: https://mvnrepository.com/artifact/commons-collections/commons-collections/3.2.2, Apache 2.0
commons-collections4 4.1: https://mvnrepository.com/artifact/org.apache.commons/commons-collections4/4.1, Apache 2.0
commons-compress 1.4.1: https://mvnrepository.com/artifact/org.apache.commons/commons-compress/1.4.1, Apache 2.0
@ -348,20 +348,20 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
snakeyaml 1.23: https://mvnrepository.com/artifact/org.yaml/snakeyaml/1.23, Apache 2.0
snappy 0.2: https://mvnrepository.com/artifact/org.iq80.snappy/snappy/0.2, Apache 2.0
snappy-java 1.0.4.1: https://github.com/xerial/snappy-java, Apache 2.0
spring-aop 5.1.5.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-aop/5.1.5.RELEASE, Apache 2.0
spring-beans 5.1.5.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-beans/5.1.5.RELEASE, Apache 2.0
spring-boot 2.1.3.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot/2.1.3.RELEASE, Apache 2.0
spring-boot-autoconfigure 2.1.3.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-autoconfigure/2.1.3.RELEASE, Apache 2.0
spring-boot-starter 2.1.3.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter/2.1.3.RELEASE, Apache 2.0
spring-boot-starter-aop 2.1.3.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-aop/2.1.3.RELEASE, Apache 2.0
spring-boot-starter-jdbc 2.1.3.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-jdbc/2.1.3.RELEASE, Apache 2.0
spring-boot-starter-jetty 2.1.3.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-jetty/2.1.3.RELEASE, Apache 2.0
spring-boot-starter-json 2.1.3.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-json/2.1.3.RELEASE, Apache 2.0
spring-boot-starter-logging 2.1.3.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-logging/2.1.3.RELEASE, Apache 2.0
spring-boot-starter-web 2.1.3.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-web/2.1.3.RELEASE, Apache 2.0
spring-context 5.1.5.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-context/5.1.5.RELEASE, Apache 2.0
spring-core 5.1.5.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-core, Apache 2.0
spring-expression 5.1.5.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-expression, Apache 2.0
spring-aop 5.1.18.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-aop/5.1.18.RELEASE, Apache 2.0
spring-beans 5.1.18.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-beans/5.1.18.RELEASE, Apache 2.0
spring-boot 2.1.17.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot/2.1.17.RELEASE, Apache 2.0
spring-boot-autoconfigure 2.1.17.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-autoconfigure/2.1.17.RELEASE, Apache 2.0
spring-boot-starter 2.1.17.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter/2.1.17.RELEASE, Apache 2.0
spring-boot-starter-aop 2.1.17.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-aop/2.1.17.RELEASE, Apache 2.0
spring-boot-starter-jdbc 2.1.17.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-jdbc/2.1.17.RELEASE, Apache 2.0
spring-boot-starter-jetty 2.1.17.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-jetty/2.1.17.RELEASE, Apache 2.0
spring-boot-starter-json 2.1.17.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-json/2.1.17.RELEASE, Apache 2.0
spring-boot-starter-logging 2.1.17.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-logging/2.1.17.RELEASE, Apache 2.0
spring-boot-starter-web 2.1.17.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-web/2.1.17.RELEASE, Apache 2.0
spring-context 5.1.18.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-context/5.1.18.RELEASE, Apache 2.0
spring-core 5.1.18.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-core/5.1.18.RELEASE, Apache 2.0
spring-expression 5.1.18.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-expression/5.1.18.RELEASE, Apache 2.0
springfox-core 2.9.2: https://mvnrepository.com/artifact/io.springfox/springfox-core, Apache 2.0
springfox-schema 2.9.2: https://mvnrepository.com/artifact/io.springfox/springfox-schema, Apache 2.0
springfox-spi 2.9.2: https://mvnrepository.com/artifact/io.springfox/springfox-spi, Apache 2.0
@ -369,13 +369,13 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
springfox-swagger2 2.9.2: https://mvnrepository.com/artifact/io.springfox/springfox-swagger2/2.9.2, Apache 2.0
springfox-swagger-common 2.9.2: https://mvnrepository.com/artifact/io.springfox/springfox-swagger-common/2.9.2, Apache 2.0
springfox-swagger-ui 2.9.2: https://mvnrepository.com/artifact/io.springfox/springfox-swagger-ui/2.9.2, Apache 2.0
spring-jcl 5.1.5.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-jcl/5.1.5.RELEASE, Apache 2.0
spring-jdbc 5.1.5.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-jdbc/5.1.5.RELEASE, Apache 2.0
spring-jcl 5.1.18.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-jcl/5.1.18.RELEASE, Apache 2.0
spring-jdbc 5.1.18.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-jdbc/5.1.18.RELEASE, Apache 2.0
spring-plugin-core 1.2.0.RELEASE: https://mvnrepository.com/artifact/org.springframework.plugin/spring-plugin-core/1.2.0.RELEASE, Apache 2.0
spring-plugin-metadata 1.2.0.RELEASE: https://mvnrepository.com/artifact/org.springframework.plugin/spring-plugin-metadata/1.2.0.RELEASE, Apache 2.0
spring-tx 5.1.5.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-tx/5.1.5.RELEASE, Apache 2.0
spring-web 5.1.5.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-web/5.1.5.RELEASE, Apache 2.0
spring-webmvc 5.1.5.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-webmvc/5.1.5.RELEASE, Apache 2.0
spring-tx 5.1.18.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-tx/5.1.18.RELEASE, Apache 2.0
spring-web 5.1.18.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-web/5.1.18.RELEASE, Apache 2.0
spring-webmvc 5.1.18.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-webmvc/5.1.18.RELEASE, Apache 2.0
swagger-annotations 1.5.20: https://mvnrepository.com/artifact/io.swagger/swagger-annotations/1.5.20, Apache 2.0
swagger-bootstrap-ui 1.9.3: https://mvnrepository.com/artifact/com.github.xiaoymin/swagger-bootstrap-ui/1.9.3, Apache 2.0
swagger-models 1.5.20: https://mvnrepository.com/artifact/io.swagger/swagger-models/1.5.20, Apache 2.0

4
dolphinscheduler-dist/release-docs/NOTICE vendored

@ -384,8 +384,8 @@ This product contains the Maven wrapper scripts from 'Maven Wrapper', that provi
Spring Framework NOTICE
========================================================================
Spring Framework 5.1.5.RELEASE
Copyright (c) 2002-2019 Pivotal, Inc.
Spring Framework 5.1.18.RELEASE
Copyright (c) 2002-2020 Pivotal, Inc.
This product is licensed to you under the Apache License, Version 2.0
(the "License"). You may not use this product except in compliance with

2
dolphinscheduler-dist/src/main/assembly/dolphinscheduler-binary.xml vendored

@ -113,7 +113,7 @@
<!--server end-->
<!--service end-->
<!--service start-->
<fileSet>
<directory>${basedir}/../dolphinscheduler-service/src/main/resources</directory>
<includes>

2
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java

@ -326,7 +326,7 @@ public class TaskPriorityQueueConsumer extends Thread{
udfFunIdsArray[i]=Integer.parseInt(udfFunIds[i]);
}
List<UdfFunc> udfFuncList = processService.queryUdfFunListByids(udfFunIdsArray);
List<UdfFunc> udfFuncList = processService.queryUdfFunListByIds(udfFunIdsArray);
Map<UdfFunc,String> udfFuncMap = new HashMap<>();
for(UdfFunc udfFunc : udfFuncList) {
String tenantCode = processService.queryTenantCodeByResName(udfFunc.getResourceName(), ResourceType.UDF);

29
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java

@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.service.process;
import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_END_DATE;
@ -144,8 +145,6 @@ public class ProcessService {
@Autowired
private ResourceMapper resourceMapper;
@Autowired
private ErrorCommandMapper errorCommandMapper;
@ -180,7 +179,7 @@ public class ProcessService {
processInstance.addHistoryCmd(command.getCommandType());
saveProcessInstance(processInstance);
this.setSubProcessParam(processInstance);
delCommandByid(command.getId());
delCommandById(command.getId());
return processInstance;
}
@ -193,7 +192,7 @@ public class ProcessService {
public void moveToErrorCommand(Command command, String message) {
ErrorCommand errorCommand = new ErrorCommand(command, message);
this.errorCommandMapper.insert(errorCommand);
delCommandByid(command.getId());
delCommandById(command.getId());
}
/**
@ -355,7 +354,6 @@ public class ProcessService {
return 1;
}
/**
* remove task log file
* @param processInstanceId processInstanceId
@ -384,13 +382,11 @@ public class ProcessService {
ip = taskInstance.getHost();
}
// remove task log from loggerserver
logClient.removeTaskLog(ip,port,taskLogPath);
}
}
/**
* calculate sub process number in the process define.
* @param processDefinitionId processDefinitionId
@ -792,7 +788,6 @@ public class ProcessService {
}
/**
* set sub work process parameters.
* handle sub work process instance, update relation table and command parameters
@ -1132,7 +1127,6 @@ public class ProcessService {
return taskInstance;
}
/**
* ${processInstancePriority}_${processInstanceId}_${taskInstancePriority}_${taskInstanceId}_${task executed by ip1},${ip2}...
* The tasks with the highest priority are selected by comparing the priorities of the above four levels from high to low.
@ -1306,11 +1300,12 @@ public class ProcessService {
int count = taskInstanceMapper.updateById(taskInstance);
return count > 0;
}
/**
* delete a command by id
* @param id id
*/
public void delCommandByid(int id) {
public void delCommandById(int id) {
commandMapper.deleteById(id);
}
@ -1323,7 +1318,6 @@ public class ProcessService {
return taskInstanceMapper.selectById(taskId);
}
/**
* package task instanceassociate processInstance and processDefine
* @param taskInstId taskInstId
@ -1345,7 +1339,6 @@ public class ProcessService {
return taskInstance;
}
/**
* get id list by task state
* @param instanceId instanceId
@ -1383,7 +1376,6 @@ public class ProcessService {
return processInstanceMapMapper.updateById(processInstanceMap);
}
/**
* create work process instance map
* @param processInstanceMap processInstanceMap
@ -1448,7 +1440,6 @@ public class ProcessService {
return processInstance;
}
/**
* change task state
* @param state state
@ -1568,7 +1559,6 @@ public class ProcessService {
* @return process instance list
*/
public List<ProcessInstance> queryNeedFailoverProcessInstances(String host) {
return processInstanceMapper.queryByHostAndStatus(host, stateArray);
}
@ -1610,7 +1600,6 @@ public class ProcessService {
return dataSourceMapper.selectById(id);
}
/**
* update process instance state by id
* @param processInstanceId processInstanceId
@ -1642,7 +1631,7 @@ public class ProcessService {
* @param ids ids
* @return udf function list
*/
public List<UdfFunc> queryUdfFunListByids(int[] ids){
public List<UdfFunc> queryUdfFunListByIds(int[] ids) {
return udfFuncMapper.queryUdfByIdStr(ids, null);
}
@ -1699,7 +1688,6 @@ public class ProcessService {
return cycleDependencyList;
}
String strCrontab = "";
CronExpression depCronExpression;
Cron depCron;
@ -1809,8 +1797,6 @@ public class ProcessService {
return queue;
}
/**
* get task worker group
* @param taskInstance taskInstance
@ -1898,6 +1884,8 @@ public class ProcessService {
Set<Integer> authorizedUdfs = udfFuncMapper.listAuthorizedUdfFunc(userId, needChecks).stream().map(t -> t.getId()).collect(toSet());
originResSet.removeAll(authorizedUdfs);
break;
default:
break;
}
resultList.addAll(originResSet);
@ -1924,7 +1912,6 @@ public class ProcessService {
return resourceMapper.selectById(resoruceId);
}
/**
* list resources by ids
* @param resIds resIds

13
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/CuratorZookeeperClient.java

@ -24,6 +24,7 @@ import org.apache.curator.framework.state.ConnectionState;
import org.apache.curator.retry.ExponentialBackoffRetry;
import org.apache.zookeeper.ZooDefs;
import org.apache.zookeeper.data.ACL;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
@ -32,6 +33,7 @@ import org.springframework.stereotype.Component;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.concurrent.TimeUnit;
import static org.apache.dolphinscheduler.common.utils.Preconditions.checkNotNull;
@ -55,9 +57,10 @@ public class CuratorZookeeperClient implements InitializingBean {
}
private CuratorFramework buildClient() {
logger.info("zookeeper registry center init, server lists is: {}.", zookeeperConfig.getServerList());
logger.info("zookeeper registry center init, server lists is: [{}]", zookeeperConfig.getServerList());
CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder().ensembleProvider(new DefaultEnsembleProvider(checkNotNull(zookeeperConfig.getServerList(),"zookeeper quorum can't be null")))
CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder()
.ensembleProvider(new DefaultEnsembleProvider(checkNotNull(zookeeperConfig.getServerList(), "zookeeper quorum can't be null")))
.retryPolicy(new ExponentialBackoffRetry(zookeeperConfig.getBaseSleepTimeMs(), zookeeperConfig.getMaxRetries(), zookeeperConfig.getMaxSleepMs()));
//these has default value
@ -84,7 +87,9 @@ public class CuratorZookeeperClient implements InitializingBean {
zkClient = builder.build();
zkClient.start();
try {
zkClient.blockUntilConnected();
logger.info("trying to connect zookeeper server list:{}", zookeeperConfig.getServerList());
zkClient.blockUntilConnected(30, TimeUnit.SECONDS);
} catch (final Exception ex) {
throw new RuntimeException(ex);
}
@ -101,6 +106,8 @@ public class CuratorZookeeperClient implements InitializingBean {
logger.info("reconnected to zookeeper");
} else if (newState == ConnectionState.SUSPENDED) {
logger.warn("connection SUSPENDED to zookeeper");
} else if (newState == ConnectionState.CONNECTED) {
logger.info("connected to zookeeper server list:[{}]", zookeeperConfig.getServerList());
}
});
}

3
dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/CuratorZookeeperClientTest.java

@ -59,9 +59,8 @@ public class CuratorZookeeperClientTest {
zookeeperConfig.setDsRoot("/dolphinscheduler");
zookeeperConfig.setMaxWaitTime(30000);
zookeeperClient.setZookeeperConfig(zookeeperConfig);
System.out.println("start");
zookeeperClient.afterPropertiesSet();
System.out.println("end");
Assert.assertNotNull(zookeeperClient.getZkClient());
}
}

4
dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.js

@ -135,10 +135,12 @@ Dag.prototype.backfill = function (arg) {
const dataObject = {}
g.nodes().forEach(function (v) {
const node = g.node(v)
const location = store.state.dag.locations[node.label]
const obj = {}
obj.name = node.label
obj.name = location.name
obj.x = node.x + marginX
obj.y = node.y
obj.targetarr = location.targetarr
dataObject[node.label] = obj
})
jsPlumb.ready(() => {

6
pom.xml

@ -59,8 +59,8 @@
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<curator.version>4.3.0</curator.version>
<spring.version>5.1.5.RELEASE</spring.version>
<spring.boot.version>2.1.3.RELEASE</spring.boot.version>
<spring.version>5.1.18.RELEASE</spring.version>
<spring.boot.version>2.1.17.RELEASE</spring.boot.version>
<java.version>1.8</java.version>
<logback.version>1.2.3</logback.version>
<hadoop.version>2.7.3</hadoop.version>
@ -71,7 +71,7 @@
<cron.utils.version>5.0.5</cron.utils.version>
<druid.version>1.1.22</druid.version>
<h2.version>1.4.200</h2.version>
<commons.codec.version>1.6</commons.codec.version>
<commons.codec.version>1.11</commons.codec.version>
<commons.logging.version>1.1.1</commons.logging.version>
<httpclient.version>4.4.1</httpclient.version>
<httpcore.version>4.4.1</httpcore.version>

90
tools/dependencies/known-dependencies.txt

@ -2,26 +2,26 @@ HikariCP-3.2.0.jar
activation-1.1.jar
ant-1.6.5.jar
aopalliance-1.0.jar
apache-el-8.5.35.1.jar
apache-el-8.5.54.jar
apacheds-i18n-2.0.0-M15.jar
apacheds-kerberos-codec-2.0.0-M15.jar
api-asn1-api-1.0.0-M20.jar
api-util-1.0.0-M20.jar
asm-3.1.jar
aspectjweaver-1.9.2.jar
aspectjweaver-1.9.6.jar
audience-annotations-0.5.0.jar
avro-1.7.4.jar
aws-java-sdk-1.7.4.jar
bonecp-0.8.0.RELEASE.jar
byte-buddy-1.9.10.jar
byte-buddy-1.9.16.jar
classmate-1.4.0.jar
clickhouse-jdbc-0.1.52.jar
commons-cli-1.2.jar
commons-codec-1.6.jar
commons-codec-1.11.jar
commons-collections-3.2.2.jar
commons-collections4-4.1.jar
commons-compress-1.4.1.jar
commons-compiler-3.0.12.jar
commons-compiler-3.0.16.jar
commons-configuration-1.10.jar
commons-daemon-1.0.13.jar
commons-beanutils-1.7.0.jar
@ -44,7 +44,7 @@ datanucleus-core-4.1.6.jar
datanucleus-rdbms-4.1.7.jar
derby-10.14.2.0.jar
druid-1.1.22.jar
gson-2.8.5.jar
gson-2.8.6.jar
guava-20.0.jar
guice-3.0.jar
guice-servlet-3.0.jar
@ -65,7 +65,7 @@ hadoop-yarn-client-2.7.3.jar
hadoop-yarn-common-2.7.3.jar
hadoop-yarn-server-common-2.7.3.jar
hamcrest-core-1.3.jar
hibernate-validator-6.0.14.Final.jar
hibernate-validator-6.0.20.Final.jar
hive-common-2.1.0.jar
hive-jdbc-2.1.0.jar
hive-metastore-2.1.0.jar
@ -77,19 +77,19 @@ hive-storage-api-2.1.0.jar
htrace-core-3.1.0-incubating.jar
httpclient-4.4.1.jar
httpcore-4.4.1.jar
httpmime-4.5.7.jar
httpmime-4.5.12.jar
jackson-annotations-2.9.8.jar
jackson-core-2.9.8.jar
jackson-core-asl-1.9.13.jar
jackson-databind-2.9.8.jar
jackson-datatype-jdk8-2.9.8.jar
jackson-datatype-jsr310-2.9.8.jar
jackson-datatype-jdk8-2.9.10.jar
jackson-datatype-jsr310-2.9.10.jar
jackson-jaxrs-1.9.13.jar
jackson-mapper-asl-1.9.13.jar
jackson-module-parameter-names-2.9.8.jar
jackson-module-parameter-names-2.9.10.jar
jackson-xc-1.9.13.jar
jamon-runtime-2.3.1.jar
janino-3.0.12.jar
janino-3.0.16.jar
java-xmlbuilder-0.4.jar
javax.activation-api-1.2.0.jar
javax.annotation-api-1.3.2.jar
@ -100,7 +100,7 @@ javax.servlet-api-3.1.0.jar
javolution-5.5.1.jar
jaxb-api-2.3.1.jar
jaxb-impl-2.2.3-1.jar
jboss-logging-3.3.2.Final.jar
jboss-logging-3.3.3.Final.jar
jdo-api-3.0.1.jar
jersey-client-1.9.jar
jersey-core-1.9.jar
@ -110,21 +110,21 @@ jersey-server-1.9.jar
jets3t-0.9.0.jar
jettison-1.1.jar
jetty-6.1.26.jar
jetty-continuation-9.4.14.v20181114.jar
jetty-http-9.4.14.v20181114.jar
jetty-io-9.4.14.v20181114.jar
jetty-security-9.4.14.v20181114.jar
jetty-server-9.4.14.v20181114.jar
jetty-servlet-9.4.14.v20181114.jar
jetty-servlets-9.4.14.v20181114.jar
jetty-continuation-9.4.31.v20200723.jar
jetty-http-9.4.31.v20200723.jar
jetty-io-9.4.31.v20200723.jar
jetty-security-9.4.31.v20200723.jar
jetty-server-9.4.31.v20200723.jar
jetty-servlet-9.4.31.v20200723.jar
jetty-servlets-9.4.31.v20200723.jar
jetty-util-6.1.26.jar
jetty-util-9.4.14.v20181114.jar
jetty-webapp-9.4.14.v20181114.jar
jetty-xml-9.4.14.v20181114.jar
jetty-util-9.4.31.v20200723.jar
jetty-webapp-9.4.31.v20200723.jar
jetty-xml-9.4.31.v20200723.jar
jline-0.9.94.jar
jna-4.5.2.jar
jna-platform-4.5.2.jar
joda-time-2.10.1.jar
joda-time-2.10.6.jar
jpam-1.1.jar
jsch-0.1.42.jar
jsp-2.1-6.1.14.jar
@ -133,7 +133,7 @@ jsp-api-2.1.jar
jsqlparser-2.1.jar
jsr305-3.0.0.jar
jta-1.1.jar
jul-to-slf4j-1.7.25.jar
jul-to-slf4j-1.7.30.jar
junit-4.12.jar
leveldbjni-all-1.8.jar
libfb303-0.9.3.jar
@ -155,7 +155,7 @@ mybatis-plus-core-3.2.0.jar
mybatis-plus-extension-3.2.0.jar
mybatis-spring-2.0.2.jar
netty-3.6.2.Final.jar
netty-all-4.1.33.Final.jar
netty-all-4.1.52.Final.jar
opencsv-2.3.jar
oshi-core-3.5.0.jar
paranamer-2.3.jar
@ -169,27 +169,27 @@ slf4j-api-1.7.5.jar
snakeyaml-1.23.jar
snappy-0.2.jar
snappy-java-1.0.4.1.jar
spring-aop-5.1.5.RELEASE.jar
spring-beans-5.1.5.RELEASE.jar
spring-boot-2.1.3.RELEASE.jar
spring-boot-autoconfigure-2.1.3.RELEASE.jar
spring-boot-starter-2.1.3.RELEASE.jar
spring-boot-starter-aop-2.1.3.RELEASE.jar
spring-boot-starter-jdbc-2.1.3.RELEASE.jar
spring-boot-starter-jetty-2.1.3.RELEASE.jar
spring-boot-starter-json-2.1.3.RELEASE.jar
spring-boot-starter-logging-2.1.3.RELEASE.jar
spring-boot-starter-web-2.1.3.RELEASE.jar
spring-context-5.1.5.RELEASE.jar
spring-core-5.1.5.RELEASE.jar
spring-expression-5.1.5.RELEASE.jar
spring-jcl-5.1.5.RELEASE.jar
spring-jdbc-5.1.5.RELEASE.jar
spring-aop-5.1.18.RELEASE.jar
spring-beans-5.1.18.RELEASE.jar
spring-boot-2.1.17.RELEASE.jar
spring-boot-autoconfigure-2.1.17.RELEASE.jar
spring-boot-starter-2.1.17.RELEASE.jar
spring-boot-starter-aop-2.1.17.RELEASE.jar
spring-boot-starter-jdbc-2.1.17.RELEASE.jar
spring-boot-starter-jetty-2.1.17.RELEASE.jar
spring-boot-starter-json-2.1.17.RELEASE.jar
spring-boot-starter-logging-2.1.17.RELEASE.jar
spring-boot-starter-web-2.1.17.RELEASE.jar
spring-context-5.1.18.RELEASE.jar
spring-core-5.1.18.RELEASE.jar
spring-expression-5.1.18.RELEASE.jar
spring-jcl-5.1.18.RELEASE.jar
spring-jdbc-5.1.18.RELEASE.jar
spring-plugin-core-1.2.0.RELEASE.jar
spring-plugin-metadata-1.2.0.RELEASE.jar
spring-tx-5.1.5.RELEASE.jar
spring-web-5.1.5.RELEASE.jar
spring-webmvc-5.1.5.RELEASE.jar
spring-tx-5.1.18.RELEASE.jar
spring-web-5.1.18.RELEASE.jar
spring-webmvc-5.1.18.RELEASE.jar
springfox-core-2.9.2.jar
springfox-schema-2.9.2.jar
springfox-spi-2.9.2.jar

Loading…
Cancel
Save