Browse Source

Merge pull request #20 from apache/dev

update from apache
pull/3/MERGE
BoYiZhang 4 years ago committed by GitHub
parent
commit
65216016ca
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 4
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java
  2. 4
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionControllerTest.java
  3. 2
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java
  4. 190
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/threadutils/ThreadUtilsTest.java
  5. 40
      dolphinscheduler-dist/release-docs/LICENSE
  6. 4
      dolphinscheduler-dist/release-docs/NOTICE
  7. 6
      dolphinscheduler-dist/src/main/assembly/dolphinscheduler-binary.xml
  8. 2
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java
  9. 403
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
  10. 19
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/CuratorZookeeperClient.java
  11. 3
      dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/CuratorZookeeperClientTest.java
  12. 4
      dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.js
  13. 6
      pom.xml
  14. 92
      tools/dependencies/known-dependencies.txt

4
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java

@ -223,7 +223,7 @@ public class ProcessDefinitionServiceImpl extends BaseService implements
// return processDefinition object with ID // return processDefinition object with ID
result.put(Constants.DATA_LIST, processDefineMapper.selectById(processDefine.getId())); result.put(Constants.DATA_LIST, processDefineMapper.selectById(processDefine.getId()));
putMsg(result, Status.SUCCESS); putMsg(result, Status.SUCCESS);
result.put("processDefinitionId", processDefine.getId()); result.put(PROCESSDEFINITIONID, processDefine.getId());
return result; return result;
} }
@ -438,7 +438,7 @@ public class ProcessDefinitionServiceImpl extends BaseService implements
if (processDefinition == null) { if (processDefinition == null) {
putMsg(result, Status.SUCCESS); putMsg(result, Status.SUCCESS);
} else { } else {
putMsg(result, Status.PROCESS_INSTANCE_EXIST, name); putMsg(result, Status.VERIFY_PROCESS_DEFINITION_NAME_UNIQUE_ERROR, name);
} }
return result; return result;
} }

4
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionControllerTest.java

@ -117,14 +117,14 @@ public class ProcessDefinitionControllerTest {
public void testVerifyProcessDefinitionName() throws Exception { public void testVerifyProcessDefinitionName() throws Exception {
Map<String, Object> result = new HashMap<>(); Map<String, Object> result = new HashMap<>();
putMsg(result, Status.PROCESS_INSTANCE_EXIST); putMsg(result, Status.VERIFY_PROCESS_DEFINITION_NAME_UNIQUE_ERROR);
String projectName = "test"; String projectName = "test";
String name = "dag_test"; String name = "dag_test";
Mockito.when(processDefinitionService.verifyProcessDefinitionName(user, projectName, name)).thenReturn(result); Mockito.when(processDefinitionService.verifyProcessDefinitionName(user, projectName, name)).thenReturn(result);
Result response = processDefinitionController.verifyProcessDefinitionName(user, projectName, name); Result response = processDefinitionController.verifyProcessDefinitionName(user, projectName, name);
Assert.assertEquals(Status.PROCESS_INSTANCE_EXIST.getCode(), response.getCode().intValue()); Assert.assertEquals(Status.VERIFY_PROCESS_DEFINITION_NAME_UNIQUE_ERROR.getCode(), response.getCode().intValue());
} }

2
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java

@ -627,7 +627,7 @@ public class ProcessDefinitionServiceTest {
Mockito.when(processDefineMapper.verifyByDefineName(project.getId(), "test_pdf")).thenReturn(getProcessDefinition()); Mockito.when(processDefineMapper.verifyByDefineName(project.getId(), "test_pdf")).thenReturn(getProcessDefinition());
Map<String, Object> processExistRes = processDefinitionService.verifyProcessDefinitionName(loginUser, Map<String, Object> processExistRes = processDefinitionService.verifyProcessDefinitionName(loginUser,
"project_test1", "test_pdf"); "project_test1", "test_pdf");
Assert.assertEquals(Status.PROCESS_INSTANCE_EXIST, processExistRes.get(Constants.STATUS)); Assert.assertEquals(Status.VERIFY_PROCESS_DEFINITION_NAME_UNIQUE_ERROR, processExistRes.get(Constants.STATUS));
} }
@Test @Test

190
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/threadutils/ThreadUtilsTest.java

@ -1,190 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.threadutils;
import org.apache.dolphinscheduler.common.thread.Stopper;
import org.apache.dolphinscheduler.common.thread.ThreadPoolExecutors;
import org.apache.dolphinscheduler.common.thread.ThreadUtils;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Calendar;
import java.util.concurrent.*;
import static org.junit.Assert.*;
public class ThreadUtilsTest {
private static final Logger logger = LoggerFactory.getLogger(ThreadUtilsTest.class);
/**
* create a naming thread
*/
@Test
public void testNewDaemonFixedThreadExecutor() {
// create core size and max size are all 3
ExecutorService testExec = ThreadUtils.newDaemonFixedThreadExecutor("test-exec-thread",10);
for (int i = 0; i < 19; i++) {
final int index = i;
testExec.submit(() -> {
System.out.println("do some work index " + index);
});
}
assertFalse(testExec.isShutdown());
testExec.shutdownNow();
assertTrue(testExec.isShutdown());
}
/**
* test schedulerThreadExecutor as for print time in scheduler
* default check thread is 1
*/
@Test
public void testNewDaemonScheduleThreadExecutor() {
ScheduledExecutorService scheduleService = ThreadUtils.newDaemonThreadScheduledExecutor("scheduler-thread", 1);
Calendar start = Calendar.getInstance();
Calendar globalTimer = Calendar.getInstance();
globalTimer.set(2019, Calendar.DECEMBER, 1, 0, 0, 0);
// current
Calendar end = Calendar.getInstance();
end.set(2019, Calendar.DECEMBER, 1, 0, 0, 3);
Runnable schedulerTask = new Runnable() {
@Override
public void run() {
start.set(2019, Calendar.DECEMBER, 1, 0, 0, 0);
int index = 0;
// send heart beat work
while (start.getTime().getTime() <= end.getTime().getTime()) {
System.out.println("worker here");
System.out.println(index ++);
start.add(Calendar.SECOND, 1);
globalTimer.add(Calendar.SECOND, 1);
}
System.out.println("time is " + System.currentTimeMillis());
}
};
scheduleService.scheduleAtFixedRate(schedulerTask, 2, 10, TimeUnit.SECONDS);
assertFalse(scheduleService.isShutdown());
try {
Thread.sleep(60000);
} catch (InterruptedException e) {
e.printStackTrace();
}
scheduleService.shutdownNow();
assertTrue(scheduleService.isShutdown());
}
/**
* test stopper is working normal
*/
@Test
public void testStopper() {
assertTrue(Stopper.isRunning());
Stopper.stop();
assertTrue(Stopper.isStopped());
}
/**
* test threadPoolExecutors with 3 workers and current each 5 tasks
* @throws InterruptedException
*/
@Test
public void testThreadInfo() throws InterruptedException {
ThreadPoolExecutors workers = ThreadPoolExecutors.getInstance("worker", 3);
for (int i = 0; i < 5; ++i ) {
int index = i;
workers.execute(() -> {
for (int j = 0; j < 10; ++j) {
try {
Thread.sleep(1000);
System.out.printf("worker %d is doing the task", index);
System.out.println();
workers.printStatus();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
});
workers.submit(() -> {
for (int j = 0; j < 10; ++j) {
try {
Thread.sleep(1000);
System.out.printf("worker_2 %d is doing the task", index);
System.out.println();
workers.printStatus();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
});
}
Thread.sleep(50001);
workers.shutdown();
}
/**
* test a single daemon thread pool
*/
@Test
public void testNewDaemonSingleThreadExecutor() {
ExecutorService threadTest = ThreadUtils.newDaemonSingleThreadExecutor("thread_test");
threadTest.execute(() -> {
for (int i = 0; i < 100; ++i) {
System.out.println("daemon working ");
}
});
assertFalse(threadTest.isShutdown());
threadTest.shutdownNow();
assertTrue(threadTest.isShutdown());
}
@Test
public void testNewDaemonCachedThreadPool() {
ThreadPoolExecutor threadPoolExecutor = ThreadUtils.newDaemonCachedThreadPool("threadTest-");
Thread thread1 = threadPoolExecutor.getThreadFactory().newThread(() -> {
for (int i = 0; i < 10; ++i) {
System.out.println("this task is with index " + i );
}
});
assertTrue(thread1.getName().startsWith("threadTest-"));
assertFalse(threadPoolExecutor.isShutdown());
threadPoolExecutor.shutdown();
assertTrue(threadPoolExecutor.isShutdown());
}
@Test
public void testNewDaemonCachedThreadPoolWithThreadNumber() {
ThreadPoolExecutor threadPoolExecutor = ThreadUtils.newDaemonCachedThreadPool("threadTest--", 3, 10);
for (int i = 0; i < 10; ++ i) {
threadPoolExecutor.getThreadFactory().newThread(() -> {
assertEquals(3, threadPoolExecutor.getActiveCount());
System.out.println("this task is first work to do");
});
}
assertFalse(threadPoolExecutor.isShutdown());
threadPoolExecutor.shutdown();
assertTrue(threadPoolExecutor.isShutdown());
}
}

40
dolphinscheduler-dist/release-docs/LICENSE vendored

@ -230,7 +230,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
clickhouse-jdbc 0.1.52: https://mvnrepository.com/artifact/ru.yandex.clickhouse/clickhouse-jdbc/0.1.52, Apache 2.0 clickhouse-jdbc 0.1.52: https://mvnrepository.com/artifact/ru.yandex.clickhouse/clickhouse-jdbc/0.1.52, Apache 2.0
commons-beanutils 1.7.0 https://mvnrepository.com/artifact/commons-beanutils/commons-beanutils/1.7.0, Apache 2.0 commons-beanutils 1.7.0 https://mvnrepository.com/artifact/commons-beanutils/commons-beanutils/1.7.0, Apache 2.0
commons-cli 1.2: https://mvnrepository.com/artifact/commons-cli/commons-cli/1.2, Apache 2.0 commons-cli 1.2: https://mvnrepository.com/artifact/commons-cli/commons-cli/1.2, Apache 2.0
commons-codec 1.6: https://mvnrepository.com/artifact/commons-codec/commons-codec/1.6, Apache 2.0 commons-codec 1.11: https://mvnrepository.com/artifact/commons-codec/commons-codec/1.11, Apache 2.0
commons-collections 3.2.2: https://mvnrepository.com/artifact/commons-collections/commons-collections/3.2.2, Apache 2.0 commons-collections 3.2.2: https://mvnrepository.com/artifact/commons-collections/commons-collections/3.2.2, Apache 2.0
commons-collections4 4.1: https://mvnrepository.com/artifact/org.apache.commons/commons-collections4/4.1, Apache 2.0 commons-collections4 4.1: https://mvnrepository.com/artifact/org.apache.commons/commons-collections4/4.1, Apache 2.0
commons-compress 1.4.1: https://mvnrepository.com/artifact/org.apache.commons/commons-compress/1.4.1, Apache 2.0 commons-compress 1.4.1: https://mvnrepository.com/artifact/org.apache.commons/commons-compress/1.4.1, Apache 2.0
@ -348,20 +348,20 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
snakeyaml 1.23: https://mvnrepository.com/artifact/org.yaml/snakeyaml/1.23, Apache 2.0 snakeyaml 1.23: https://mvnrepository.com/artifact/org.yaml/snakeyaml/1.23, Apache 2.0
snappy 0.2: https://mvnrepository.com/artifact/org.iq80.snappy/snappy/0.2, Apache 2.0 snappy 0.2: https://mvnrepository.com/artifact/org.iq80.snappy/snappy/0.2, Apache 2.0
snappy-java 1.0.4.1: https://github.com/xerial/snappy-java, Apache 2.0 snappy-java 1.0.4.1: https://github.com/xerial/snappy-java, Apache 2.0
spring-aop 5.1.5.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-aop/5.1.5.RELEASE, Apache 2.0 spring-aop 5.1.18.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-aop/5.1.18.RELEASE, Apache 2.0
spring-beans 5.1.5.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-beans/5.1.5.RELEASE, Apache 2.0 spring-beans 5.1.18.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-beans/5.1.18.RELEASE, Apache 2.0
spring-boot 2.1.3.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot/2.1.3.RELEASE, Apache 2.0 spring-boot 2.1.17.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot/2.1.17.RELEASE, Apache 2.0
spring-boot-autoconfigure 2.1.3.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-autoconfigure/2.1.3.RELEASE, Apache 2.0 spring-boot-autoconfigure 2.1.17.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-autoconfigure/2.1.17.RELEASE, Apache 2.0
spring-boot-starter 2.1.3.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter/2.1.3.RELEASE, Apache 2.0 spring-boot-starter 2.1.17.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter/2.1.17.RELEASE, Apache 2.0
spring-boot-starter-aop 2.1.3.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-aop/2.1.3.RELEASE, Apache 2.0 spring-boot-starter-aop 2.1.17.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-aop/2.1.17.RELEASE, Apache 2.0
spring-boot-starter-jdbc 2.1.3.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-jdbc/2.1.3.RELEASE, Apache 2.0 spring-boot-starter-jdbc 2.1.17.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-jdbc/2.1.17.RELEASE, Apache 2.0
spring-boot-starter-jetty 2.1.3.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-jetty/2.1.3.RELEASE, Apache 2.0 spring-boot-starter-jetty 2.1.17.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-jetty/2.1.17.RELEASE, Apache 2.0
spring-boot-starter-json 2.1.3.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-json/2.1.3.RELEASE, Apache 2.0 spring-boot-starter-json 2.1.17.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-json/2.1.17.RELEASE, Apache 2.0
spring-boot-starter-logging 2.1.3.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-logging/2.1.3.RELEASE, Apache 2.0 spring-boot-starter-logging 2.1.17.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-logging/2.1.17.RELEASE, Apache 2.0
spring-boot-starter-web 2.1.3.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-web/2.1.3.RELEASE, Apache 2.0 spring-boot-starter-web 2.1.17.RELEASE: https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-web/2.1.17.RELEASE, Apache 2.0
spring-context 5.1.5.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-context/5.1.5.RELEASE, Apache 2.0 spring-context 5.1.18.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-context/5.1.18.RELEASE, Apache 2.0
spring-core 5.1.5.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-core, Apache 2.0 spring-core 5.1.18.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-core/5.1.18.RELEASE, Apache 2.0
spring-expression 5.1.5.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-expression, Apache 2.0 spring-expression 5.1.18.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-expression/5.1.18.RELEASE, Apache 2.0
springfox-core 2.9.2: https://mvnrepository.com/artifact/io.springfox/springfox-core, Apache 2.0 springfox-core 2.9.2: https://mvnrepository.com/artifact/io.springfox/springfox-core, Apache 2.0
springfox-schema 2.9.2: https://mvnrepository.com/artifact/io.springfox/springfox-schema, Apache 2.0 springfox-schema 2.9.2: https://mvnrepository.com/artifact/io.springfox/springfox-schema, Apache 2.0
springfox-spi 2.9.2: https://mvnrepository.com/artifact/io.springfox/springfox-spi, Apache 2.0 springfox-spi 2.9.2: https://mvnrepository.com/artifact/io.springfox/springfox-spi, Apache 2.0
@ -369,13 +369,13 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
springfox-swagger2 2.9.2: https://mvnrepository.com/artifact/io.springfox/springfox-swagger2/2.9.2, Apache 2.0 springfox-swagger2 2.9.2: https://mvnrepository.com/artifact/io.springfox/springfox-swagger2/2.9.2, Apache 2.0
springfox-swagger-common 2.9.2: https://mvnrepository.com/artifact/io.springfox/springfox-swagger-common/2.9.2, Apache 2.0 springfox-swagger-common 2.9.2: https://mvnrepository.com/artifact/io.springfox/springfox-swagger-common/2.9.2, Apache 2.0
springfox-swagger-ui 2.9.2: https://mvnrepository.com/artifact/io.springfox/springfox-swagger-ui/2.9.2, Apache 2.0 springfox-swagger-ui 2.9.2: https://mvnrepository.com/artifact/io.springfox/springfox-swagger-ui/2.9.2, Apache 2.0
spring-jcl 5.1.5.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-jcl/5.1.5.RELEASE, Apache 2.0 spring-jcl 5.1.18.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-jcl/5.1.18.RELEASE, Apache 2.0
spring-jdbc 5.1.5.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-jdbc/5.1.5.RELEASE, Apache 2.0 spring-jdbc 5.1.18.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-jdbc/5.1.18.RELEASE, Apache 2.0
spring-plugin-core 1.2.0.RELEASE: https://mvnrepository.com/artifact/org.springframework.plugin/spring-plugin-core/1.2.0.RELEASE, Apache 2.0 spring-plugin-core 1.2.0.RELEASE: https://mvnrepository.com/artifact/org.springframework.plugin/spring-plugin-core/1.2.0.RELEASE, Apache 2.0
spring-plugin-metadata 1.2.0.RELEASE: https://mvnrepository.com/artifact/org.springframework.plugin/spring-plugin-metadata/1.2.0.RELEASE, Apache 2.0 spring-plugin-metadata 1.2.0.RELEASE: https://mvnrepository.com/artifact/org.springframework.plugin/spring-plugin-metadata/1.2.0.RELEASE, Apache 2.0
spring-tx 5.1.5.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-tx/5.1.5.RELEASE, Apache 2.0 spring-tx 5.1.18.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-tx/5.1.18.RELEASE, Apache 2.0
spring-web 5.1.5.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-web/5.1.5.RELEASE, Apache 2.0 spring-web 5.1.18.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-web/5.1.18.RELEASE, Apache 2.0
spring-webmvc 5.1.5.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-webmvc/5.1.5.RELEASE, Apache 2.0 spring-webmvc 5.1.18.RELEASE: https://mvnrepository.com/artifact/org.springframework/spring-webmvc/5.1.18.RELEASE, Apache 2.0
swagger-annotations 1.5.20: https://mvnrepository.com/artifact/io.swagger/swagger-annotations/1.5.20, Apache 2.0 swagger-annotations 1.5.20: https://mvnrepository.com/artifact/io.swagger/swagger-annotations/1.5.20, Apache 2.0
swagger-bootstrap-ui 1.9.3: https://mvnrepository.com/artifact/com.github.xiaoymin/swagger-bootstrap-ui/1.9.3, Apache 2.0 swagger-bootstrap-ui 1.9.3: https://mvnrepository.com/artifact/com.github.xiaoymin/swagger-bootstrap-ui/1.9.3, Apache 2.0
swagger-models 1.5.20: https://mvnrepository.com/artifact/io.swagger/swagger-models/1.5.20, Apache 2.0 swagger-models 1.5.20: https://mvnrepository.com/artifact/io.swagger/swagger-models/1.5.20, Apache 2.0

4
dolphinscheduler-dist/release-docs/NOTICE vendored

@ -384,8 +384,8 @@ This product contains the Maven wrapper scripts from 'Maven Wrapper', that provi
Spring Framework NOTICE Spring Framework NOTICE
======================================================================== ========================================================================
Spring Framework 5.1.5.RELEASE Spring Framework 5.1.18.RELEASE
Copyright (c) 2002-2019 Pivotal, Inc. Copyright (c) 2002-2020 Pivotal, Inc.
This product is licensed to you under the Apache License, Version 2.0 This product is licensed to you under the Apache License, Version 2.0
(the "License"). You may not use this product except in compliance with (the "License"). You may not use this product except in compliance with

6
dolphinscheduler-dist/src/main/assembly/dolphinscheduler-binary.xml vendored

@ -112,8 +112,8 @@
</fileSet> </fileSet>
<!--server end--> <!--server end-->
<!--service end--> <!--service start-->
<fileSet> <fileSet>
<directory>${basedir}/../dolphinscheduler-service/src/main/resources</directory> <directory>${basedir}/../dolphinscheduler-service/src/main/resources</directory>
<includes> <includes>
@ -226,4 +226,4 @@
</excludes> </excludes>
</dependencySet> </dependencySet>
</dependencySets> </dependencySets>
</assembly> </assembly>

2
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java

@ -326,7 +326,7 @@ public class TaskPriorityQueueConsumer extends Thread{
udfFunIdsArray[i]=Integer.parseInt(udfFunIds[i]); udfFunIdsArray[i]=Integer.parseInt(udfFunIds[i]);
} }
List<UdfFunc> udfFuncList = processService.queryUdfFunListByids(udfFunIdsArray); List<UdfFunc> udfFuncList = processService.queryUdfFunListByIds(udfFunIdsArray);
Map<UdfFunc,String> udfFuncMap = new HashMap<>(); Map<UdfFunc,String> udfFuncMap = new HashMap<>();
for(UdfFunc udfFunc : udfFuncList) { for(UdfFunc udfFunc : udfFuncList) {
String tenantCode = processService.queryTenantCodeByResName(udfFunc.getResourceName(), ResourceType.UDF); String tenantCode = processService.queryTenantCodeByResName(udfFunc.getResourceName(), ResourceType.UDF);

403
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java

File diff suppressed because it is too large Load Diff

19
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/CuratorZookeeperClient.java

@ -24,6 +24,7 @@ import org.apache.curator.framework.state.ConnectionState;
import org.apache.curator.retry.ExponentialBackoffRetry; import org.apache.curator.retry.ExponentialBackoffRetry;
import org.apache.zookeeper.ZooDefs; import org.apache.zookeeper.ZooDefs;
import org.apache.zookeeper.data.ACL; import org.apache.zookeeper.data.ACL;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.InitializingBean;
@ -32,6 +33,7 @@ import org.springframework.stereotype.Component;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.List; import java.util.List;
import java.util.concurrent.TimeUnit;
import static org.apache.dolphinscheduler.common.utils.Preconditions.checkNotNull; import static org.apache.dolphinscheduler.common.utils.Preconditions.checkNotNull;
@ -55,9 +57,10 @@ public class CuratorZookeeperClient implements InitializingBean {
} }
private CuratorFramework buildClient() { private CuratorFramework buildClient() {
logger.info("zookeeper registry center init, server lists is: {}.", zookeeperConfig.getServerList()); logger.info("zookeeper registry center init, server lists is: [{}]", zookeeperConfig.getServerList());
CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder().ensembleProvider(new DefaultEnsembleProvider(checkNotNull(zookeeperConfig.getServerList(),"zookeeper quorum can't be null"))) CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder()
.ensembleProvider(new DefaultEnsembleProvider(checkNotNull(zookeeperConfig.getServerList(), "zookeeper quorum can't be null")))
.retryPolicy(new ExponentialBackoffRetry(zookeeperConfig.getBaseSleepTimeMs(), zookeeperConfig.getMaxRetries(), zookeeperConfig.getMaxSleepMs())); .retryPolicy(new ExponentialBackoffRetry(zookeeperConfig.getBaseSleepTimeMs(), zookeeperConfig.getMaxRetries(), zookeeperConfig.getMaxSleepMs()));
//these has default value //these has default value
@ -84,7 +87,9 @@ public class CuratorZookeeperClient implements InitializingBean {
zkClient = builder.build(); zkClient = builder.build();
zkClient.start(); zkClient.start();
try { try {
zkClient.blockUntilConnected(); logger.info("trying to connect zookeeper server list:{}", zookeeperConfig.getServerList());
zkClient.blockUntilConnected(30, TimeUnit.SECONDS);
} catch (final Exception ex) { } catch (final Exception ex) {
throw new RuntimeException(ex); throw new RuntimeException(ex);
} }
@ -95,12 +100,14 @@ public class CuratorZookeeperClient implements InitializingBean {
checkNotNull(zkClient); checkNotNull(zkClient);
zkClient.getConnectionStateListenable().addListener((client, newState) -> { zkClient.getConnectionStateListenable().addListener((client, newState) -> {
if(newState == ConnectionState.LOST){ if (newState == ConnectionState.LOST) {
logger.error("connection lost from zookeeper"); logger.error("connection lost from zookeeper");
} else if(newState == ConnectionState.RECONNECTED){ } else if (newState == ConnectionState.RECONNECTED) {
logger.info("reconnected to zookeeper"); logger.info("reconnected to zookeeper");
} else if(newState == ConnectionState.SUSPENDED){ } else if (newState == ConnectionState.SUSPENDED) {
logger.warn("connection SUSPENDED to zookeeper"); logger.warn("connection SUSPENDED to zookeeper");
} else if (newState == ConnectionState.CONNECTED) {
logger.info("connected to zookeeper server list:[{}]", zookeeperConfig.getServerList());
} }
}); });
} }

3
dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/CuratorZookeeperClientTest.java

@ -59,9 +59,8 @@ public class CuratorZookeeperClientTest {
zookeeperConfig.setDsRoot("/dolphinscheduler"); zookeeperConfig.setDsRoot("/dolphinscheduler");
zookeeperConfig.setMaxWaitTime(30000); zookeeperConfig.setMaxWaitTime(30000);
zookeeperClient.setZookeeperConfig(zookeeperConfig); zookeeperClient.setZookeeperConfig(zookeeperConfig);
System.out.println("start");
zookeeperClient.afterPropertiesSet(); zookeeperClient.afterPropertiesSet();
System.out.println("end");
Assert.assertNotNull(zookeeperClient.getZkClient()); Assert.assertNotNull(zookeeperClient.getZkClient());
} }
} }

4
dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.js

@ -135,10 +135,12 @@ Dag.prototype.backfill = function (arg) {
const dataObject = {} const dataObject = {}
g.nodes().forEach(function (v) { g.nodes().forEach(function (v) {
const node = g.node(v) const node = g.node(v)
const location = store.state.dag.locations[node.label]
const obj = {} const obj = {}
obj.name = node.label obj.name = location.name
obj.x = node.x + marginX obj.x = node.x + marginX
obj.y = node.y obj.y = node.y
obj.targetarr = location.targetarr
dataObject[node.label] = obj dataObject[node.label] = obj
}) })
jsPlumb.ready(() => { jsPlumb.ready(() => {

6
pom.xml

@ -59,8 +59,8 @@
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<curator.version>4.3.0</curator.version> <curator.version>4.3.0</curator.version>
<spring.version>5.1.5.RELEASE</spring.version> <spring.version>5.1.18.RELEASE</spring.version>
<spring.boot.version>2.1.3.RELEASE</spring.boot.version> <spring.boot.version>2.1.17.RELEASE</spring.boot.version>
<java.version>1.8</java.version> <java.version>1.8</java.version>
<logback.version>1.2.3</logback.version> <logback.version>1.2.3</logback.version>
<hadoop.version>2.7.3</hadoop.version> <hadoop.version>2.7.3</hadoop.version>
@ -71,7 +71,7 @@
<cron.utils.version>5.0.5</cron.utils.version> <cron.utils.version>5.0.5</cron.utils.version>
<druid.version>1.1.22</druid.version> <druid.version>1.1.22</druid.version>
<h2.version>1.4.200</h2.version> <h2.version>1.4.200</h2.version>
<commons.codec.version>1.6</commons.codec.version> <commons.codec.version>1.11</commons.codec.version>
<commons.logging.version>1.1.1</commons.logging.version> <commons.logging.version>1.1.1</commons.logging.version>
<httpclient.version>4.4.1</httpclient.version> <httpclient.version>4.4.1</httpclient.version>
<httpcore.version>4.4.1</httpcore.version> <httpcore.version>4.4.1</httpcore.version>

92
tools/dependencies/known-dependencies.txt

@ -2,26 +2,26 @@ HikariCP-3.2.0.jar
activation-1.1.jar activation-1.1.jar
ant-1.6.5.jar ant-1.6.5.jar
aopalliance-1.0.jar aopalliance-1.0.jar
apache-el-8.5.35.1.jar apache-el-8.5.54.jar
apacheds-i18n-2.0.0-M15.jar apacheds-i18n-2.0.0-M15.jar
apacheds-kerberos-codec-2.0.0-M15.jar apacheds-kerberos-codec-2.0.0-M15.jar
api-asn1-api-1.0.0-M20.jar api-asn1-api-1.0.0-M20.jar
api-util-1.0.0-M20.jar api-util-1.0.0-M20.jar
asm-3.1.jar asm-3.1.jar
aspectjweaver-1.9.2.jar aspectjweaver-1.9.6.jar
audience-annotations-0.5.0.jar audience-annotations-0.5.0.jar
avro-1.7.4.jar avro-1.7.4.jar
aws-java-sdk-1.7.4.jar aws-java-sdk-1.7.4.jar
bonecp-0.8.0.RELEASE.jar bonecp-0.8.0.RELEASE.jar
byte-buddy-1.9.10.jar byte-buddy-1.9.16.jar
classmate-1.4.0.jar classmate-1.4.0.jar
clickhouse-jdbc-0.1.52.jar clickhouse-jdbc-0.1.52.jar
commons-cli-1.2.jar commons-cli-1.2.jar
commons-codec-1.6.jar commons-codec-1.11.jar
commons-collections-3.2.2.jar commons-collections-3.2.2.jar
commons-collections4-4.1.jar commons-collections4-4.1.jar
commons-compress-1.4.1.jar commons-compress-1.4.1.jar
commons-compiler-3.0.12.jar commons-compiler-3.0.16.jar
commons-configuration-1.10.jar commons-configuration-1.10.jar
commons-daemon-1.0.13.jar commons-daemon-1.0.13.jar
commons-beanutils-1.7.0.jar commons-beanutils-1.7.0.jar
@ -44,7 +44,7 @@ datanucleus-core-4.1.6.jar
datanucleus-rdbms-4.1.7.jar datanucleus-rdbms-4.1.7.jar
derby-10.14.2.0.jar derby-10.14.2.0.jar
druid-1.1.22.jar druid-1.1.22.jar
gson-2.8.5.jar gson-2.8.6.jar
guava-20.0.jar guava-20.0.jar
guice-3.0.jar guice-3.0.jar
guice-servlet-3.0.jar guice-servlet-3.0.jar
@ -65,7 +65,7 @@ hadoop-yarn-client-2.7.3.jar
hadoop-yarn-common-2.7.3.jar hadoop-yarn-common-2.7.3.jar
hadoop-yarn-server-common-2.7.3.jar hadoop-yarn-server-common-2.7.3.jar
hamcrest-core-1.3.jar hamcrest-core-1.3.jar
hibernate-validator-6.0.14.Final.jar hibernate-validator-6.0.20.Final.jar
hive-common-2.1.0.jar hive-common-2.1.0.jar
hive-jdbc-2.1.0.jar hive-jdbc-2.1.0.jar
hive-metastore-2.1.0.jar hive-metastore-2.1.0.jar
@ -77,19 +77,19 @@ hive-storage-api-2.1.0.jar
htrace-core-3.1.0-incubating.jar htrace-core-3.1.0-incubating.jar
httpclient-4.4.1.jar httpclient-4.4.1.jar
httpcore-4.4.1.jar httpcore-4.4.1.jar
httpmime-4.5.7.jar httpmime-4.5.12.jar
jackson-annotations-2.9.8.jar jackson-annotations-2.9.8.jar
jackson-core-2.9.8.jar jackson-core-2.9.8.jar
jackson-core-asl-1.9.13.jar jackson-core-asl-1.9.13.jar
jackson-databind-2.9.8.jar jackson-databind-2.9.8.jar
jackson-datatype-jdk8-2.9.8.jar jackson-datatype-jdk8-2.9.10.jar
jackson-datatype-jsr310-2.9.8.jar jackson-datatype-jsr310-2.9.10.jar
jackson-jaxrs-1.9.13.jar jackson-jaxrs-1.9.13.jar
jackson-mapper-asl-1.9.13.jar jackson-mapper-asl-1.9.13.jar
jackson-module-parameter-names-2.9.8.jar jackson-module-parameter-names-2.9.10.jar
jackson-xc-1.9.13.jar jackson-xc-1.9.13.jar
jamon-runtime-2.3.1.jar jamon-runtime-2.3.1.jar
janino-3.0.12.jar janino-3.0.16.jar
java-xmlbuilder-0.4.jar java-xmlbuilder-0.4.jar
javax.activation-api-1.2.0.jar javax.activation-api-1.2.0.jar
javax.annotation-api-1.3.2.jar javax.annotation-api-1.3.2.jar
@ -100,7 +100,7 @@ javax.servlet-api-3.1.0.jar
javolution-5.5.1.jar javolution-5.5.1.jar
jaxb-api-2.3.1.jar jaxb-api-2.3.1.jar
jaxb-impl-2.2.3-1.jar jaxb-impl-2.2.3-1.jar
jboss-logging-3.3.2.Final.jar jboss-logging-3.3.3.Final.jar
jdo-api-3.0.1.jar jdo-api-3.0.1.jar
jersey-client-1.9.jar jersey-client-1.9.jar
jersey-core-1.9.jar jersey-core-1.9.jar
@ -110,21 +110,21 @@ jersey-server-1.9.jar
jets3t-0.9.0.jar jets3t-0.9.0.jar
jettison-1.1.jar jettison-1.1.jar
jetty-6.1.26.jar jetty-6.1.26.jar
jetty-continuation-9.4.14.v20181114.jar jetty-continuation-9.4.31.v20200723.jar
jetty-http-9.4.14.v20181114.jar jetty-http-9.4.31.v20200723.jar
jetty-io-9.4.14.v20181114.jar jetty-io-9.4.31.v20200723.jar
jetty-security-9.4.14.v20181114.jar jetty-security-9.4.31.v20200723.jar
jetty-server-9.4.14.v20181114.jar jetty-server-9.4.31.v20200723.jar
jetty-servlet-9.4.14.v20181114.jar jetty-servlet-9.4.31.v20200723.jar
jetty-servlets-9.4.14.v20181114.jar jetty-servlets-9.4.31.v20200723.jar
jetty-util-6.1.26.jar jetty-util-6.1.26.jar
jetty-util-9.4.14.v20181114.jar jetty-util-9.4.31.v20200723.jar
jetty-webapp-9.4.14.v20181114.jar jetty-webapp-9.4.31.v20200723.jar
jetty-xml-9.4.14.v20181114.jar jetty-xml-9.4.31.v20200723.jar
jline-0.9.94.jar jline-0.9.94.jar
jna-4.5.2.jar jna-4.5.2.jar
jna-platform-4.5.2.jar jna-platform-4.5.2.jar
joda-time-2.10.1.jar joda-time-2.10.6.jar
jpam-1.1.jar jpam-1.1.jar
jsch-0.1.42.jar jsch-0.1.42.jar
jsp-2.1-6.1.14.jar jsp-2.1-6.1.14.jar
@ -133,7 +133,7 @@ jsp-api-2.1.jar
jsqlparser-2.1.jar jsqlparser-2.1.jar
jsr305-3.0.0.jar jsr305-3.0.0.jar
jta-1.1.jar jta-1.1.jar
jul-to-slf4j-1.7.25.jar jul-to-slf4j-1.7.30.jar
junit-4.12.jar junit-4.12.jar
leveldbjni-all-1.8.jar leveldbjni-all-1.8.jar
libfb303-0.9.3.jar libfb303-0.9.3.jar
@ -155,7 +155,7 @@ mybatis-plus-core-3.2.0.jar
mybatis-plus-extension-3.2.0.jar mybatis-plus-extension-3.2.0.jar
mybatis-spring-2.0.2.jar mybatis-spring-2.0.2.jar
netty-3.6.2.Final.jar netty-3.6.2.Final.jar
netty-all-4.1.33.Final.jar netty-all-4.1.52.Final.jar
opencsv-2.3.jar opencsv-2.3.jar
oshi-core-3.5.0.jar oshi-core-3.5.0.jar
paranamer-2.3.jar paranamer-2.3.jar
@ -169,27 +169,27 @@ slf4j-api-1.7.5.jar
snakeyaml-1.23.jar snakeyaml-1.23.jar
snappy-0.2.jar snappy-0.2.jar
snappy-java-1.0.4.1.jar snappy-java-1.0.4.1.jar
spring-aop-5.1.5.RELEASE.jar spring-aop-5.1.18.RELEASE.jar
spring-beans-5.1.5.RELEASE.jar spring-beans-5.1.18.RELEASE.jar
spring-boot-2.1.3.RELEASE.jar spring-boot-2.1.17.RELEASE.jar
spring-boot-autoconfigure-2.1.3.RELEASE.jar spring-boot-autoconfigure-2.1.17.RELEASE.jar
spring-boot-starter-2.1.3.RELEASE.jar spring-boot-starter-2.1.17.RELEASE.jar
spring-boot-starter-aop-2.1.3.RELEASE.jar spring-boot-starter-aop-2.1.17.RELEASE.jar
spring-boot-starter-jdbc-2.1.3.RELEASE.jar spring-boot-starter-jdbc-2.1.17.RELEASE.jar
spring-boot-starter-jetty-2.1.3.RELEASE.jar spring-boot-starter-jetty-2.1.17.RELEASE.jar
spring-boot-starter-json-2.1.3.RELEASE.jar spring-boot-starter-json-2.1.17.RELEASE.jar
spring-boot-starter-logging-2.1.3.RELEASE.jar spring-boot-starter-logging-2.1.17.RELEASE.jar
spring-boot-starter-web-2.1.3.RELEASE.jar spring-boot-starter-web-2.1.17.RELEASE.jar
spring-context-5.1.5.RELEASE.jar spring-context-5.1.18.RELEASE.jar
spring-core-5.1.5.RELEASE.jar spring-core-5.1.18.RELEASE.jar
spring-expression-5.1.5.RELEASE.jar spring-expression-5.1.18.RELEASE.jar
spring-jcl-5.1.5.RELEASE.jar spring-jcl-5.1.18.RELEASE.jar
spring-jdbc-5.1.5.RELEASE.jar spring-jdbc-5.1.18.RELEASE.jar
spring-plugin-core-1.2.0.RELEASE.jar spring-plugin-core-1.2.0.RELEASE.jar
spring-plugin-metadata-1.2.0.RELEASE.jar spring-plugin-metadata-1.2.0.RELEASE.jar
spring-tx-5.1.5.RELEASE.jar spring-tx-5.1.18.RELEASE.jar
spring-web-5.1.5.RELEASE.jar spring-web-5.1.18.RELEASE.jar
spring-webmvc-5.1.5.RELEASE.jar spring-webmvc-5.1.18.RELEASE.jar
springfox-core-2.9.2.jar springfox-core-2.9.2.jar
springfox-schema-2.9.2.jar springfox-schema-2.9.2.jar
springfox-spi-2.9.2.jar springfox-spi-2.9.2.jar
@ -210,4 +210,4 @@ xmlenc-0.52.jar
xz-1.0.jar xz-1.0.jar
zookeeper-3.4.14.jar zookeeper-3.4.14.jar
guava-retrying-2.0.0.jar guava-retrying-2.0.0.jar
presto-jdbc-0.238.1.jar presto-jdbc-0.238.1.jar

Loading…
Cancel
Save