From 4b027fbf020c55c2f8244ac2c7ef8fe4807e46c5 Mon Sep 17 00:00:00 2001 From: Jave-Chen Date: Tue, 7 Jan 2020 21:53:55 +0800 Subject: [PATCH 01/32] =?UTF-8?q?Fix=20SonarCloud=20bug=20=E2=80=9CResourc?= =?UTF-8?q?es=20should=20be=20closed=E2=80=9D=20(#1667)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #1666 Fix SonarCloud bug “Resources should be closed” --- .../api/utils/ZooKeeperState.java | 58 +++++++++---------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZooKeeperState.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZooKeeperState.java index 523f8103aa..968a51ad63 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZooKeeperState.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZooKeeperState.java @@ -55,41 +55,41 @@ public class ZooKeeperState { public void getZookeeperInfo() { String content = cmd("srvr"); if (StringUtils.isNotBlank(content)) { - Scanner scannerForStat = new Scanner(content); - while (scannerForStat.hasNext()) { - String line = scannerForStat.nextLine(); - if (line.startsWith("Latency min/avg/max:")) { - String[] latencys = getStringValueFromLine(line).split("/"); - minLatency = Integer.parseInt(latencys[0]); - avgLatency = Integer.parseInt(latencys[1]); - maxLatency = Integer.parseInt(latencys[2]); - } else if (line.startsWith("Received:")) { - received = Long.parseLong(getStringValueFromLine(line)); - } else if (line.startsWith("Sent:")) { - sent = Long.parseLong(getStringValueFromLine(line)); - } else if (line.startsWith("Outstanding:")) { - outStanding = Integer.parseInt(getStringValueFromLine(line)); - } else if (line.startsWith("Zxid:")) { - zxid = Long.parseLong(getStringValueFromLine(line).substring(2), 16); - } else if (line.startsWith("Mode:")) { - mode = getStringValueFromLine(line); - } else if (line.startsWith("Node count:")) { - nodeCount = Integer.parseInt(getStringValueFromLine(line)); + try (Scanner scannerForStat = new Scanner(content)) { + while (scannerForStat.hasNext()) { + String line = scannerForStat.nextLine(); + if (line.startsWith("Latency min/avg/max:")) { + String[] latencys = getStringValueFromLine(line).split("/"); + minLatency = Integer.parseInt(latencys[0]); + avgLatency = Integer.parseInt(latencys[1]); + maxLatency = Integer.parseInt(latencys[2]); + } else if (line.startsWith("Received:")) { + received = Long.parseLong(getStringValueFromLine(line)); + } else if (line.startsWith("Sent:")) { + sent = Long.parseLong(getStringValueFromLine(line)); + } else if (line.startsWith("Outstanding:")) { + outStanding = Integer.parseInt(getStringValueFromLine(line)); + } else if (line.startsWith("Zxid:")) { + zxid = Long.parseLong(getStringValueFromLine(line).substring(2), 16); + } else if (line.startsWith("Mode:")) { + mode = getStringValueFromLine(line); + } else if (line.startsWith("Node count:")) { + nodeCount = Integer.parseInt(getStringValueFromLine(line)); + } } - } - scannerForStat.close(); + } } String wchsText = cmd("wchs"); if (StringUtils.isNotBlank(wchsText)) { - Scanner scannerForWchs = new Scanner(wchsText); - while (scannerForWchs.hasNext()) { - String line = scannerForWchs.nextLine(); - if (line.startsWith("Total watches:")) { - watches = Integer.parseInt(getStringValueFromLine(line)); + try (Scanner scannerForWchs = new Scanner(wchsText)) { + while (scannerForWchs.hasNext()) { + String line = scannerForWchs.nextLine(); + if (line.startsWith("Total watches:")) { + watches = Integer.parseInt(getStringValueFromLine(line)); + } } - } - scannerForWchs.close(); + } } String consText = cmd("cons"); From 2087f464fb0eb7e362ca2abffb9700eb6ee8fd53 Mon Sep 17 00:00:00 2001 From: zhukai Date: Tue, 7 Jan 2020 21:59:50 +0800 Subject: [PATCH 02/32] Implements the sql type task, now it can be cache the previous input. (#1745) --- .../pages/dag/_source/formModel/formModel.vue | 1 + .../formModel/tasks/_source/datasource.vue | 17 +++++++-- .../pages/dag/_source/formModel/tasks/sql.vue | 35 +++++++++++++++++-- .../src/js/conf/home/store/dag/actions.js | 5 +++ 4 files changed, 54 insertions(+), 4 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue index 7e3853f104..9d6a10f545 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue @@ -143,6 +143,7 @@ diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/datasource.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/datasource.vue index 8bf855b720..aa067d80e4 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/datasource.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/datasource.vue @@ -112,7 +112,20 @@ }) } }, - watch: {}, + computed: { + cacheParams () { + return { + type: this.type, + datasource: this.datasource + } + } + }, + // Watch the cacheParams + watch: { + cacheParams (val) { + this.$emit('on-dsData', val); + } + }, created () { let supportType = this.supportType || [] this.typeList = _.cloneDeep(this.store.state.dag.dsTypeListS) @@ -146,4 +159,4 @@ }, components: { } } - \ No newline at end of file + diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sql.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sql.vue index 7ab032a798..eb5c40992e 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sql.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sql.vue @@ -371,6 +371,10 @@ this.connParams = '' } }, + //Watch the cacheParams + cacheParams (val) { + this.$emit('on-cache-params', val); + } }, created () { let o = this.backfillItem @@ -396,7 +400,8 @@ this.receivers = o.params.receivers && o.params.receivers.split(',') || [] this.receiversCc = o.params.receiversCc && o.params.receiversCc.split(',') || [] } - if (!_.some(this.store.state.dag.tasks, { id: this.createNodeId }) && + // read tasks from cache + if (!_.some(this.store.state.dag.cacheTasks, { id: this.createNodeId }) && this.router.history.current.name !== 'definition-create') { this._getReceiver() } @@ -415,7 +420,33 @@ editor.off($('.code-sql-mirror'), 'keypress', this.keypress) } }, - computed: {}, + computed: { + cacheParams () { + return { + type: this.type, + datasource: this.rtDatasource, + sql: editor ? editor.getValue() : '', + udfs: this.udfs, + sqlType: this.sqlType, + title: this.title, + receivers: this.receivers.join(','), + receiversCc: this.receiversCc.join(','), + showType: (() => { + + let showType = this.showType + if (showType.length === 2 && showType[0] === 'ATTACHMENT') { + return [showType[1], showType[0]].join(',') + } else { + return showType.join(',') + } + })(), + localParams: this.localParams, + connParams: this.connParams, + preStatements: this.preStatements, + postStatements: this.postStatements + } + } + }, components: { mListBox, mDatasource, mLocalParams, mUdfs, mSqlType, mStatementList, mEmail } } diff --git a/dolphinscheduler-ui/src/js/conf/home/store/dag/actions.js b/dolphinscheduler-ui/src/js/conf/home/store/dag/actions.js index b854dc8ade..52237a63ef 100644 --- a/dolphinscheduler-ui/src/js/conf/home/store/dag/actions.js +++ b/dolphinscheduler-ui/src/js/conf/home/store/dag/actions.js @@ -110,6 +110,11 @@ export default { let processDefinitionJson = JSON.parse(res.data.processDefinitionJson) // tasks info state.tasks = processDefinitionJson.tasks + // tasks cache + state.cacheTasks = {} + processDefinitionJson.tasks.forEach(v => { + state.cacheTasks[v.id] = v + }) // global params state.globalParams = processDefinitionJson.globalParams // timeout From 074fd41aa64ecd0b688b357d12395da6d2095ac1 Mon Sep 17 00:00:00 2001 From: Tboy Date: Tue, 7 Jan 2020 22:33:31 +0800 Subject: [PATCH 03/32] remove freemarker dependency (#1757) * remove freemarker dependency --- .../api/service/ResourcesService.java | 10 +++++----- .../api/service/UdfFuncService.java | 2 +- pom.xml | 18 +++++------------- 3 files changed, 11 insertions(+), 19 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java index 3093daed5a..09b1d31151 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java @@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.api.service; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import org.apache.commons.collections.BeanMap; -import org.apache.commons.lang.StringUtils; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; @@ -28,6 +27,7 @@ import org.apache.dolphinscheduler.common.enums.ResourceType; import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.common.utils.HadoopUtils; import org.apache.dolphinscheduler.common.utils.PropertyUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.dao.entity.Resource; import org.apache.dolphinscheduler.dao.entity.Tenant; import org.apache.dolphinscheduler.dao.entity.UdfFunc; @@ -104,7 +104,7 @@ public class ResourcesService extends BaseService { String nameSuffix = FileUtils.suffix(name); // determine file suffix - if (!StringUtils.equals(fileSuffix, nameSuffix)) { + if (!(StringUtils.isNotEmpty(fileSuffix) && fileSuffix.equalsIgnoreCase(nameSuffix))) { /** * rename file suffix and original suffix must be consistent */ @@ -341,7 +341,7 @@ public class ResourcesService extends BaseService { String nameSuffix = FileUtils.suffix(name); // determine file suffix - if (!StringUtils.equals(fileSuffix, nameSuffix)) { + if (!(StringUtils.isNotEmpty(fileSuffix) && fileSuffix.equalsIgnoreCase(nameSuffix))) { return false; } // query tenant @@ -539,7 +539,7 @@ public class ResourcesService extends BaseService { putMsg(result, Status.SUCCESS); Map map = new HashMap<>(); map.put(ALIAS, resource.getAlias()); - map.put(CONTENT, StringUtils.join(content.toArray(), "\n")); + map.put(CONTENT, StringUtils.join(content, "\n")); result.setData(map); }else{ logger.error("read file {} not exist in hdfs", hdfsFileName); @@ -602,7 +602,7 @@ public class ResourcesService extends BaseService { putMsg(result, Status.SUCCESS); Map dataMap = new BeanMap(resource); - Map resultMap = new HashMap<>(5); + Map resultMap = new HashMap<>(); for (Map.Entry entry: dataMap.entrySet()) { if (!Constants.CLASS.equalsIgnoreCase(entry.getKey().toString())) { resultMap.put(entry.getKey().toString(), entry.getValue()); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java index 20324928fb..249c7ec8df 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java @@ -22,6 +22,7 @@ import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.UdfType; import org.apache.dolphinscheduler.common.utils.PropertyUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.dao.entity.Resource; import org.apache.dolphinscheduler.dao.entity.UdfFunc; import org.apache.dolphinscheduler.dao.entity.User; @@ -30,7 +31,6 @@ import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper; import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; diff --git a/pom.xml b/pom.xml index 6451dcebac..e0d02b460b 100644 --- a/pom.xml +++ b/pom.xml @@ -86,7 +86,6 @@ 1.10 1.5 3.17 - 2.3.21 3.1.0 4.1 20.0 @@ -358,11 +357,11 @@ ${slf4j.log4j12.version} - - commons-collections - commons-collections - ${commons.collections.version} - + + commons-collections + commons-collections + ${commons.collections.version} + commons-httpclient @@ -406,13 +405,6 @@ ${poi.version} - - org.freemarker - freemarker - ${freemarker.version} - - - org.apache.hadoop From 23cec993f984824b1889fca2eea53d8bc107fb43 Mon Sep 17 00:00:00 2001 From: Yelli <51317527+Yeleights@users.noreply.github.com> Date: Wed, 8 Jan 2020 09:59:49 +0800 Subject: [PATCH 04/32] fix bug: fix ResourcesServiceTest (#1759) --- .../dolphinscheduler/api/service/ResourcesServiceTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java index c0caaaaa71..6d07ebd99c 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java @@ -41,6 +41,7 @@ import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; import org.slf4j.Logger; @@ -53,6 +54,7 @@ import java.util.List; import java.util.Map; @RunWith(PowerMockRunner.class) +@PowerMockIgnore({"sun.security.*", "javax.net.*"}) @PrepareForTest({HadoopUtils.class,PropertyUtils.class, FileUtils.class,org.apache.dolphinscheduler.api.utils.FileUtils.class}) public class ResourcesServiceTest { private static final Logger logger = LoggerFactory.getLogger(ResourcesServiceTest.class); @@ -115,6 +117,7 @@ public class ResourcesServiceTest { //UDF_RESOURCE_SUFFIX_NOT_JAR mockMultipartFile = new MockMultipartFile("ResourcesServiceTest.pdf","ResourcesServiceTest.pdf","pdf",new String("test").getBytes()); + PowerMockito.when(FileUtils.suffix("ResourcesServiceTest.pdf")).thenReturn("pdf"); result = resourcesService.createResource(user,"ResourcesServiceTest.pdf","ResourcesServiceTest",ResourceType.UDF,mockMultipartFile); logger.info(result.toString()); Assert.assertEquals(Status.UDF_RESOURCE_SUFFIX_NOT_JAR.getMsg(),result.getMsg()); From 963ed4a8c694482ba4e89b5cc6692cde74627986 Mon Sep 17 00:00:00 2001 From: Yelli <51317527+Yeleights@users.noreply.github.com> Date: Wed, 8 Jan 2020 10:00:34 +0800 Subject: [PATCH 05/32] ds.server & ds.common.zk regularize log print (#1754) --- .../common/zk/AbstractZKClient.java | 22 +++++-------------- .../server/worker/WorkerServer.java | 4 ++-- .../worker/task/AbstractCommandExecutor.java | 6 ++--- .../server/zk/ZKMasterClient.java | 6 ++--- .../server/zk/ZKWorkerClient.java | 2 +- .../server/worker/EnvFileTest.java | 2 +- 6 files changed, 16 insertions(+), 26 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java index b03b88ae65..1e4502e56f 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java @@ -16,18 +16,6 @@ */ package org.apache.dolphinscheduler.common.zk; -import static org.apache.dolphinscheduler.common.Constants.ADD_ZK_OP; -import static org.apache.dolphinscheduler.common.Constants.DELETE_ZK_OP; -import static org.apache.dolphinscheduler.common.Constants.MASTER_PREFIX; -import static org.apache.dolphinscheduler.common.Constants.SINGLE_SLASH; -import static org.apache.dolphinscheduler.common.Constants.UNDERLINE; -import static org.apache.dolphinscheduler.common.Constants.WORKER_PREFIX; - -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.imps.CuratorFrameworkState; import org.apache.curator.framework.recipes.locks.InterProcessMutex; @@ -42,6 +30,8 @@ import org.apache.dolphinscheduler.common.utils.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.*; +import static org.apache.dolphinscheduler.common.Constants.*; /** * abstract zookeeper client @@ -84,7 +74,7 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{ zkClient.setData().forPath(znode,str.getBytes()); } catch (Exception e) { - logger.error("heartbeat for zk failed : " + e.getMessage(), e); + logger.error("heartbeat for zk failed", e); stoppable.stop("heartbeat for zk exception, release resources and stop myself"); } } @@ -274,7 +264,7 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{ masterMap.putIfAbsent(server, super.get(path + "/" + server)); } } catch (Exception e) { - logger.error("get server list failed : " + e.getMessage(), e); + logger.error("get server list failed", e); } return masterMap; @@ -406,7 +396,7 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{ if(e.getMessage().equals("instance must be started before calling this method")){ logger.warn("lock release"); }else{ - logger.error("lock release failed : " + e.getMessage(),e); + logger.error("lock release failed",e); } } @@ -423,7 +413,7 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{ persist(getDeadZNodeParentPath(), ""); } catch (Exception e) { - logger.error("init system znode failed : " + e.getMessage(),e); + logger.error("init system znode failed",e); } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java index ea8330e18a..a0a8c5f5ae 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java @@ -211,7 +211,7 @@ public class WorkerServer implements IStoppable { //thread sleep 3 seconds for thread quitely stop Thread.sleep(3000L); }catch (Exception e){ - logger.warn("thread sleep exception:" + e.getMessage(), e); + logger.warn("thread sleep exception", e); } try { @@ -252,7 +252,7 @@ public class WorkerServer implements IStoppable { logger.info("zookeeper service stopped"); } catch (Exception e) { - logger.error("worker server stop exception : " + e.getMessage(), e); + logger.error("worker server stop exception ", e); System.exit(-1); } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java index 715fda108e..7a14223b95 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java @@ -292,7 +292,7 @@ public abstract class AbstractCommandExecutor { Runtime.getRuntime().exec(cmd); } catch (IOException e) { - logger.info("kill attempt failed." + e.getMessage(), e); + logger.info("kill attempt failed", e); } } @@ -312,7 +312,7 @@ public abstract class AbstractCommandExecutor { Runtime.getRuntime().exec(cmd); } catch (IOException e) { - logger.error("kill attempt failed." + e.getMessage(), e); + logger.error("kill attempt failed ", e); } } } @@ -407,7 +407,7 @@ public abstract class AbstractCommandExecutor { } } } catch (Exception e) { - logger.error(String.format("yarn applications: %s status failed : " + e.getMessage(), appIds.toString()),e); + logger.error(String.format("yarn applications: %s status failed ", appIds.toString()),e); result = false; } return result; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java index 1c7b737f5b..0f7f49e899 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java @@ -107,7 +107,7 @@ public class ZKMasterClient extends AbstractZKClient { } }catch (Exception e){ - logger.error("master start up exception : " + e.getMessage(),e); + logger.error("master start up exception",e); }finally { releaseMutex(mutex); } @@ -143,7 +143,7 @@ public class ZKMasterClient extends AbstractZKClient { } masterZNode = serverPath; } catch (Exception e) { - logger.error("register master failure : " + e.getMessage(),e); + logger.error("register master failure ",e); System.exit(-1); } } @@ -192,7 +192,7 @@ public class ZKMasterClient extends AbstractZKClient { } }catch (Exception e){ logger.error("{} server failover failed.", zkNodeType.toString()); - logger.error("failover exception : " + e.getMessage(),e); + logger.error("failover exception ",e); } finally { releaseMutex(mutex); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKWorkerClient.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKWorkerClient.java index 2e063d50d5..eabfc54520 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKWorkerClient.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKWorkerClient.java @@ -69,7 +69,7 @@ public class ZKWorkerClient extends AbstractZKClient { } workerZNode = serverPath; } catch (Exception e) { - logger.error("register worker failure : " + e.getMessage(),e); + logger.error("register worker failure",e); System.exit(-1); } } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/EnvFileTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/EnvFileTest.java index d6dc34a67a..4fedae7d4b 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/EnvFileTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/EnvFileTest.java @@ -65,7 +65,7 @@ public class EnvFileTest { } }catch (IOException e){ - logger.error("read file failed : " + e.getMessage(),e); + logger.error("read file failed",e); }finally { try { if (br != null){ From 995b5ca4a16097d6ce82ad1d8255534fdac2264b Mon Sep 17 00:00:00 2001 From: break60 <790061044@qq.com> Date: Wed, 8 Jan 2020 10:40:40 +0800 Subject: [PATCH 06/32] jest unit test demo (#1760) * Fix api url * Fixed DAG zoom in and zoom out nodes separated from arrows * Fix front-end code specifications * Fix front-end code specifications * Fix front-end code specifications * jest unit test demo * jest unit test demo --- dolphinscheduler-ui/_test_/.babelrc | 8 +++ dolphinscheduler-ui/_test_/Counter.spec.js | 55 +++++++++++++++++++ dolphinscheduler-ui/_test_/package.json | 37 +++++++++++++ dolphinscheduler-ui/_test_/test.spec.js | 34 ++++++++++++ .../src/components/Counter.vue | 53 ++++++++++++++++++ .../src/components/Message.vue | 28 ++++++++++ 6 files changed, 215 insertions(+) create mode 100644 dolphinscheduler-ui/_test_/.babelrc create mode 100644 dolphinscheduler-ui/_test_/Counter.spec.js create mode 100644 dolphinscheduler-ui/_test_/package.json create mode 100644 dolphinscheduler-ui/_test_/test.spec.js create mode 100644 dolphinscheduler-ui/src/components/Counter.vue create mode 100644 dolphinscheduler-ui/src/components/Message.vue diff --git a/dolphinscheduler-ui/_test_/.babelrc b/dolphinscheduler-ui/_test_/.babelrc new file mode 100644 index 0000000000..bae693584c --- /dev/null +++ b/dolphinscheduler-ui/_test_/.babelrc @@ -0,0 +1,8 @@ +{ + "presets": [["env", { "modules": false }]], + "env": { + "test": { + "presets": [["env", { "targets": { "node": "current" } }]] + } + } + } \ No newline at end of file diff --git a/dolphinscheduler-ui/_test_/Counter.spec.js b/dolphinscheduler-ui/_test_/Counter.spec.js new file mode 100644 index 0000000000..9530d76300 --- /dev/null +++ b/dolphinscheduler-ui/_test_/Counter.spec.js @@ -0,0 +1,55 @@ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { mount } from '@vue/test-utils' +import Counter from '../src/components/Counter.vue' + +describe("Counter.vue", () => { + it("渲染Counter组件", () => { + const wrapper = mount(Counter) + expect(wrapper.element).toMatchSnapshot() + }) + + it("初始化之为0", () => { + const wrapper = mount(Counter) + expect(wrapper.vm.count).toEqual(0) + }) + + it("加1", () => { + const wrapper = mount(Counter) + wrapper.vm.inc() + expect(wrapper.vm.count).toEqual(1) + }) + + it("减1", () => { + const wrapper = mount(Counter) + wrapper.vm.dec() + expect(wrapper.vm.count).toEqual(-1) + }) + + it("重置", () => { + const wrapper = mount(Counter) + wrapper.vm.reset() + expect(wrapper.vm.count).toEqual(0) + }) + + it("因数为10加1操作", () => { + const wrapper = mount(Counter, { propsData: { factor: 10 } }) + wrapper.vm.inc() + expect(wrapper.vm.computedCount).toEqual(10) + }) +}) \ No newline at end of file diff --git a/dolphinscheduler-ui/_test_/package.json b/dolphinscheduler-ui/_test_/package.json new file mode 100644 index 0000000000..df47d10c0a --- /dev/null +++ b/dolphinscheduler-ui/_test_/package.json @@ -0,0 +1,37 @@ +{ + "name": "testjest", + "description": "jest", + "version": "1.0.0", + "author": "xiangcaibiao", + "private": true, + "scripts": { + "test": "jest --coverage" + }, + "dependencies": { + "vue": "^2.4.4" + }, + "jest": { + "moduleFileExtensions": [ + "js", + "vue" + ], + "moduleNameMapper": { + "^@/(.*)$": "/src/$1" + }, + "transform": { + "^.+\\.js$": "/node_modules/babel-jest", + ".*\\.(vue)$": "/node_modules/vue-jest" + }, + "snapshotSerializers": [ + "/node_modules/jest-serializer-vue" + ] + }, + "devDependencies": { + "@vue/test-utils": "^1.0.0-beta.30", + "babel-jest": "^24.9.0", + "babel-preset-env": "^1.7.0", + "jest": "^24.9.0", + "jest-serializer-vue": "^2.0.2", + "vue-jest": "^3.0.5" + } +} diff --git a/dolphinscheduler-ui/_test_/test.spec.js b/dolphinscheduler-ui/_test_/test.spec.js new file mode 100644 index 0000000000..27b863fe8a --- /dev/null +++ b/dolphinscheduler-ui/_test_/test.spec.js @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { shallowMount } from '@vue/test-utils' +import Message from '../src/components/Message.vue' + +describe('Message', () => { + it('renders props.msg when passed', () => { + const msg = 'new message' + const wrapper = shallowMount(Message, { + propsData: { msg } + }) + expect(wrapper.text()).toBe(msg) + }) + + it('renders default message if not passed a prop', () => { + const defaultMessage = 'default message' + const wrapper = shallowMount(Message) + expect(wrapper.text()).toBe(defaultMessage) + }) +}) diff --git a/dolphinscheduler-ui/src/components/Counter.vue b/dolphinscheduler-ui/src/components/Counter.vue new file mode 100644 index 0000000000..6fafb5a54e --- /dev/null +++ b/dolphinscheduler-ui/src/components/Counter.vue @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + \ No newline at end of file diff --git a/dolphinscheduler-ui/src/components/Message.vue b/dolphinscheduler-ui/src/components/Message.vue new file mode 100644 index 0000000000..95f5236b6e --- /dev/null +++ b/dolphinscheduler-ui/src/components/Message.vue @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + \ No newline at end of file From 03ef3943997f43de8c5bc06810bff4e284f6401e Mon Sep 17 00:00:00 2001 From: elonlo Date: Wed, 8 Jan 2020 11:51:01 +0800 Subject: [PATCH 07/32] fixbug: login interface cannot jump to home interface (#1765) --- dolphinscheduler-ui/src/js/conf/login/App.vue | 8 ++++++-- dolphinscheduler-ui/src/js/module/io/index.js | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/login/App.vue b/dolphinscheduler-ui/src/js/conf/login/App.vue index 886ff2ea54..a79743a262 100644 --- a/dolphinscheduler-ui/src/js/conf/login/App.vue +++ b/dolphinscheduler-ui/src/js/conf/login/App.vue @@ -84,8 +84,12 @@ this._gLogin().then(res => { setTimeout(() => { this.spinnerLoading = false - sessionStorage.setItem("sessionId", res.data) - cookies.set('sessionId', res.data,{ path: '/' }) + if (res.data.hasOwnProperty("sessionId")) { + let sessionId=res.data.sessionId + sessionStorage.setItem("sessionId", sessionId) + cookies.set('sessionId', sessionId,{ path: '/' }) + } + if (this.userName === 'admin') { window.location.href = `${PUBLIC_PATH}/#/security/tenant` } else { diff --git a/dolphinscheduler-ui/src/js/module/io/index.js b/dolphinscheduler-ui/src/js/module/io/index.js index f5713a51a5..eedb40e00b 100644 --- a/dolphinscheduler-ui/src/js/module/io/index.js +++ b/dolphinscheduler-ui/src/js/module/io/index.js @@ -76,7 +76,7 @@ io.interceptors.request.use( let sIdCookie = cookies.get('sessionId') let sessionId = sessionStorage.getItem("sessionId") let requstUrl = config.url.substring(config.url.lastIndexOf("/")+1) - if(requstUrl!=='login' && sIdCookie!=sessionId) { + if(sIdCookie !== null && requstUrl!=='login' && sIdCookie!=sessionId) { window.location.href = `${PUBLIC_PATH}/view/login/index.html` } else { let { method } = config From 74a5506b9a65e5adf34a06d4999f7f8cb0082867 Mon Sep 17 00:00:00 2001 From: break60 <790061044@qq.com> Date: Wed, 8 Jan 2020 19:35:27 +0800 Subject: [PATCH 08/32] Fix delete operation, no query interface request is made on the list page #1768 (#1777) * Fix api url * Fixed DAG zoom in and zoom out nodes separated from arrows * Fix front-end code specifications * Fix front-end code specifications * Fix front-end code specifications * jest unit test demo * jest unit test demo * Fix delete operation, no query interface request is made on the list page * Fix delete operation, no query interface request is made on the list page --- .../home/pages/datasource/pages/list/_source/list.vue | 2 +- .../conf/home/pages/datasource/pages/list/index.vue | 11 +++++++---- .../home/pages/monitor/pages/servers/servers.scss | 2 +- .../projects/pages/_source/taskRecordList/index.vue | 4 ++-- .../projects/pages/definition/pages/list/index.vue | 4 ++-- .../projects/pages/instance/pages/list/index.vue | 4 ++-- .../home/pages/projects/pages/list/_source/list.vue | 5 +++-- .../js/conf/home/pages/projects/pages/list/index.vue | 4 ++-- .../resource/pages/file/pages/list/_source/list.vue | 2 +- .../pages/resource/pages/file/pages/list/index.vue | 9 ++++++--- .../pages/udf/pages/function/_source/list.vue | 2 +- .../pages/resource/pages/udf/pages/function/index.vue | 6 ++---- .../pages/udf/pages/resource/_source/list.vue | 2 +- .../pages/resource/pages/udf/pages/resource/index.vue | 9 ++++++--- .../js/conf/home/pages/security/pages/queue/index.vue | 4 ++-- .../pages/security/pages/tenement/_source/list.vue | 2 +- .../conf/home/pages/security/pages/tenement/index.vue | 8 ++++++-- .../home/pages/security/pages/users/_source/list.vue | 2 +- .../js/conf/home/pages/security/pages/users/index.vue | 8 ++++++-- .../security/pages/warningGroups/_source/list.vue | 2 +- .../home/pages/security/pages/warningGroups/index.vue | 8 ++++++-- .../security/pages/workerGroups/_source/list.vue | 2 +- .../home/pages/security/pages/workerGroups/index.vue | 10 +++++++--- .../conf/home/pages/user/pages/token/_source/list.vue | 2 +- .../src/js/conf/home/pages/user/pages/token/index.vue | 11 ++++++++--- .../src/js/module/components/listBoxF/listBoxF.vue | 2 +- .../src/js/module/components/nav/nav.vue | 2 +- 27 files changed, 79 insertions(+), 50 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/list.vue b/dolphinscheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/list.vue index ebf296d11c..ef6f83912f 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/list.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/list.vue @@ -148,7 +148,7 @@ this.deleteDatasource({ id: item.id }).then(res => { - this.list.splice(i, 1) + this.$emit('on-update') this.$message.success(res.msg) }).catch(e => { this.$message.error(e.msg || '') diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/datasource/pages/list/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/datasource/pages/list/index.vue index 5f746b0c68..19640ccd38 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/datasource/pages/list/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/datasource/pages/list/index.vue @@ -24,13 +24,13 @@