From f2bfdc14f7abfb8ce5a886f5b796056330eaa569 Mon Sep 17 00:00:00 2001 From: qiaozhanwei Date: Thu, 23 Apr 2020 17:37:04 +0800 Subject: [PATCH 1/2] #2499 bug fix (#2505) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * dispatch task fail will set task status failed * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，task status statistics and process status statistics bug fix (#2357) 2，worker group bug fix * 1，task status statistics and process status statistics bug fix (#2357) 2，worker group bug fix * 1，task status statistics and process status statistics bug fix (#2357) 2，worker group bug fix * 1，task status statistics and process status statistics bug fix (#2357) 2，worker group bug fix * send mail error, #2466 bug fix * send mail error, #2466 bug fix * send mail error, #2466 bug fix * send mail error, #2466 bug fix * #2486 bug fix * host and workergroup compatible * EnterpriseWeChatUtils modify * EnterpriseWeChatUtils modify * EnterpriseWeChatUtils modify * #2499 bug fix Co-authored-by: qiaozhanwei --- .../src/test/resources/alert.properties | 67 ------------------- .../server/master/MasterServer.java | 7 +- .../server/zk/ZKMasterClient.java | 12 +++- 3 files changed, 15 insertions(+), 71 deletions(-) delete mode 100644 dolphinscheduler-alert/src/test/resources/alert.properties diff --git a/dolphinscheduler-alert/src/test/resources/alert.properties b/dolphinscheduler-alert/src/test/resources/alert.properties deleted file mode 100644 index ce233cea37..0000000000 --- a/dolphinscheduler-alert/src/test/resources/alert.properties +++ /dev/null @@ -1,67 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# For unit test - -#alert type is EMAIL/SMS -alert.type=EMAIL - -# mail server configuration -mail.protocol=SMTP -mail.server.host=xxx.xxx.test -mail.server.port=25 -mail.sender=xxx@xxx.com -mail.user=xxx@xxx.com -mail.passwd=111111 - -# Test double -test.server.factor=3.0 - - -# Test NumberFormat -test.server.testnumber=abc - -# Test array -test.server.list=xxx.xxx.test1,xxx.xxx.test2,xxx.xxx.test3 - -# Test enum -test.server.enum1=MASTER -test.server.enum2=DEAD_SERVER -test.server.enum3=abc - -# TLS -mail.smtp.starttls.enable=true -# SSL -mail.smtp.ssl.enable=false -mail.smtp.ssl.trust=xxx.xxx.com - -#xls file path,need create if not exist -xls.file.path=/tmp/xls - -# Enterprise WeChat configuration -enterprise.wechat.enable=false -enterprise.wechat.corp.id=xxxxxxx -enterprise.wechat.secret=xxxxxxx -enterprise.wechat.agent.id=xxxxxxx -enterprise.wechat.users=xxxxxxx -enterprise.wechat.token.url=https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid=$corpId&corpsecret=$secret -enterprise.wechat.push.url=https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=$token -enterprise.wechat.team.send.msg={\"toparty\":\"$toParty\",\"agentid\":\"$agentId\",\"msgtype\":\"text\",\"text\":{\"content\":\"$msg\"},\"safe\":\"0\"} -enterprise.wechat.user.send.msg={\"touser\":\"$toUser\",\"agentid\":\"$agentId\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"$msg\"}} - - - diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java index 6df82b5a60..d86374244f 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java @@ -117,9 +117,12 @@ public class MasterServer { this.nettyRemotingServer.registerProcessor(CommandType.TASK_KILL_RESPONSE, new TaskKillResponseProcessor()); this.nettyRemotingServer.start(); - // - this.zkMasterClient.start(); + // register this.masterRegistry.registry(); + + // self tolerant + this.zkMasterClient.start(); + // masterSchedulerService.start(); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java index 46f48b6d76..e923157360 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java @@ -24,6 +24,7 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.ZKNodeType; import org.apache.dolphinscheduler.common.model.Server; +import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance; @@ -40,6 +41,8 @@ import org.springframework.stereotype.Component; import java.util.Date; import java.util.List; +import static org.apache.dolphinscheduler.common.Constants.*; + /** * zookeeper master client @@ -72,8 +75,13 @@ public class ZKMasterClient extends AbstractZKClient { // init system znode this.initSystemZNode(); - // check if fault tolerance is required?failure and tolerance - if (getActiveMasterNum() == 1 && checkZKNodeExists(OSUtils.getHost(), ZKNodeType.MASTER)) { + while (!checkZKNodeExists(OSUtils.getHost(), ZKNodeType.MASTER)){ + ThreadUtils.sleep(SLEEP_TIME_MILLIS); + } + + + // self tolerant + if (getActiveMasterNum() == 1) { failoverWorker(null, true); failoverMaster(null); } From 1d76d0a9fd3e350cce9730198715f55d9d279f58 Mon Sep 17 00:00:00 2001 From: break60 <790061044@qq.com> Date: Thu, 23 Apr 2020 18:24:23 +0800 Subject: [PATCH 2/2] =?UTF-8?q?Add=20force=20not=20to=20cache=E3=80=81Modi?= =?UTF-8?q?fy=20style=20and=20fix=20jar=20instance=20of=20workflow=20insta?= =?UTF-8?q?nce=20not=20showing=20(#2507)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix the problem of data echo in script edit box * [Refactor worker] simplify and optimize config (#2386) * simplify config * simplify config * simplify and optimize config * Optimize resource tree * Change the name of the shell node and modify the parameter transmission method of spark, mr, python, and flink nodes * Repair naming * Modify list style * Online editing is prohibited * Update HadoopUtils.java optimize HadoopUtils * Update HadoopUtilsTest.java * Change the background color of the disabled state of the radio button when the workflow goes online * Update HadoopUtilsTest.java * Update HadoopUtilsTest.java * Update HttpUtils.java * Update pom.xml * Update HadoopUtilsTest.java * Update HadoopUtilsTest.java * Update HadoopUtilsTest.java * Update HadoopUtilsTest.java * Update HadoopUtilsTest.java * Update HadoopUtilsTest.java * Modify worker parameters * delete img * Add force not to cache and fix jar instance of workflow instance not showing * Add force not to cache and fix jar instance of workflow instance not showing * disabled * fix style Co-authored-by: dailidong Co-authored-by: qiaozhanwei --- .../pages/dag/_source/formModel/tasks/flink.vue | 10 +++++++++- .../pages/dag/_source/formModel/tasks/mr.vue | 10 +++++++++- .../pages/dag/_source/formModel/tasks/python.vue | 10 ++++++++++ .../pages/dag/_source/formModel/tasks/shell.vue | 9 ++++++++- .../pages/dag/_source/formModel/tasks/spark.vue | 10 +++++++++- .../js/conf/home/pages/dag/instanceDetails.vue | 4 +++- .../src/js/module/components/nav/m_logo.png | Bin 3571 -> 0 bytes .../src/js/module/components/nav/nav.vue | 1 - dolphinscheduler-ui/src/view/home/index.html | 4 +++- 9 files changed, 51 insertions(+), 7 deletions(-) delete mode 100644 dolphinscheduler-ui/src/js/module/components/nav/m_logo.png diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue index 9d4189406c..3f85f36992 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue @@ -48,7 +48,7 @@
{{$t('Main jar package')}}
- +
{{ node.raw.fullName }}
@@ -557,4 +557,12 @@ } } } + .vue-treeselect--disabled { + .vue-treeselect__control { + background-color: #ecf3f8; + .vue-treeselect__single-value { + color: #6d859e; + } + } + } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue index a5c23d45c7..8fb2ebadfe 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue @@ -44,7 +44,7 @@
{{$t('Main jar package')}}
- +
{{ node.raw.fullName }}
@@ -427,4 +427,12 @@ } } } + .vue-treeselect--disabled { + .vue-treeselect__control { + background-color: #ecf3f8; + .vue-treeselect__single-value { + color: #6d859e; + } + } + } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue index b9ed72c1ea..851f8bee8a 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue @@ -333,3 +333,13 @@ components: { mLocalParams, mListBox, mResources,Treeselect } } + \ No newline at end of file diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue index d4fa70e63f..7a462a1f27 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue @@ -396,5 +396,12 @@ right: -12px; top: -16px; } - + .vue-treeselect--disabled { + .vue-treeselect__control { + background-color: #ecf3f8; + .vue-treeselect__single-value { + color: #6d859e; + } + } + } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/spark.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/spark.vue index 6e94374101..4cceb3b27b 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/spark.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/spark.vue @@ -63,7 +63,7 @@
{{$t('Main jar package')}}
- +
{{ node.raw.fullName }}
@@ -606,4 +606,12 @@ } } } + .vue-treeselect--disabled { + .vue-treeselect__control { + background-color: #ecf3f8; + .vue-treeselect__single-value { + color: #6d859e; + } + } + } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/instanceDetails.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/instanceDetails.vue index 22acfba408..daa30d7c44 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/instanceDetails.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/instanceDetails.vue @@ -43,7 +43,7 @@ props: {}, methods: { ...mapMutations('dag', ['setIsDetails', 'resetParams']), - ...mapActions('dag', ['getProcessList','getProjectList', 'getResourcesList', 'getInstancedetail']), + ...mapActions('dag', ['getProcessList','getProjectList', 'getResourcesList', 'getInstancedetail','getResourcesListJar']), ...mapActions('security', ['getTenantList','getWorkerGroupsAll']), /** * init @@ -62,6 +62,8 @@ this.getProjectList(), // get resources this.getResourcesList(), + // get jar + this.getResourcesListJar(), // get worker group list this.getWorkerGroupsAll(), this.getTenantList() diff --git a/dolphinscheduler-ui/src/js/module/components/nav/m_logo.png b/dolphinscheduler-ui/src/js/module/components/nav/m_logo.png deleted file mode 100644 index d597f2d0bfc0354d0fb1fa5d54a43a52a5815b54..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3571 zcmcInX;>528V=$GL=-AiE24&o3d&?73ke`1kxc{0BKtC#nUDxsNCF9qr7BCIVp#+c zs34Zg)uI*`a05jUP%4WXf`U*LDg{9hP%6C>S=xJ_*2h0L&ojxHnfJWk`+eVg&UupT z>TJJAQ(qH@#AZAosM_L~ zt+m`X5!TStY;AJYnO7X&(Vf9eXzm)#SHHus*5J9Dte-#nuD$dszCQYx$qii`%g3k= zx~qR8_WY&m-(8$KfT?iGbzhU&W(F@-s zaqPV0k;5@#pRGDo-I72K7zqjwUwKKiN?5sl(g3*j*rh3NQ__&i%T+imAGX4IQQ1Vc*cRdO9ik=^=7+bZAB6UOU2Uwh(sY` z=?pxHMx;)Ad(4WGJqlgX_Qa~DgdE$%`5{8kwOx_SjHC#u}VgeBZ`n&;ZUX1 zDG0=Kv_k1jnb3j}6rh+u!V{G#%>vo%k3$85IkZ%^H4=?7qxWx#r5;gYgy4oqMG+DR z*&2xmWou@GiFqQKNXiragSNTX|D_Oy<^qZ%Bw-)Q149HP3=yE*QWQ)2pij)Sl^~!@ zB=Hc5!e$%gIvWvdYpZOGIo6mBLVTf8hKW+v%q)Zj$`C6YieLalGC-nwkSI(Fok^u` zL>ovXPD9zK8DLNbei~Q=^SM!<0%fzAjzXyn6heq2%L<2z!t?nslT0HbkU0YaD0G@R zKw)qw00>j)00eTl6bb`oKosr=UFdw42#QdaLpguiyRZmCIp#{rggJD&8OVSEI+0EU zI2;NIFz3)I0GUC9A+i~l!vV>&Y%UT$x_rQ}zp^S-!6>6S$&5^gIUoSRL>QpZ5D4Ia zTrNN*gIo$sqM0L9RMnqxZAD=s30nlC)iwL;b4PTam58|fF!aUjev8Ax5R)sC2td?i zd;u7O5X8a|rRCA+XqFCqDcawtnMLG|gwKq^_}CB5WP*@#L0I7+r4tYsH}jbPH`V>L zJf>;oJOm~Eh&iTVQV~}s2PKG22-=BHb0FcLGLV9iAJ_kB?)ZP}pD8fJ1BD?7y3Z1D z%8Urg9cj8fg#Yc_pU#+dmksKBWnZN;wb%lN%vEL6-lE#c$wKPdmp-a0!O-LRFzxsw9DnLBF`A)2@ZJxQw1 zE{Mcn{v~&0*?4#-47A0ic_J(8h96!Wm&o~s!9FENNyf=)b{lo2#ydiC)0W1i9^1Zn z6e_|U)1u@#)tM9;HM3TC1zcChe!XLtZrAi5eVdG)-D|1vNb~c3dJ_hfH;0?nf^yp+7ImE9mDcN|IZgFPGg- z-*DwP@&%2!VC*~ zy{Gk1Pi=15p{a{8O_d<^XsPZGwx+#zS`Q7sbUc=?^J8ZCB4Mp%QCw?f)WJjD!$j_a z#4%lNrjGr;nb+1k^;<4!vWyB+yQKMTG4V|9_X+!HROIsK3cdH?K`V|F864ay{UssN zDCF%n$>Nr?8?4fFd-~p{sGlRl;g&3m7iV*WTQP}Q|z*|u*=ohnT{_Nkt@VYuFW z=-esYvuYNDdg`}z*L8Q8TTuH#RU3P-gE#dv6y?LlPt$a^1!U5?ZQA0|?~!jy(b#CuIJmfKY2wpU#S?2z-N^K|e=&CD0{CmqZFP+M(bnIr*RF0D zaK2MB>{Fp%P}{bvkh^`EzeH= znmp!xMg|w$sby)p=cZa*!W~YSXwi!fb}EajusVB2!R4CS6?{7qp3q$M;OUm_Cr&Bq zs;M;0~`&`X> z3R~`fb5GOHV7z;SXQSbUi=@D(!xL@>rC$oSy`u+p9yJ>p8EudM#^I$jMcUeRyyx9L z{L>WCcy&LAnmu${W{|rVJK2A}0dcW5aA8b(+WH=cBP|~lzAg95G;=OkGBDB6d+ZJA z(a4pZX4}ec*ZZ4NByIVFJ#T(p7{NEHx@I3vO;EMZAX&s^#NN_Amgh@O8Q=+bl)tbj z4(<0^=hY~5|6|oIfpI`{h9}dX9Q!z@QTxjcPSKhZW6V;CW=BdvviA@`8~Hp*)%Ol>%q)LTdQA3FIg<7WPablsxf1C=;0KR5SG z#pF;!3*=H-kB_a~33A7~79Z}olX^k0yE;C5S!~~DzCkq-Lx1NJ%z^P9jUB2hSFi}D zsw|G|p6L8OJk~xV?;)t6*?zArYyaxykG(0?RR{bteXdF>UJ%kcPM$S|#s0?2%kOQd z--vyuy)&Xb_5S9|XY4hG9* zsWbli`TYRUKP5FF$>~Xsp+Nw*j;wuQh3TuB%7+1^+AkN6s~E0~af`}}EJ}ZNy=py# z#3ZgF>KvstUVrtkW!dMKb)0eIt@*{I{gp>*v1dE-rrYmK(B^pe&gTlltAt>cP04TD zQpOX!cb#8*v2klLHIVGorF~GO^|hL#IXh - + + +