From da459e4fc07510699f7d139fdae289ef5b2bccf3 Mon Sep 17 00:00:00 2001 From: liwenhe1993 <411934049@qq.com> Date: Sat, 28 Mar 2020 21:46:42 +0800 Subject: [PATCH 01/55] Remove .helmignore file --- charts/dolphinscheduler/.helmignore | 23 ----------------------- 1 file changed, 23 deletions(-) delete mode 100644 charts/dolphinscheduler/.helmignore diff --git a/charts/dolphinscheduler/.helmignore b/charts/dolphinscheduler/.helmignore deleted file mode 100644 index 0e8a0eb36f..0000000000 --- a/charts/dolphinscheduler/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ From a07b3590e2eabdecc431c96aec77d18f192da2bb Mon Sep 17 00:00:00 2001 From: break60 <790061044@qq.com> Date: Thu, 2 Apr 2020 17:58:06 +0800 Subject: [PATCH 02/55] Fix the problem of data echo in script edit box --- .../js/conf/home/pages/dag/_source/formModel/tasks/python.vue | 3 +-- .../js/conf/home/pages/dag/_source/formModel/tasks/shell.vue | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue index 10cad6cb38..28fded41d3 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue @@ -161,8 +161,7 @@ resourceList: _.map(this.resourceList, v => { return {id: v} }), - localParams: this.localParams, - rawScript: editor ? editor.getValue() : '' + localParams: this.localParams } } }, diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue index 2be5f12809..bee095acd5 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue @@ -221,8 +221,7 @@ resourceList: _.map(this.resourceList, v => { return {id: v} }), - localParams: this.localParams, - rawScript: editor ? editor.getValue() : '' + localParams: this.localParams } } }, From b854d6b77af6168e6fe300180dc151185d667319 Mon Sep 17 00:00:00 2001 From: dailidong Date: Fri, 10 Apr 2020 11:10:37 +0800 Subject: [PATCH 03/55] [Refactor worker] simplify and optimize config (#2386) * simplify config * simplify config * simplify and optimize config --- .../alert/template/AlertTemplateFactory.java | 17 +- .../alert/utils/Constants.java | 2 - .../alert/utils/MailUtils.java | 6 +- .../src/main/resources/alert.properties | 3 - .../template/AlertTemplateFactoryTest.java | 1 - .../common/utils/HadoopUtils.java | 56 +-- .../src/main/resources/common.properties | 24 +- .../src/main/resources/datasource.properties | 3 - .../main/resources/config/install_config.conf | 131 +++++- install.sh | 407 ++---------------- script/dolphinscheduler-daemon.sh | 6 +- script/scp-hosts.sh | 22 +- script/start-all.sh | 13 +- script/stop-all.sh | 14 +- 14 files changed, 228 insertions(+), 477 deletions(-) diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/AlertTemplateFactory.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/AlertTemplateFactory.java index 58e3800339..965677e7e1 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/AlertTemplateFactory.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/AlertTemplateFactory.java @@ -17,9 +17,6 @@ package org.apache.dolphinscheduler.alert.template; import org.apache.dolphinscheduler.alert.template.impl.DefaultHTMLTemplate; -import org.apache.dolphinscheduler.alert.utils.Constants; -import org.apache.dolphinscheduler.alert.utils.PropertyUtils; -import org.apache.dolphinscheduler.common.utils.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,8 +27,6 @@ public class AlertTemplateFactory { private static final Logger logger = LoggerFactory.getLogger(AlertTemplateFactory.class); - private static final String alertTemplate = PropertyUtils.getString(Constants.ALERT_TEMPLATE); - private AlertTemplateFactory(){} /** @@ -39,16 +34,6 @@ public class AlertTemplateFactory { * @return a template, default is DefaultHTMLTemplate */ public static AlertTemplate getMessageTemplate() { - - if(StringUtils.isEmpty(alertTemplate)){ - return new DefaultHTMLTemplate(); - } - - switch (alertTemplate){ - case "html": - return new DefaultHTMLTemplate(); - default: - throw new IllegalArgumentException(String.format("not support alert template: %s",alertTemplate)); - } + return new DefaultHTMLTemplate(); } } diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/Constants.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/Constants.java index 94d95b3c26..28be8aa195 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/Constants.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/Constants.java @@ -77,8 +77,6 @@ public class Constants { public static final int NUMBER_1000 = 1000; - public static final String ALERT_TEMPLATE = "alert.template"; - public static final String SPRING_DATASOURCE_DRIVER_CLASS_NAME = "spring.datasource.driver-class-name"; public static final String SPRING_DATASOURCE_URL = "spring.datasource.url"; diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java index b0aa418630..db96f8d2f3 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java @@ -262,9 +262,13 @@ public class MailUtils { part1.setContent(partContent, Constants.TEXT_HTML_CHARSET_UTF_8); // set attach file MimeBodyPart part2 = new MimeBodyPart(); + File file = new File(xlsFilePath + Constants.SINGLE_SLASH + title + Constants.EXCEL_SUFFIX_XLS); + if (!file.getParentFile().exists()) { + file.getParentFile().mkdirs(); + } // make excel file ExcelUtils.genExcelFile(content,title,xlsFilePath); - File file = new File(xlsFilePath + Constants.SINGLE_SLASH + title + Constants.EXCEL_SUFFIX_XLS); + part2.attachFile(file); part2.setFileName(MimeUtility.encodeText(title + Constants.EXCEL_SUFFIX_XLS,Constants.UTF_8,"B")); // add components to collection diff --git a/dolphinscheduler-alert/src/main/resources/alert.properties b/dolphinscheduler-alert/src/main/resources/alert.properties index db34452fb4..839eb61475 100644 --- a/dolphinscheduler-alert/src/main/resources/alert.properties +++ b/dolphinscheduler-alert/src/main/resources/alert.properties @@ -18,9 +18,6 @@ #alert type is EMAIL/SMS alert.type=EMAIL -# alter msg template, default is html template -#alert.template=html - # mail server configuration mail.protocol=SMTP mail.server.host=xxx.xxx.com diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/template/AlertTemplateFactoryTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/template/AlertTemplateFactoryTest.java index 6865b895e2..32201e6011 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/template/AlertTemplateFactoryTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/template/AlertTemplateFactoryTest.java @@ -47,7 +47,6 @@ public class AlertTemplateFactoryTest { public void testGetMessageTemplate(){ PowerMockito.mockStatic(PropertyUtils.class); - when(PropertyUtils.getString(Constants.ALERT_TEMPLATE)).thenReturn("html"); AlertTemplate defaultTemplate = AlertTemplateFactory.getMessageTemplate(); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java index 292568d60c..94c5cf8331 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java @@ -50,7 +50,8 @@ public class HadoopUtils implements Closeable { private static String hdfsUser = PropertyUtils.getString(Constants.HDFS_ROOT_USER); public static final String resourceUploadPath = PropertyUtils.getString(RESOURCE_UPLOAD_PATH, "/dolphinscheduler"); - + public static final String rmHaIds = PropertyUtils.getString(Constants.YARN_RESOURCEMANAGER_HA_RM_IDS); + public static final String appAddress = PropertyUtils.getString(Constants.YARN_APPLICATION_STATUS_ADDRESS); private static volatile HadoopUtils instance = new HadoopUtils(); private static volatile Configuration configuration; private static volatile boolean yarnEnabled = false; @@ -58,9 +59,6 @@ public class HadoopUtils implements Closeable { private HadoopUtils(){ - if(StringUtils.isEmpty(hdfsUser)){ - hdfsUser = PropertyUtils.getString(Constants.HDFS_ROOT_USER); - } init(); initHdfsPath(); } @@ -99,14 +97,15 @@ public class HadoopUtils implements Closeable { try { configuration = new Configuration(); - String resUploadStartupType = PropertyUtils.getString(Constants.RESOURCE_STORAGE_TYPE); - ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType); + String resourceStorageType = PropertyUtils.getString(Constants.RESOURCE_STORAGE_TYPE); + ResUploadType resUploadType = ResUploadType.valueOf(resourceStorageType); if (resUploadType == ResUploadType.HDFS){ if (PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE,false)){ System.setProperty(Constants.JAVA_SECURITY_KRB5_CONF, PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH)); configuration.set(Constants.HADOOP_SECURITY_AUTHENTICATION,"kerberos"); + hdfsUser = ""; UserGroupInformation.setConfiguration(configuration); UserGroupInformation.loginUserFromKeytab(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME), PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH)); @@ -155,27 +154,6 @@ public class HadoopUtils implements Closeable { fs = FileSystem.get(configuration); } - /** - * if rmHaIds includes xx, it signs not use resourcemanager - * otherwise: - * if rmHaIds is empty, single resourcemanager enabled - * if rmHaIds not empty: resourcemanager HA enabled - */ - String rmHaIds = PropertyUtils.getString(Constants.YARN_RESOURCEMANAGER_HA_RM_IDS); - String appAddress = PropertyUtils.getString(Constants.YARN_APPLICATION_STATUS_ADDRESS); - //not use resourcemanager - if (rmHaIds.contains(Constants.YARN_RESOURCEMANAGER_HA_XX)){ - yarnEnabled = false; - } else if (!StringUtils.isEmpty(rmHaIds)) { - //resourcemanager HA enabled - appAddress = getAppAddress(appAddress, rmHaIds); - yarnEnabled = true; - logger.info("appAddress : {}", appAddress); - } else { - //single resourcemanager enabled - yarnEnabled = true; - } - configuration.set(Constants.YARN_APPLICATION_STATUS_ADDRESS, appAddress); } catch (Exception e) { logger.error(e.getMessage(), e); @@ -200,7 +178,29 @@ public class HadoopUtils implements Closeable { * @return url of application */ public String getApplicationUrl(String applicationId) { - return String.format(configuration.get(Constants.YARN_APPLICATION_STATUS_ADDRESS), applicationId); + /** + * if rmHaIds contains xx, it signs not use resourcemanager + * otherwise: + * if rmHaIds is empty, single resourcemanager enabled + * if rmHaIds not empty: resourcemanager HA enabled + */ + String appUrl = ""; + //not use resourcemanager + if (rmHaIds.contains(Constants.YARN_RESOURCEMANAGER_HA_XX)){ + + yarnEnabled = false; + logger.warn("should not step here"); + } else if (!StringUtils.isEmpty(rmHaIds)) { + //resourcemanager HA enabled + appUrl = getAppAddress(appAddress, rmHaIds); + yarnEnabled = true; + logger.info("application url : {}", appUrl); + } else { + //single resourcemanager enabled + yarnEnabled = true; + } + + return String.format(appUrl, applicationId); } /** diff --git a/dolphinscheduler-common/src/main/resources/common.properties b/dolphinscheduler-common/src/main/resources/common.properties index baead1ba42..5b883b7468 100644 --- a/dolphinscheduler-common/src/main/resources/common.properties +++ b/dolphinscheduler-common/src/main/resources/common.properties @@ -19,22 +19,22 @@ resource.storage.type=NONE # resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended -#resource.upload.path=/dolphinscheduler +resource.upload.path=/dolphinscheduler # user data local directory path, please make sure the directory exists and have read write permissions #data.basedir.path=/tmp/dolphinscheduler # whether kerberos starts -#hadoop.security.authentication.startup.state=false +hadoop.security.authentication.startup.state=false # java.security.krb5.conf path -#java.security.krb5.conf.path=/opt/krb5.conf +java.security.krb5.conf.path=/opt/krb5.conf -# loginUserFromKeytab user -#login.user.keytab.username=hdfs-mycluster@ESZ.COM +# login user from keytab username +login.user.keytab.username=hdfs-mycluster@ESZ.COM # loginUserFromKeytab path -#login.user.keytab.path=/opt/hdfs.headless.keytab +login.user.keytab.path=/opt/hdfs.headless.keytab #resource.view.suffixs #resource.view.suffixs=txt,log,sh,conf,cfg,py,java,sql,hql,xml,properties @@ -46,19 +46,19 @@ hdfs.root.user=hdfs fs.defaultFS=hdfs://mycluster:8020 # if resource.storage.type=S3，s3 endpoint -#fs.s3a.endpoint=http://192.168.199.91:9010 +fs.s3a.endpoint=http://192.168.199.91:9010 # if resource.storage.type=S3，s3 access key -#fs.s3a.access.key=A3DXS30FO22544RE +fs.s3a.access.key=A3DXS30FO22544RE # if resource.storage.type=S3，s3 secret key -#fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK +fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK -# if not use hadoop resourcemanager, please keep default value; if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty TODO +# if not use hadoop resourcemanager, please keep default value; if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx -# If resourcemanager HA enable or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ark1 to actual resourcemanager hostname. +# if resourcemanager HA enable or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ark1 to actual resourcemanager hostname. yarn.application.status.address=http://ark1:8088/ws/v1/cluster/apps/%s -# system env path. self configuration, please make sure the directory and file exists and have read write execute permissions, TODO +# system env path #dolphinscheduler.env.path=env/dolphinscheduler_env.sh diff --git a/dolphinscheduler-dao/src/main/resources/datasource.properties b/dolphinscheduler-dao/src/main/resources/datasource.properties index cc7efe3b4e..8dca4ca095 100644 --- a/dolphinscheduler-dao/src/main/resources/datasource.properties +++ b/dolphinscheduler-dao/src/main/resources/datasource.properties @@ -25,9 +25,6 @@ spring.datasource.url=jdbc:mysql://localhost:3306/dolphinscheduler?useUnicode=tr spring.datasource.username=root spring.datasource.password=root@123 -## base spring data source configuration todo need to remove -#spring.datasource.type=com.alibaba.druid.pool.DruidDataSource - # connection configuration #spring.datasource.initialSize=5 # min connection number diff --git a/dolphinscheduler-server/src/main/resources/config/install_config.conf b/dolphinscheduler-server/src/main/resources/config/install_config.conf index 0378490abb..4671be7371 100644 --- a/dolphinscheduler-server/src/main/resources/config/install_config.conf +++ b/dolphinscheduler-server/src/main/resources/config/install_config.conf @@ -15,11 +15,126 @@ # limitations under the License. # -installPath=/data1_1T/dolphinscheduler -deployUser=dolphinscheduler -ips=ark0,ark1,ark2,ark3,ark4 -sshPort=22 -masters=ark0,ark1 -workers=ark2,ark3,ark4 -alertServer=ark3 -apiServers=ark1 + +# NOTICE : If the following config has special characters in the variable `.*[]^${}\+?|()@#&`, Please escape, for example, `[` escape to `\[` +# postgresql or mysql +dbtype="mysql" + +# db config +# db address and port +dbhost="192.168.xx.xx:3306" + +# db username +username="xx" + +# db passwprd +# NOTICE: if there are special characters, please use the \ to escape, for example, `[` escape to `\[` +password="xx" + +# zk cluster +zkQuorum="192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181" + +# Note: the target installation path for dolphinscheduler, please not config as the same as the current path (pwd) +installPath="/data1_1T/dolphinscheduler" + +# deployment user +# Note: the deployment user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled, the root directory needs to be created by itself +deployUser="dolphinscheduler" + + +# alert config +# mail server host +mailServerHost="smtp.exmail.qq.com" + +# mail server port +# note: Different protocols and encryption methods correspond to different ports, when SSL/TLS is enabled, make sure the port is correct. +mailServerPort="25" + +# sender +mailSender="xxxxxxxxxx" + +# user +mailUser="xxxxxxxxxx" + +# sender password +# note: The mail.passwd is email service authorization code, not the email login password. +mailPassword="xxxxxxxxxx" + +# TLS mail protocol support +starttlsEnable="false" + +sslTrust="xxxxxxxxxx" + +# SSL mail protocol support +# note: The SSL protocol is enabled by default. +# only one of TLS and SSL can be in the true state. +sslEnable="true" + + +# resource storage type：HDFS,S3,NONE +resourceStorageType="NONE" + +# if resourceStorageType is HDFS，defaultFS write namenode address，HA you need to put core-site.xml and hdfs-site.xml in the conf directory. +# if S3，write S3 address，HA，for example ：s3a://dolphinscheduler， +# Note，s3 be sure to create the root directory /dolphinscheduler +defaultFS="hdfs://mycluster:8020" + +# if resourceStorageType is S3, the following three configuration is required, otherwise please ignore +s3Endpoint="http://192.168.xx.xx:9010" +s3AccessKey="xxxxxxxxxx" +s3SecretKey="xxxxxxxxxx" + +# if not use hadoop resourcemanager, please keep default value; if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty +yarnHaIps="192.168.xx.xx,192.168.xx.xx" + +# if resourcemanager HA enable or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ark1 to actual resourcemanager hostname. +singleYarnIp="ark1" + +# resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。/dolphinscheduler is recommended +resourceUploadPath="/dolphinscheduler" + +# who have permissions to create directory under HDFS/S3 root path +# Note: if kerberos is enabled, please config hdfsRootUser= +hdfsRootUser="hdfs" + +# kerberos config +# whether kerberos starts, if kerberos starts, following four items need to config, otherwise please ignore +kerberosStartUp="false" +# kdc krb5 config file path +krb5ConfPath="$installPath/conf/krb5.conf" +# keytab username +keytabUserName="hdfs-mycluster@ESZ.COM" +# username keytab path +keytabPath="$installPath/conf/hdfs.headless.keytab" + + +# api server port +apiServerPort="12345" + + +# install hosts +# Note: install the scheduled hostname list. If it is pseudo-distributed, just write a pseudo-distributed hostname +ips="ark0,ark1,ark2,ark3,ark4" + +# ssh port, default 22 +# Note: if ssh port is not default, modify here +sshPort="22" + +# run master machine +# Note: list of hosts hostname for deploying master +masters="ark0,ark1" + +# run worker machine +# note: list of machine hostnames for deploying workers +workers="ark2,ark3,ark4" + +# run alert machine +# note: list of machine hostnames for deploying alert server +alertServer="ark3" + +# run api machine +# note: list of machine hostnames for deploying api server +apiServers="ark1" + +# whether to start monitoring self-starting scripts +monitorServerState="false" diff --git a/install.sh b/install.sh index 354cdd2be3..20b293f697 100644 --- a/install.sh +++ b/install.sh @@ -19,307 +19,25 @@ workDir=`dirname $0` workDir=`cd ${workDir};pwd` -#To be compatible with MacOS and Linux +source ${workDir}/conf/config/install_config.conf + +# 1.replace file +echo "1.replace file" + txt="" if [[ "$OSTYPE" == "darwin"* ]]; then # Mac OSX txt="''" -elif [[ "$OSTYPE" == "linux-gnu" ]]; then - # linux - txt="" -elif [[ "$OSTYPE" == "cygwin" ]]; then - # POSIX compatibility layer and Linux environment emulation for Windows - echo "DolphinScheduler not support Windows operating system" - exit 1 -elif [[ "$OSTYPE" == "msys" ]]; then - # Lightweight shell and GNU utilities compiled for Windows (part of MinGW) - echo "DolphinScheduler not support Windows operating system" - exit 1 -elif [[ "$OSTYPE" == "win32" ]]; then - echo "DolphinScheduler not support Windows operating system" - exit 1 -elif [[ "$OSTYPE" == "freebsd"* ]]; then - # ... - txt="" -else - # Unknown. - echo "Operating system unknown, please tell us(submit issue) for better service" - exit 1 -fi - -source ${workDir}/conf/config/install_config.conf - -# for example postgresql or mysql ... -dbtype="postgresql" - -# db config -# db address and port -dbhost="192.168.xx.xx:5432" - -# db name -dbname="dolphinscheduler" - -# db username -username="xx" - -# db passwprd -# Note: if there are special characters, please use the \ transfer character to transfer -passowrd="xx" - -# conf/config/install_config.conf config -# Note: the installation path is not the same as the current path (pwd) -installPath="/data1_1T/dolphinscheduler" - -# deployment user -# Note: the deployment user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled, the root directory needs to be created by itself -deployUser="dolphinscheduler" - -# zk cluster -zkQuorum="192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181" - -# install hosts -# Note: install the scheduled hostname list. If it is pseudo-distributed, just write a pseudo-distributed hostname -ips="ark0,ark1,ark2,ark3,ark4" - -# ssh port, default 22 -# Note: if ssh port is not default, modify here -sshPort=22 - -# run master machine -# Note: list of hosts hostname for deploying master -masters="ark0,ark1" - -# run worker machine -# note: list of machine hostnames for deploying workers -workers="ark2,ark3,ark4" - -# run alert machine -# note: list of machine hostnames for deploying alert server -alertServer="ark3" - -# run api machine -# note: list of machine hostnames for deploying api server -apiServers="ark1" - -# alert config -# mail protocol -mailProtocol="SMTP" - -# mail server host -mailServerHost="smtp.exmail.qq.com" - -# mail server port -mailServerPort="25" - -# sender -mailSender="xxxxxxxxxx" - -# user -mailUser="xxxxxxxxxx" - -# sender password -mailPassword="xxxxxxxxxx" - -# TLS mail protocol support -starttlsEnable="false" - -sslTrust="xxxxxxxxxx" - -# SSL mail protocol support -# note: The SSL protocol is enabled by default. -# only one of TLS and SSL can be in the true state. -sslEnable="true" - -# download excel path -xlsFilePath="/tmp/xls" - -# Enterprise WeChat Enterprise ID Configuration -enterpriseWechatCorpId="xxxxxxxxxx" - -# Enterprise WeChat application Secret configuration -enterpriseWechatSecret="xxxxxxxxxx" - -# Enterprise WeChat Application AgentId Configuration -enterpriseWechatAgentId="xxxxxxxxxx" - -# Enterprise WeChat user configuration, multiple users to , split -enterpriseWechatUsers="xxxxx,xxxxx" - - -# whether to start monitoring self-starting scripts -monitorServerState="false" - -# resource Center upload and select storage method：HDFS,S3,NONE -resUploadStartupType="NONE" - -# if resUploadStartupType is HDFS，defaultFS write namenode address，HA you need to put core-site.xml and hdfs-site.xml in the conf directory. -# if S3，write S3 address，HA，for example ：s3a://dolphinscheduler， -# Note，s3 be sure to create the root directory /dolphinscheduler -defaultFS="hdfs://mycluster:8020" - -# if S3 is configured, the following configuration is required. -s3Endpoint="http://192.168.xx.xx:9010" -s3AccessKey="xxxxxxxxxx" -s3SecretKey="xxxxxxxxxx" - -# resourcemanager HA configuration, if it is a single resourcemanager, here is yarnHaIps="" -yarnHaIps="192.168.xx.xx,192.168.xx.xx" - -# if it is a single resourcemanager, you only need to configure one host name. If it is resourcemanager HA, the default configuration is fine. -singleYarnIp="ark1" - -# hdfs root path, the owner of the root path must be the deployment user. -# versions prior to 1.1.0 do not automatically create the hdfs root directory, you need to create it yourself. -hdfsPath="/dolphinscheduler" - -# have users who create directory permissions under hdfs root path / -# Note: if kerberos is enabled, hdfsRootUser="" can be used directly. -hdfsRootUser="hdfs" - -# common config -# Program root path -programPath="/tmp/dolphinscheduler" - -# download path -downloadPath="/tmp/dolphinscheduler/download" - -# task execute path -execPath="/tmp/dolphinscheduler/exec" - -# SHELL environmental variable path -shellEnvPath="$installPath/conf/env/dolphinscheduler_env.sh" - -# suffix of the resource file -resSuffixs="txt,log,sh,conf,cfg,py,java,sql,hql,xml" - -# development status, if true, for the SHELL script, you can view the encapsulated SHELL script in the execPath directory. -# If it is false, execute the direct delete -devState="true" - -# kerberos config -# kerberos whether to start -kerberosStartUp="false" - -# kdc krb5 config file path -krb5ConfPath="$installPath/conf/krb5.conf" - -# keytab username -keytabUserName="hdfs-mycluster@ESZ.COM" - -# username keytab path -keytabPath="$installPath/conf/hdfs.headless.keytab" - -# zk config -# zk root directory -zkRoot="/dolphinscheduler" - -# zk session timeout -zkSessionTimeout="300" - -# zk connection timeout -zkConnectionTimeout="300" - -# zk retry interval -zkRetryMaxSleep="100" - -# zk retry maximum number of times -zkRetryMaxtime="5" - - -# master config -# master execution thread maximum number, maximum parallelism of process instance -masterExecThreads="100" - -# the maximum number of master task execution threads, the maximum degree of parallelism for each process instance -masterExecTaskNum="20" - -# master heartbeat interval -masterHeartbeatInterval="10" - -# master task submission retries -masterTaskCommitRetryTimes="5" - -# master task submission retry interval -masterTaskCommitInterval="1000" - -# master maximum cpu average load, used to determine whether the master has execution capability -masterMaxCpuLoadAvg="100" - -# master reserve memory to determine if the master has execution capability -masterReservedMemory="0.1" - -# worker config -# worker execution thread -workerExecThreads="100" - -# worker heartbeat interval -workerHeartbeatInterval="10" - -# worker number of fetch tasks -workerFetchTaskNum="3" - -# worker reserve memory to determine if the master has execution capability -workerReservedMemory="0.1" - -# api config -# api server port -apiServerPort="12345" - -# api session timeout -apiServerSessionTimeout="7200" - -# api server context path -apiServerContextPath="/dolphinscheduler/" - -# spring max file size -springMaxFileSize="1024MB" - -# spring max request size -springMaxRequestSize="1024MB" - -# api max http post size -apiMaxHttpPostSize="5000000" - - -# 1,replace file -echo "1,replace file" -if [ $dbtype == "mysql" ];then - sed -i ${txt} "s#spring.datasource.url.*#spring.datasource.url=jdbc:mysql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/application.properties - sed -i ${txt} "s#spring.datasource.username.*#spring.datasource.username=${username}#g" conf/application.properties - sed -i ${txt} "s#spring.datasource.password.*#spring.datasource.password=${passowrd}#g" conf/application.properties - sed -i ${txt} "s#spring.datasource.driver-class-name.*#spring.datasource.driver-class-name=com.mysql.jdbc.Driver#g" conf/application.properties - - sed -i ${txt} "s#org.quartz.dataSource.myDs.URL.*#org.quartz.dataSource.myDs.URL=jdbc:mysql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/quartz.properties - sed -i ${txt} "s#org.quartz.dataSource.myDs.user.*#org.quartz.dataSource.myDs.user=${username}#g" conf/quartz.properties - sed -i ${txt} "s#org.quartz.dataSource.myDs.password.*#org.quartz.dataSource.myDs.password=${passowrd}#g" conf/quartz.properties - sed -i ${txt} "s#org.quartz.dataSource.myDs.driver.*#org.quartz.dataSource.myDs.driver=com.mysql.jdbc.Driver#g" conf/quartz.properties - sed -i ${txt} "s#org.quartz.jobStore.driverDelegateClass.*#org.quartz.jobStore.driverDelegateClass=org.quartz.impl.jdbcjobstore.StdJDBCDelegate#g" conf/quartz.properties fi +datasourceDriverClassname="com.mysql.jdbc.Driver" if [ $dbtype == "postgresql" ];then - sed -i ${txt} "s#spring.datasource.url.*#spring.datasource.url=jdbc:postgresql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/application.properties - sed -i ${txt} "s#spring.datasource.username.*#spring.datasource.username=${username}#g" conf/application.properties - sed -i ${txt} "s#spring.datasource.password.*#spring.datasource.password=${passowrd}#g" conf/application.properties - sed -i ${txt} "s#spring.datasource.driver-class-name.*#spring.datasource.driver-class-name=org.postgresql.Driver#g" conf/application.properties - - sed -i ${txt} "s#org.quartz.dataSource.myDs.URL.*#org.quartz.dataSource.myDs.URL=jdbc:postgresql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/quartz.properties - sed -i ${txt} "s#org.quartz.dataSource.myDs.user.*#org.quartz.dataSource.myDs.user=${username}#g" conf/quartz.properties - sed -i ${txt} "s#org.quartz.dataSource.myDs.password.*#org.quartz.dataSource.myDs.password=${passowrd}#g" conf/quartz.properties - sed -i ${txt} "s#org.quartz.dataSource.myDs.driver.*#org.quartz.dataSource.myDs.driver=org.postgresql.Driver#g" conf/quartz.properties - sed -i ${txt} "s#org.quartz.jobStore.driverDelegateClass.*#org.quartz.jobStore.driverDelegateClass=org.quartz.impl.jdbcjobstore.PostgreSQLDelegate#g" conf/quartz.properties + datasourceDriverClassname="org.postgresql.Driver" fi - -sed -i ${txt} "s#master.exec.threads.*#master.exec.threads=${masterExecThreads}#g" conf/application.properties -sed -i ${txt} "s#master.exec.task.num.*#master.exec.task.num=${masterExecTaskNum}#g" conf/application.properties -sed -i ${txt} "s#master.heartbeat.interval.*#master.heartbeat.interval=${masterHeartbeatInterval}#g" conf/application.properties -sed -i ${txt} "s#master.task.commit.retryTimes.*#master.task.commit.retryTimes=${masterTaskCommitRetryTimes}#g" conf/application.properties -sed -i ${txt} "s#master.task.commit.interval.*#master.task.commit.interval=${masterTaskCommitInterval}#g" conf/application.properties -sed -i ${txt} "s#master.reserved.memory.*#master.reserved.memory=${masterReservedMemory}#g" conf/application.properties - -sed -i ${txt} "s#worker.exec.threads.*#worker.exec.threads=${workerExecThreads}#g" conf/application.properties -sed -i ${txt} "s#worker.heartbeat.interval.*#worker.heartbeat.interval=${workerHeartbeatInterval}#g" conf/application.properties -sed -i ${txt} "s#worker.fetch.task.num.*#worker.fetch.task.num=${workerFetchTaskNum}#g" conf/application.properties -sed -i ${txt} "s#worker.reserved.memory.*#worker.reserved.memory=${workerReservedMemory}#g" conf/application.properties +sed -i ${txt} "s#spring.datasource.driver-class-name.*#spring.datasource.driver-class-name=${datasourceDriverClassname}#g" conf/datasource.properties +sed -i ${txt} "s#spring.datasource.url.*#spring.datasource.url=jdbc:${dbtype}://${dbhost}/dolphinscheduler?characterEncoding=UTF-8#g" conf/datasource.properties +sed -i ${txt} "s#spring.datasource.username.*#spring.datasource.username=${username}#g" conf/datasource.properties +sed -i ${txt} "s#spring.datasource.password.*#spring.datasource.password=${password}#g" conf/datasource.properties sed -i ${txt} "s#fs.defaultFS.*#fs.defaultFS=${defaultFS}#g" conf/common.properties sed -i ${txt} "s#fs.s3a.endpoint.*#fs.s3a.endpoint=${s3Endpoint}#g" conf/common.properties @@ -327,37 +45,18 @@ sed -i ${txt} "s#fs.s3a.access.key.*#fs.s3a.access.key=${s3AccessKey}#g" conf/co sed -i ${txt} "s#fs.s3a.secret.key.*#fs.s3a.secret.key=${s3SecretKey}#g" conf/common.properties sed -i ${txt} "s#yarn.resourcemanager.ha.rm.ids.*#yarn.resourcemanager.ha.rm.ids=${yarnHaIps}#g" conf/common.properties sed -i ${txt} "s#yarn.application.status.address.*#yarn.application.status.address=http://${singleYarnIp}:8088/ws/v1/cluster/apps/%s#g" conf/common.properties - -sed -i ${txt} "s#data.basedir.path.*#data.basedir.path=${programPath}#g" conf/common.properties -sed -i ${txt} "s#data.download.basedir.path.*#data.download.basedir.path=${downloadPath}#g" conf/common.properties -sed -i ${txt} "s#process.exec.basepath.*#process.exec.basepath=${execPath}#g" conf/common.properties sed -i ${txt} "s#hdfs.root.user.*#hdfs.root.user=${hdfsRootUser}#g" conf/common.properties -sed -i ${txt} "s#data.store2hdfs.basepath.*#data.store2hdfs.basepath=${hdfsPath}#g" conf/common.properties -sed -i ${txt} "s#res.upload.startup.type.*#res.upload.startup.type=${resUploadStartupType}#g" conf/common.properties -sed -i ${txt} "s#dolphinscheduler.env.path.*#dolphinscheduler.env.path=${shellEnvPath}#g" conf/common.properties -sed -i ${txt} "s#resource.view.suffixs.*#resource.view.suffixs=${resSuffixs}#g" conf/common.properties -sed -i ${txt} "s#development.state.*#development.state=${devState}#g" conf/common.properties +sed -i ${txt} "s#resource.upload.path.*#resource.upload.path=${resourceUploadPath}#g" conf/common.properties +sed -i ${txt} "s#resource.storage.type.*#resource.storage.type=${resourceStorageType}#g" conf/common.properties sed -i ${txt} "s#hadoop.security.authentication.startup.state.*#hadoop.security.authentication.startup.state=${kerberosStartUp}#g" conf/common.properties sed -i ${txt} "s#java.security.krb5.conf.path.*#java.security.krb5.conf.path=${krb5ConfPath}#g" conf/common.properties sed -i ${txt} "s#login.user.keytab.username.*#login.user.keytab.username=${keytabUserName}#g" conf/common.properties sed -i ${txt} "s#login.user.keytab.path.*#login.user.keytab.path=${keytabPath}#g" conf/common.properties - sed -i ${txt} "s#zookeeper.quorum.*#zookeeper.quorum=${zkQuorum}#g" conf/common.properties -sed -i ${txt} "s#zookeeper.dolphinscheduler.root.*#zookeeper.dolphinscheduler.root=${zkRoot}#g" conf/common.properties -sed -i ${txt} "s#zookeeper.session.timeout.*#zookeeper.session.timeout=${zkSessionTimeout}#g" conf/common.properties -sed -i ${txt} "s#zookeeper.connection.timeout.*#zookeeper.connection.timeout=${zkConnectionTimeout}#g" conf/common.properties -sed -i ${txt} "s#zookeeper.retry.max.sleep.*#zookeeper.retry.max.sleep=${zkRetryMaxSleep}#g" conf/common.properties -sed -i ${txt} "s#zookeeper.retry.maxtime.*#zookeeper.retry.maxtime=${zkRetryMaxtime}#g" conf/common.properties sed -i ${txt} "s#server.port.*#server.port=${apiServerPort}#g" conf/application-api.properties -sed -i ${txt} "s#server.servlet.session.timeout.*#server.servlet.session.timeout=${apiServerSessionTimeout}#g" conf/application-api.properties -sed -i ${txt} "s#server.servlet.context-path.*#server.servlet.context-path=${apiServerContextPath}#g" conf/application-api.properties -sed -i ${txt} "s#spring.servlet.multipart.max-file-size.*#spring.servlet.multipart.max-file-size=${springMaxFileSize}#g" conf/application-api.properties -sed -i ${txt} "s#spring.servlet.multipart.max-request-size.*#spring.servlet.multipart.max-request-size=${springMaxRequestSize}#g" conf/application-api.properties -sed -i ${txt} "s#server.jetty.max-http-post-size.*#server.jetty.max-http-post-size=${apiMaxHttpPostSize}#g" conf/application-api.properties -sed -i ${txt} "s#mail.protocol.*#mail.protocol=${mailProtocol}#g" conf/alert.properties sed -i ${txt} "s#mail.server.host.*#mail.server.host=${mailServerHost}#g" conf/alert.properties sed -i ${txt} "s#mail.server.port.*#mail.server.port=${mailServerPort}#g" conf/alert.properties sed -i ${txt} "s#mail.sender.*#mail.sender=${mailSender}#g" conf/alert.properties @@ -366,82 +65,38 @@ sed -i ${txt} "s#mail.passwd.*#mail.passwd=${mailPassword}#g" conf/alert.propert sed -i ${txt} "s#mail.smtp.starttls.enable.*#mail.smtp.starttls.enable=${starttlsEnable}#g" conf/alert.properties sed -i ${txt} "s#mail.smtp.ssl.trust.*#mail.smtp.ssl.trust=${sslTrust}#g" conf/alert.properties sed -i ${txt} "s#mail.smtp.ssl.enable.*#mail.smtp.ssl.enable=${sslEnable}#g" conf/alert.properties -sed -i ${txt} "s#xls.file.path.*#xls.file.path=${xlsFilePath}#g" conf/alert.properties -sed -i ${txt} "s#enterprise.wechat.corp.id.*#enterprise.wechat.corp.id=${enterpriseWechatCorpId}#g" conf/alert.properties -sed -i ${txt} "s#enterprise.wechat.secret.*#enterprise.wechat.secret=${enterpriseWechatSecret}#g" conf/alert.properties -sed -i ${txt} "s#enterprise.wechat.agent.id.*#enterprise.wechat.agent.id=${enterpriseWechatAgentId}#g" conf/alert.properties -sed -i ${txt} "s#enterprise.wechat.users.*#enterprise.wechat.users=${enterpriseWechatUsers}#g" conf/alert.properties - - -sed -i ${txt} "s#installPath.*#installPath=${installPath}#g" conf/config/install_config.conf -sed -i ${txt} "s#deployUser.*#deployUser=${deployUser}#g" conf/config/install_config.conf -sed -i ${txt} "s#ips.*#ips=${ips}#g" conf/config/install_config.conf -sed -i ${txt} "s#sshPort.*#sshPort=${sshPort}#g" conf/config/install_config.conf - - -sed -i ${txt} "s#masters.*#masters=${masters}#g" conf/config/install_config.conf -sed -i ${txt} "s#workers.*#workers=${workers}#g" conf/config/install_config.conf -sed -i ${txt} "s#alertServer.*#alertServer=${alertServer}#g" conf/config/install_config.conf -sed -i ${txt} "s#apiServers.*#apiServers=${apiServers}#g" conf/config/install_config.conf -sed -i ${txt} "s#sshPort.*#sshPort=${sshPort}#g" conf/config/install_config.conf - - -# 2,create directory -echo "2,create directory" +# 2.create directory +echo "2.create directory" if [ ! -d $installPath ];then sudo mkdir -p $installPath sudo chown -R $deployUser:$deployUser $installPath fi -hostsArr=(${ips//,/ }) -for host in ${hostsArr[@]} -do - -# create if programPath does not exist -if ! ssh -p $sshPort $host test -e $programPath; then - ssh -p $sshPort $host "sudo mkdir -p $programPath;sudo chown -R $deployUser:$deployUser $programPath" -fi - -# create if downloadPath does not exist -if ! ssh -p $sshPort $host test -e $downloadPath; then - ssh -p $sshPort $host "sudo mkdir -p $downloadPath;sudo chown -R $deployUser:$deployUser $downloadPath" -fi - -# create if execPath does not exist -if ! ssh -p $sshPort $host test -e $execPath; then - ssh -p $sshPort $host "sudo mkdir -p $execPath; sudo chown -R $deployUser:$deployUser $execPath" -fi - -# create if xlsFilePath does not exist -if ! ssh -p $sshPort $host test -e $xlsFilePath; then - ssh -p $sshPort $host "sudo mkdir -p $xlsFilePath; sudo chown -R $deployUser:$deployUser $xlsFilePath" +# 3.scp resources +echo "3.scp resources" +sh ${workDir}/script/scp-hosts.sh +if [ $? -eq 0 ] +then + echo 'scp copy completed' +else + echo 'scp copy failed to exit' + exit 1 fi -done - -# 3,stop server -echo "3,stop server" +# 4.stop server +echo "4.stop server" sh ${workDir}/script/stop-all.sh -# 4,delete zk node -echo "4,delete zk node" + +# 5.delete zk node +echo "5.delete zk node" sh ${workDir}/script/remove-zk-node.sh $zkRoot -# 5,scp resources -echo "5,scp resources" -sh ${workDir}/script/scp-hosts.sh -if [ $? -eq 0 ] -then - echo 'scp copy completed' -else - echo 'scp copy failed to exit' - exit -1 -fi -# 6,startup -echo "6,startup" +# 6.startup +echo "6.startup" sh ${workDir}/script/start-all.sh \ No newline at end of file diff --git a/script/dolphinscheduler-daemon.sh b/script/dolphinscheduler-daemon.sh index d7ef66f8bd..5fb50b86ca 100644 --- a/script/dolphinscheduler-daemon.sh +++ b/script/dolphinscheduler-daemon.sh @@ -35,6 +35,8 @@ BIN_DIR=`dirname $0` BIN_DIR=`cd "$BIN_DIR"; pwd` DOLPHINSCHEDULER_HOME=$BIN_DIR/.. +source /etc/profile + export JAVA_HOME=$JAVA_HOME #export JAVA_HOME=/opt/soft/jdk export HOSTNAME=`hostname` @@ -90,8 +92,8 @@ case $startStop in exec_command="$LOG_FILE $DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS" - echo "nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 > /dev/null &" - nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 > /dev/null & + echo "nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 &" + nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 & echo $! > $pid ;; diff --git a/script/scp-hosts.sh b/script/scp-hosts.sh index 05878c3c51..f4949f311a 100644 --- a/script/scp-hosts.sh +++ b/script/scp-hosts.sh @@ -24,16 +24,18 @@ hostsArr=(${ips//,/ }) for host in ${hostsArr[@]} do - if ! ssh -p $sshPort $host test -e $installPath; then - ssh -p $sshPort $host "sudo mkdir -p $installPath; sudo chown -R $deployUser:$deployUser $installPath" - fi + if ! ssh -p $sshPort $host test -e $installPath; then + ssh -p $sshPort $host "sudo mkdir -p $installPath; sudo chown -R $deployUser:$deployUser $installPath" + fi + echo "scp dirs to $host/$installPath starting" ssh -p $sshPort $host "cd $installPath/; rm -rf bin/ conf/ lib/ script/ sql/ ui/" - scp -P $sshPort -r $workDir/../bin $host:$installPath - scp -P $sshPort -r $workDir/../conf $host:$installPath - scp -P $sshPort -r $workDir/../lib $host:$installPath - scp -P $sshPort -r $workDir/../script $host:$installPath - scp -P $sshPort -r $workDir/../sql $host:$installPath - scp -P $sshPort -r $workDir/../ui $host:$installPath - scp -P $sshPort $workDir/../install.sh $host:$installPath + + for dsDir in bin conf lib script sql ui install.sh + do + echo "start to scp $dsDir to $host/$installPath" + scp -P $sshPort -r $workDir/../$dsDir $host:$installPath + done + + echo "scp dirs to $host/$installPath complete" done diff --git a/script/start-all.sh b/script/start-all.sh index bb4b0a1fcb..11e4572059 100644 --- a/script/start-all.sh +++ b/script/start-all.sh @@ -23,7 +23,7 @@ source $workDir/../conf/config/install_config.conf mastersHost=(${masters//,/ }) for master in ${mastersHost[@]} do - echo $master + echo "$master master server is starting" ssh -p $sshPort $master "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start master-server;" done @@ -31,10 +31,10 @@ done workersHost=(${workers//,/ }) for worker in ${workersHost[@]} do - echo $worker + echo "$worker worker server is starting" - ssh -p $sshPort $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start worker-server;" - ssh -p $sshPort $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start logger-server;" + ssh -p $sshPort $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start worker-server;" + ssh -p $sshPort $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start logger-server;" done ssh -p $sshPort $alertServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start alert-server;" @@ -42,8 +42,7 @@ ssh -p $sshPort $alertServer "cd $installPath/; sh bin/dolphinscheduler-daemon. apiServersHost=(${apiServers//,/ }) for apiServer in ${apiServersHost[@]} do - echo $apiServer - - ssh -p $sshPort $apiServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start api-server;" + echo "$apiServer worker server is starting" + ssh -p $sshPort $apiServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start api-server;" done diff --git a/script/stop-all.sh b/script/stop-all.sh index c0c6f4dd45..f761579cc8 100644 --- a/script/stop-all.sh +++ b/script/stop-all.sh @@ -24,7 +24,7 @@ source $workDir/../conf/config/install_config.conf mastersHost=(${masters//,/ }) for master in ${mastersHost[@]} do - echo $master + echo "$master master server is stopping" ssh -p $sshPort $master "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop master-server;" done @@ -32,10 +32,9 @@ done workersHost=(${workers//,/ }) for worker in ${workersHost[@]} do - echo $worker - - ssh -p $sshPort $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop worker-server;" - ssh -p $sshPort $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop logger-server;" + echo "$worker worker server is stopping" + ssh -p $sshPort $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop worker-server;" + ssh -p $sshPort $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop logger-server;" done ssh -p $sshPort $alertServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop alert-server;" @@ -43,8 +42,7 @@ ssh -p $sshPort $alertServer "cd $installPath/; sh bin/dolphinscheduler-daemon. apiServersHost=(${apiServers//,/ }) for apiServer in ${apiServersHost[@]} do - echo $apiServer - - ssh -p $sshPort $apiServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop api-server;" + echo "$apiServer worker server is stopping" + ssh -p $sshPort $apiServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop api-server;" done From 8f00ec0a542874974f15bdf7313ee76c8aefbe7c Mon Sep 17 00:00:00 2001 From: break60 <790061044@qq.com> Date: Sat, 11 Apr 2020 09:20:43 +0800 Subject: [PATCH 04/55] Optimize resource tree --- .../js/conf/home/pages/dag/_source/dag.scss | 4 +- .../js/conf/home/pages/dag/_source/dag.vue | 4 +- .../dag/_source/formModel/tasks/flink.vue | 11 +- .../dag/_source/formModel/tasks/python.vue | 36 ++++-- .../dag/_source/formModel/tasks/shell.vue | 119 ++++++++++++++++-- .../dag/_source/formModel/tasks/spark.vue | 11 +- .../pages/dag/_source/jumpAffirm/index.js | 2 +- .../pages/dag/_source/plugIn/jsPlumbHandle.js | 2 +- .../security/pages/users/_source/list.vue | 26 ++++ .../module/components/transfer/resource.vue | 74 ++++++++++- .../src/js/module/i18n/locale/en_US.js | 4 +- .../src/js/module/i18n/locale/zh_CN.js | 2 + 12 files changed, 265 insertions(+), 30 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss index 9973750d98..886ee692bf 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss @@ -130,12 +130,12 @@ } .toolbar-btn { overflow: hidden; - padding: 11px 11px 0 11px; + padding: 8px 11px 0 11px; .bar-box { width: 36px; height: 36px; float: left; - margin-bottom: 8px; + margin-bottom: 3px; border-radius: 3px; .disabled { .icos { diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue index 8628fdb8ef..6f630071c1 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue @@ -177,7 +177,7 @@ Endpoint: [ 'Dot', { radius: 1, cssClass: 'dot-style' } ], - Connector: 'Straight', + Connector: 'Bezier', PaintStyle: { lineWidth: 2, stroke: '#456' }, // Connection style ConnectionOverlays: [ [ @@ -606,7 +606,7 @@ Endpoint: [ 'Dot', { radius: 1, cssClass: 'dot-style' } ], - Connector: 'Straight', + Connector: 'Bezier', PaintStyle: { lineWidth: 2, stroke: '#456' }, // Connection style ConnectionOverlays: [ [ diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue index abf04571fd..50866afc42 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue @@ -141,7 +141,7 @@
{{$t('Resources')}}
- +
{{ node.raw.fullName }}
@@ -173,6 +173,7 @@ name: 'flink', data () { return { + valueConsistsOf: 'LEAF_PRIORITY', // Main function class mainClass: '', // Master jar package @@ -329,9 +330,15 @@ diGuiTree(item) { // Recursive convenience tree structure item.forEach(item => { item.children === '' || item.children === undefined || item.children === null || item.children.length === 0?　　　　　　　　 - delete item.children : this.diGuiTree(item.children); + this.operationTree(item) : this.diGuiTree(item.children); }) }, + operationTree(item) { + if(item.dirctory) { + item.isDisabled =true + } + delete item.children + }, }, watch: { // Listening type diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue index 28fded41d3..67669b4654 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue @@ -28,12 +28,15 @@
{{$t('Resources')}}
- - + +
{{ node.raw.fullName }}
+ +
@@ -56,6 +59,8 @@ import mListBox from './_source/listBox' import mResources from './_source/resources' import mLocalParams from './_source/localParams' + import Treeselect from '@riophae/vue-treeselect' + import '@riophae/vue-treeselect/dist/vue-treeselect.css' import disabledState from '@/module/mixin/disabledState' import codemirror from '@/conf/home/pages/resource/pages/file/pages/_source/codemirror' @@ -65,6 +70,7 @@ name: 'python', data () { return { + valueConsistsOf: 'LEAF_PRIORITY', // script rawScript: '', // Custom parameter @@ -72,7 +78,8 @@ // resource(list) resourceList: [], // Cache ResourceList - cacheResourceList: [] + cacheResourceList: [], + resourceOptions: [], } }, mixins: [disabledState], @@ -147,6 +154,18 @@ editor.setValue(this.rawScript) return editor + }, + diGuiTree(item) { // Recursive convenience tree structure + item.forEach(item => { + item.children === '' || item.children === undefined || item.children === null || item.children.length === 0?　　　　　　　　 + this.operationTree(item) : this.diGuiTree(item.children); + }) + }, + operationTree(item) { + if(item.dirctory) { + item.isDisabled =true + } + delete item.children } }, watch: { @@ -166,6 +185,9 @@ } }, created () { + let item = this.store.state.dag.resourcesListS + this.diGuiTree(item) + this.options = item let o = this.backfillItem // Non-null objects represent backfill diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue index bee095acd5..df315a14f1 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue @@ -34,7 +34,7 @@
{{$t('Resources')}}
- +
{{ node.raw.fullName }}
@@ -81,6 +81,7 @@ name: 'shell', data () { return { + valueConsistsOf: 'LEAF_PRIORITY', // script rawScript: '', // Custom parameter @@ -95,7 +96,9 @@ return { label: node.name } - } + }, + allNoResources: [], + noRes: [] } }, mixins: [disabledState], @@ -164,6 +167,11 @@ if (!this.$refs.refLocalParams._verifProp()) { return false } + // noRes + if (this.noRes.length>0) { + this.$message.warning(`${i18n.$t('Please delete all non-existent resources')}`) + return false + } // Process resourcelist let dataProcessing= _.map(this.resourceList, v => { return { @@ -205,8 +213,75 @@ diGuiTree(item) { // Recursive convenience tree structure item.forEach(item => { item.children === '' || item.children === undefined || item.children === null || item.children.length === 0?　　　　　　　　 - delete item.children : this.diGuiTree(item.children); + this.operationTree(item) : this.diGuiTree(item.children); }) + }, + operationTree(item) { + if(item.dirctory) { + item.isDisabled =true + } + delete item.children + }, + searchTree(element, id) { + // 根据id查找节点 + if (element.id == id) { + return element; + } else if (element.children != null) { + var i; + var result = null; + for (i = 0; result == null && i < element.children.length; i++) { + result = this.searchTree(element.children[i], id); + } + return result; + } + return null; + }, + dataProcess(backResource) { + let isResourceId = [] + let resourceIdArr = [] + if(this.resourceList.length>0) { + this.resourceList.forEach(v=>{ + this.options.forEach(v1=>{ + if(this.searchTree(v1,v)) { + isResourceId.push(this.searchTree(v1,v)) + } + }) + }) + resourceIdArr = isResourceId.map(item=>{ + return item.id + }) + Array.prototype.diff = function(a) { + return this.filter(function(i) {return a.indexOf(i) < 0;}); + }; + let diffSet = this.resourceList.diff(resourceIdArr); + let optionsCmp = [] + if(diffSet.length>0) { + diffSet.forEach(item=>{ + backResource.forEach(item1=>{ + if(item==item1.id || item==item1.res) { + optionsCmp.push(item1) + } + }) + }) + } + let noResources = [{ + id: -1, + name: $t('No resources'), + fullName: '/'+$t('No resources'), + children: [] + }] + if(optionsCmp.length>0) { + this.allNoResources = optionsCmp + optionsCmp = optionsCmp.map(item=>{ + return {id: item.id,name: item.name,fullName: item.res} + }) + optionsCmp.forEach(item=>{ + item.isNew = true + }) + noResources[0].children = optionsCmp + this.options = this.options.concat(noResources) + } + } } }, watch: { @@ -217,10 +292,32 @@ }, computed: { cacheParams () { + let isResourceId = [] + let resourceIdArr = [] + if(this.resourceList.length>0) { + this.resourceList.forEach(v=>{ + this.options.forEach(v1=>{ + if(this.searchTree(v1,v)) { + isResourceId.push(this.searchTree(v1,v)) + } + }) + }) + resourceIdArr = isResourceId.map(item=>{ + return {id: item.id,name: item.name,res: item.fullName} + }) + } + let result = [] + resourceIdArr.forEach(item=>{ + this.allNoResources.forEach(item1=>{ + if(item.id==item1.id) { + // resultBool = true + result.push(item1) + } + }) + }) + this.noRes = result return { - resourceList: _.map(this.resourceList, v => { - return {id: v} - }), + resourceList: resourceIdArr, localParams: this.localParams } } @@ -230,29 +327,35 @@ this.diGuiTree(item) this.options = item let o = this.backfillItem + // Non-null objects represent backfill if (!_.isEmpty(o)) { this.rawScript = o.params.rawScript || '' // backfill resourceList + let backResource = o.params.resourceList || [] let resourceList = o.params.resourceList || [] if (resourceList.length) { _.map(resourceList, v => { - if(v.res) { + if(!v.id) { this.store.dispatch('dag/getResourceId',{ type: 'FILE', fullName: '/'+v.res }).then(res => { this.resourceList.push(res.id) + this.dataProcess(backResource) }).catch(e => { - this.$message.error(e.msg || '') + this.resourceList.push(v.res) + this.dataProcess(backResource) }) } else { this.resourceList.push(v.id) + this.dataProcess(backResource) } }) this.cacheResourceList = resourceList } + // backfill localParams let localParams = o.params.localParams || [] if (localParams.length) { diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/spark.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/spark.vue index 61662d96bc..7a00528149 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/spark.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/spark.vue @@ -169,7 +169,7 @@
{{$t('Resources')}}
- +
{{ node.raw.fullName }}
@@ -212,6 +212,7 @@ name: 'spark', data () { return { + valueConsistsOf: 'LEAF_PRIORITY', // Main function class mainClass: '', // Master jar package @@ -295,9 +296,15 @@ diGuiTree(item) { // Recursive convenience tree structure item.forEach(item => { item.children === '' || item.children === undefined || item.children === null || item.children.length === 0?　　　　　　　　 - delete item.children : this.diGuiTree(item.children); + this.operationTree(item) : this.diGuiTree(item.children); }) }, + operationTree(item) { + if(item.dirctory) { + item.isDisabled =true + } + delete item.children + }, /** * verification */ diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/jumpAffirm/index.js b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/jumpAffirm/index.js index 6ac87b3372..88a258c6fe 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/jumpAffirm/index.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/jumpAffirm/index.js @@ -100,7 +100,7 @@ Affirm.isPop = (fn) => { Vue.$modal.destroy() }) }, - close () { + close () { fn() Vue.$modal.destroy() } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js index 6a17239e65..c77127d49a 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js @@ -332,7 +332,7 @@ JSP.prototype.tasksContextmenu = function (event) { }) }) } - if (!isTwo) { + if (!isTwo) { // edit node $(`#editNodes`).click(ev => { findComponentDownward(this.dag.$root, 'dag-chart')._createNodes({ diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/users/_source/list.vue b/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/users/_source/list.vue index d88d6e81de..ead385c109 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/users/_source/list.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/users/_source/list.vue @@ -215,6 +215,23 @@ }) }) }, + /* + getAllLeaf + */ + getAllLeaf (data) { + let result = [] + let getLeaf = (data)=> { + data.forEach(item => { + if (item.children.length==0) { + result.push(item) + } else { + getLeaf(item.children) + } + }) + } + getLeaf(data) + return result + }, _authFile (item, i) { this.$refs[`poptip-auth-${i}`][0].doClose() this.getResourceList({ @@ -240,6 +257,15 @@ }) let fileTargetList = [] let udfTargetList = [] + + let pathId = [] + data[1].forEach(v=>{ + let arr = [] + arr[0] = v + if(this.getAllLeaf(arr).length>0) { + pathId.push(this.getAllLeaf(arr)[0]) + } + }) data[1].forEach((value,index,array)=>{ if(value.type =='FILE'){ fileTargetList.push(value) diff --git a/dolphinscheduler-ui/src/js/module/components/transfer/resource.vue b/dolphinscheduler-ui/src/js/module/components/transfer/resource.vue index 97397c1809..41964a0860 100644 --- a/dolphinscheduler-ui/src/js/module/components/transfer/resource.vue +++ b/dolphinscheduler-ui/src/js/module/components/transfer/resource.vue @@ -24,10 +24,10 @@ {{$t('UDF resources')}} - +
{{ node.raw.fullName }}
 - +
{{ node.raw.fullName }}
 + v.code === code)[0].desc }, @@ -306,6 +307,21 @@ releaseState: 1 }) }, + /** + * copy + */ + _copyProcess (item) { + this.copyProcess({ + processId: item.id + }).then(res => { + this.$message.success(res.msg) + $('body').find('.tooltip.fade.top.in').remove() + this._onUpdate() + }).catch(e => { + this.$message.error(e.msg || '') + }) + }, + _export (item) { this.exportDefinition({ processDefinitionId: item.id, diff --git a/dolphinscheduler-ui/src/js/conf/home/store/dag/actions.js b/dolphinscheduler-ui/src/js/conf/home/store/dag/actions.js index a63c9edb8c..f282c8e30a 100644 --- a/dolphinscheduler-ui/src/js/conf/home/store/dag/actions.js +++ b/dolphinscheduler-ui/src/js/conf/home/store/dag/actions.js @@ -90,6 +90,7 @@ export default { }) }) }, + /** * Get process definition DAG diagram details */ @@ -127,6 +128,22 @@ export default { }) }) }, + +/** + * Get process definition DAG diagram details + */ + copyProcess ({ state }, payload) { + return new Promise((resolve, reject) => { + io.post(`projects/${state.projectName}/process/copy`, { + processId: payload.processId + }, res => { + resolve(res) + }).catch(e => { + reject(e) + }) + }) + }, + /** * Get the process instance DAG diagram details */ diff --git a/pom.xml b/pom.xml index 8f81e2aea9..0647724ed0 100644 --- a/pom.xml +++ b/pom.xml @@ -764,6 +764,7 @@ **/common/utils/HadoopUtilsTest.java **/common/utils/HttpUtilsTest.java **/common/ConstantsTest.java + **/common/utils/HadoopUtils.java **/dao/mapper/AccessTokenMapperTest.java **/dao/mapper/AlertGroupMapperTest.java **/dao/mapper/CommandMapperTest.java From 733e8bcb0daaec64d694b98bd38736bb703216e8 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Fri, 24 Apr 2020 14:09:36 +0800 Subject: [PATCH 44/55] support download the resources existed in process definition which was old version (#2506) * fix #2442 and remove unavailable code * revert verifyResourceName method * Add ServiceException * add ServiceExceptionTest * update ServiceExceptionTest * add ServiceExceptionTest in pom * support download the resources existed in process definition which was old version --- .../consumer/TaskPriorityQueueConsumer.java | 44 ++++++++++--------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java index cdd9ff2219..480d6657c2 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java @@ -48,13 +48,13 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; -import java.util.HashSet; +import java.util.ArrayList; import java.util.List; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.apache.dolphinscheduler.common.Constants.*; +import static org.apache.dolphinscheduler.common.Constants.SLEEP_TIME_MILLIS; /** * TaskUpdateQueue consumer @@ -328,36 +328,38 @@ public class TaskPriorityQueueConsumer extends Thread{ return false; } - /** - * create project resource files + * get resource full name list */ - private List getResourceFullNames(TaskNode taskNode){ - - Set resourceIdsSet = new HashSet<>(); + private List getResourceFullNames(TaskNode taskNode) { + List resourceFullNameList = new ArrayList<>(); AbstractParameters baseParam = TaskParametersUtils.getParameters(taskNode.getType(), taskNode.getParams()); if (baseParam != null) { List projectResourceFiles = baseParam.getResourceFilesList(); if (projectResourceFiles != null) { - Stream resourceInfotream = projectResourceFiles.stream().map(resourceInfo -> resourceInfo.getId()); - resourceIdsSet.addAll(resourceInfotream.collect(Collectors.toSet())); - } - } + // filter the resources that the resource id equals 0 + Set oldVersionResources = projectResourceFiles.stream().filter(t -> t.getId() == 0).collect(Collectors.toSet()); + if (CollectionUtils.isNotEmpty(oldVersionResources)) { + resourceFullNameList.addAll(oldVersionResources.stream().map(resource -> resource.getRes()).collect(Collectors.toSet())); + } - if (CollectionUtils.isEmpty(resourceIdsSet)){ - return null; - } - - Integer[] resourceIds = resourceIdsSet.toArray(new Integer[resourceIdsSet.size()]); + // get the resource id in order to get the resource names in batch + Stream resourceIdStream = projectResourceFiles.stream().map(resourceInfo -> resourceInfo.getId()); + Set resourceIdsSet = resourceIdStream.collect(Collectors.toSet()); - List resources = processService.listResourceByIds(resourceIds); + if (CollectionUtils.isNotEmpty(resourceIdsSet)) { + Integer[] resourceIds = resourceIdsSet.toArray(new Integer[resourceIdsSet.size()]); - List resourceFullNames = resources.stream() - .map(resourceInfo -> resourceInfo.getFullName()) - .collect(Collectors.toList()); + List resources = processService.listResourceByIds(resourceIds); + resourceFullNameList.addAll(resources.stream() + .map(resourceInfo -> resourceInfo.getFullName()) + .collect(Collectors.toList())); + } + } + } - return resourceFullNames; + return resourceFullNameList; } } From af4e3c3e1b74cabe51eb1d9692b97ebf209770f1 Mon Sep 17 00:00:00 2001 From: break60 <790061044@qq.com> Date: Fri, 24 Apr 2020 15:32:44 +0800 Subject: [PATCH 45/55] Echo when the workflow is running --- .../pages/definition/pages/list/_source/start.vue | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/start.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/start.vue index ff56de53e5..e47d8c757f 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/start.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/start.vue @@ -193,6 +193,7 @@ runMode: 'RUN_MODE_SERIAL', processInstancePriority: 'MEDIUM', workerGroup: 'default' + } }, props: { @@ -277,6 +278,18 @@ this.workflowName = this.item.name this._getReceiver() + let stateWorkerGroupsList = this.store.state.security.workerGroupsListAll || [] + if (stateWorkerGroupsList.length) { + this.workerGroup = stateWorkerGroupsList[0].id + } else { + this.store.dispatch('security/getWorkerGroupsAll').then(res => { + this.$nextTick(() => { + if(res.length>0) { + this.workerGroup = res[0].id + } + }) + }) + } }, mounted () { this._getNotifyGroupList().then(() => { From db7c0cbd8c98cfce369531b236b7ab53fb3b5a48 Mon Sep 17 00:00:00 2001 From: qiaozhanwei Date: Fri, 24 Apr 2020 16:51:16 +0800 Subject: [PATCH 46/55] #2499 buf fix (#2518) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * dispatch task fail will set task status failed * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，task status statistics and process status statistics bug fix (#2357) 2，worker group bug fix * 1，task status statistics and process status statistics bug fix (#2357) 2，worker group bug fix * 1，task status statistics and process status statistics bug fix (#2357) 2，worker group bug fix * 1，task status statistics and process status statistics bug fix (#2357) 2，worker group bug fix * send mail error, #2466 bug fix * send mail error, #2466 bug fix * send mail error, #2466 bug fix * send mail error, #2466 bug fix * #2486 bug fix * host and workergroup compatible * EnterpriseWeChatUtils modify * EnterpriseWeChatUtils modify * EnterpriseWeChatUtils modify * #2499 bug fix * add comment * revert comment * revert comment * #2499 buf fix Co-authored-by: qiaozhanwei --- .../org/apache/dolphinscheduler/server/zk/ZKMasterClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java index e923157360..69aecee444 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java @@ -155,7 +155,7 @@ public class ZKMasterClient extends AbstractZKClient { * @throws Exception exception */ private void failoverServerWhenDown(String serverHost, ZKNodeType zkNodeType) throws Exception { - if(StringUtils.isEmpty(serverHost)){ + if(StringUtils.isEmpty(serverHost) || serverHost.startsWith(OSUtils.getHost())){ return ; } switch (zkNodeType){ From f1d8c1514807bf88750316dd398ae8c66bc6be45 Mon Sep 17 00:00:00 2001 From: liwenhe1993 <411934049@qq.com> Date: Sun, 26 Apr 2020 10:30:44 +0800 Subject: [PATCH 47/55] Add docker-compose.yml --- docker-compose/docker-compose.yml | 253 ++++++++++++++++++++++++++++++ 1 file changed, 253 insertions(+) create mode 100644 docker-compose/docker-compose.yml diff --git a/docker-compose/docker-compose.yml b/docker-compose/docker-compose.yml new file mode 100644 index 0000000000..804f39a5d4 --- /dev/null +++ b/docker-compose/docker-compose.yml @@ -0,0 +1,253 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "3" + +networks: + dolphinscheduler-postgresql: + driver: bridge + dolphinscheduler-zookeeper: + driver: bridge + dolphinscheduler-api: + driver: bridge + dolphinscheduler-frontend: + driver: bridge + dolphinscheduler-alert: + driver: bridge + dolphinscheduler-master: + driver: bridge + dolphinscheduler-worker: + driver: bridge + +volumes: + dolphinscheduler-postgresql: + dolphinscheduler-zookeeper: + dolphinscheduler-api: + dolphinscheduler-frontend: + dolphinscheduler-alert: + dolphinscheduler-master: + dolphinscheduler-worker-data: + dolphinscheduler-worker-logs: + dolphinscheduler-worker-task-env: + +services: + + dolphinscheduler-postgresql: + image: bitnami/postgresql:latest + container_name: dolphinscheduler-postgresql + ports: + - 5432:5432 + environment: + TZ: Asia/Shanghai + POSTGRESQL_USERNAME: root + POSTGRESQL_PASSWORD: root + POSTGRESQL_DATABASE: dolphinscheduler + healthcheck: + test: ["CMD", "pg_isready", "-U", "${POSTGRESQL_USERNAME}", "-d", "{POSTGRESQL_PASSWORD}", "-h", "localhost", "5432"] + interval: 30s + timeout: 5s + retries: 3 + # start_period: 30s + volumes: + - dolphinscheduler-postgresql:/bitnami/postgresql + networks: + - dolphinscheduler-postgresql + + dolphinscheduler-zookeeper: + image: bitnami/zookeeper:latest + container_name: dolphinscheduler-zookeeper + ports: + - 2181:2181 + environment: + TZ: Asia/Shanghai + ALLOW_ANONYMOUS_LOGIN: "yes" + healthcheck: + test: ["CMD-SHELL", "nc -z localhost 2181"] + interval: 30s + timeout: 5s + retries: 3 + # start_period: 30s + volumes: + - dolphinscheduler-zookeeper:/bitnami/zookeeper + networks: + - dolphinscheduler-zookeeper + + dolphinscheduler-api: + image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:1.2.1 + container_name: dolphinscheduler-api + command: ["api-server"] + ports: + - 12345:12345 + environment: + TZ: Asia/Shanghai + POSTGRESQL_HOST: dolphinscheduler-postgresql + POSTGRESQL_PORT: 5432 + POSTGRESQL_USERNAME: root + POSTGRESQL_PASSWORD: root + POSTGRESQL_DATABASE: dolphinscheduler + ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:12345"] + interval: 30s + timeout: 5s + retries: 3 + # start_period: 30s + depends_on: + - dolphinscheduler-postgresql + - dolphinscheduler-zookeeper + volumes: + - dolphinscheduler-api:/opt/dolphinscheduler/logs + networks: + - dolphinscheduler-api + - dolphinscheduler-postgresql + - dolphinscheduler-zookeeper + + dolphinscheduler-frontend: + image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:1.2.1 + container_name: dolphinscheduler-frontend + command: ["frontend"] + ports: + - 8888:8888 + environment: + TZ: Asia/Shanghai + FRONTEND_API_SERVER_HOST: dolphinscheduler-api + FRONTEND_API_SERVER_PORT: 12345 + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:8888"] + interval: 30s + timeout: 5s + retries: 3 + # start_period: 30s + depends_on: + - dolphinscheduler-api + volumes: + - dolphinscheduler-frontend:/var/log/nginx + networks: + - dolphinscheduler-api + + dolphinscheduler-alert: + image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:1.2.1 + container_name: dolphinscheduler-alert + command: ["alert-server"] + environment: + TZ: Asia/Shanghai + XLS_FILE_PATH: "/tmp/xls" + MAIL_SERVER_HOST: "" + MAIL_SERVER_PORT: "" + MAIL_SENDER: "" + MAIL_USER: "" + MAIL_PASSWD: "" + MAIL_SMTP_STARTTLS_ENABLE: "false" + MAIL_SMTP_SSL_ENABLE: "false" + MAIL_SMTP_SSL_TRUST: "" + ENTERPRISE_WECHAT_ENABLE: "false" + ENTERPRISE_WECHAT_CORP_ID: "" + ENTERPRISE_WECHAT_SECRET: "" + ENTERPRISE_WECHAT_AGENT_ID: "" + ENTERPRISE_WECHAT_USERS: "" + POSTGRESQL_HOST: dolphinscheduler-postgresql + POSTGRESQL_PORT: 5432 + POSTGRESQL_USERNAME: root + POSTGRESQL_PASSWORD: root + POSTGRESQL_DATABASE: dolphinscheduler + healthcheck: + test: ["CMD", "/root/checkpoint.sh", "AlertServer"] + interval: 30s + timeout: 5s + retries: 3 + # start_period: 30s + depends_on: + - dolphinscheduler-postgresql + volumes: + - dolphinscheduler-alert:/opt/dolphinscheduler/logs + networks: + - dolphinscheduler-alert + - dolphinscheduler-postgresql + + dolphinscheduler-master: + image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:1.2.1 + container_name: dolphinscheduler-master + command: ["master-server"] + ports: + - 5678:5678 + environment: + TZ: Asia/Shanghai + MASTER_EXEC_THREADS: "100" + MASTER_EXEC_TASK_NUM: "20" + MASTER_HEARTBEAT_INTERVAL: "10" + MASTER_TASK_COMMIT_RETRYTIMES: "5" + MASTER_TASK_COMMIT_INTERVAL: "1000" + MASTER_MAX_CPULOAD_AVG: "100" + MASTER_RESERVED_MEMORY: "0.1" + POSTGRESQL_HOST: dolphinscheduler-postgresql + POSTGRESQL_PORT: 5432 + POSTGRESQL_USERNAME: root + POSTGRESQL_PASSWORD: root + POSTGRESQL_DATABASE: dolphinscheduler + ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 + healthcheck: + test: ["CMD", "/root/checkpoint.sh", "MasterServer"] + interval: 30s + timeout: 5s + retries: 3 + # start_period: 30s + depends_on: + - dolphinscheduler-postgresql + - dolphinscheduler-zookeeper + volumes: + - dolphinscheduler-master:/opt/dolphinscheduler/logs + networks: + - dolphinscheduler-postgresql + - dolphinscheduler-zookeeper + + dolphinscheduler-worker: + image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:1.2.1 + container_name: dolphinscheduler-worker + command: ["worker-server"] + ports: + - 1234:1234 + - 50051:50051 + environment: + TZ: Asia/Shanghai + WORKER_EXEC_THREADS: "100" + WORKER_HEARTBEAT_INTERVAL: "10" + WORKER_FETCH_TASK_NUM: "3" + WORKER_MAX_CPULOAD_AVG: "100" + WORKER_RESERVED_MEMORY: "0.1" + WORKER_GROUP: "default" + DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler" + POSTGRESQL_HOST: dolphinscheduler-postgresql + POSTGRESQL_PORT: 5432 + POSTGRESQL_USERNAME: root + POSTGRESQL_PASSWORD: root + POSTGRESQL_DATABASE: dolphinscheduler + ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 + healthcheck: + test: ["CMD", "/root/checkpoint.sh", "WorkerServer"] + interval: 30s + timeout: 5s + retries: 3 + # start_period: 30s + depends_on: + - dolphinscheduler-postgresql + - dolphinscheduler-zookeeper + volumes: + - dolphinscheduler-worker-data:/tmp/dolphinscheduler + - dolphinscheduler-worker-logs:/opt/dolphinscheduler/logs + - dolphinscheduler-worker-task-env:/opt/dolphinscheduler/conf/env/ + networks: + - dolphinscheduler-postgresql + - dolphinscheduler-zookeeper \ No newline at end of file From 7eecfe45eaa34536c4c67f6f95871d97b4ecc4ce Mon Sep 17 00:00:00 2001 From: liwenhe1993 <411934049@qq.com> Date: Sun, 26 Apr 2020 14:13:07 +0800 Subject: [PATCH 48/55] Modify docker-compose and kubernetes config 1. Move kubernetes to docker directory 2. Change docker-compose directory's name, and move docker-compose to docker directory 3. Modify docker-compose config and kubernetes config 4. Modify startup.sh in dockerfile --- .../docker-swarm}/docker-compose.yml | 16 +++++++++--- docker/docker-swarm/dolphinscheduler_env.sh | 26 +++++++++++++++++++ .../kubernetes}/dolphinscheduler/Chart.yaml | 0 .../kubernetes}/dolphinscheduler/README.md | 0 .../dolphinscheduler/requirements.yaml | 0 .../dolphinscheduler/templates/NOTES.txt | 0 .../dolphinscheduler/templates/_helpers.tpl | 0 .../configmap-dolphinscheduler-alert.yaml | 0 .../configmap-dolphinscheduler-master.yaml | 0 .../configmap-dolphinscheduler-worker.yaml | 0 .../deployment-dolphinscheduler-alert.yaml | 14 +++++++--- .../deployment-dolphinscheduler-api.yaml | 12 ++++++--- .../deployment-dolphinscheduler-frontend.yaml | 0 .../dolphinscheduler/templates/ingress.yaml | 0 .../templates/pvc-dolphinscheduler-alert.yaml | 0 .../templates/pvc-dolphinscheduler-api.yaml | 0 .../pvc-dolphinscheduler-frontend.yaml | 0 .../templates/secret-external-postgresql.yaml | 0 .../statefulset-dolphinscheduler-master.yaml | 6 +++++ .../statefulset-dolphinscheduler-worker.yaml | 8 +++++- .../templates/svc-dolphinscheduler-api.yaml | 0 .../svc-dolphinscheduler-frontend.yaml | 0 .../svc-dolphinscheduler-master-headless.yaml | 0 .../svc-dolphinscheduler-worker-headless.yaml | 0 .../kubernetes}/dolphinscheduler/values.yaml | 0 dockerfile/startup.sh | 4 +-- 26 files changed, 72 insertions(+), 14 deletions(-) rename {docker-compose => docker/docker-swarm}/docker-compose.yml (95%) create mode 100644 docker/docker-swarm/dolphinscheduler_env.sh rename {kubernetes => docker/kubernetes}/dolphinscheduler/Chart.yaml (100%) rename {kubernetes => docker/kubernetes}/dolphinscheduler/README.md (100%) rename {kubernetes => docker/kubernetes}/dolphinscheduler/requirements.yaml (100%) rename {kubernetes => docker/kubernetes}/dolphinscheduler/templates/NOTES.txt (100%) rename {kubernetes => docker/kubernetes}/dolphinscheduler/templates/_helpers.tpl (100%) rename {kubernetes => docker/kubernetes}/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml (100%) rename {kubernetes => docker/kubernetes}/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml (100%) rename {kubernetes => docker/kubernetes}/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml (100%) rename {kubernetes => docker/kubernetes}/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml (96%) rename {kubernetes => docker/kubernetes}/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml (94%) rename {kubernetes => docker/kubernetes}/dolphinscheduler/templates/deployment-dolphinscheduler-frontend.yaml (100%) rename {kubernetes => docker/kubernetes}/dolphinscheduler/templates/ingress.yaml (100%) rename {kubernetes => docker/kubernetes}/dolphinscheduler/templates/pvc-dolphinscheduler-alert.yaml (100%) rename {kubernetes => docker/kubernetes}/dolphinscheduler/templates/pvc-dolphinscheduler-api.yaml (100%) rename {kubernetes => docker/kubernetes}/dolphinscheduler/templates/pvc-dolphinscheduler-frontend.yaml (100%) rename {kubernetes => docker/kubernetes}/dolphinscheduler/templates/secret-external-postgresql.yaml (100%) rename {kubernetes => docker/kubernetes}/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml (97%) rename {kubernetes => docker/kubernetes}/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml (97%) rename {kubernetes => docker/kubernetes}/dolphinscheduler/templates/svc-dolphinscheduler-api.yaml (100%) rename {kubernetes => docker/kubernetes}/dolphinscheduler/templates/svc-dolphinscheduler-frontend.yaml (100%) rename {kubernetes => docker/kubernetes}/dolphinscheduler/templates/svc-dolphinscheduler-master-headless.yaml (100%) rename {kubernetes => docker/kubernetes}/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml (100%) rename {kubernetes => docker/kubernetes}/dolphinscheduler/values.yaml (100%) diff --git a/docker-compose/docker-compose.yml b/docker/docker-swarm/docker-compose.yml similarity index 95% rename from docker-compose/docker-compose.yml rename to docker/docker-swarm/docker-compose.yml index 804f39a5d4..20fb0cced3 100644 --- a/docker-compose/docker-compose.yml +++ b/docker/docker-swarm/docker-compose.yml @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -version: "3" +version: "3.4" networks: dolphinscheduler-postgresql: @@ -41,7 +41,10 @@ volumes: dolphinscheduler-master: dolphinscheduler-worker-data: dolphinscheduler-worker-logs: + +configs: dolphinscheduler-worker-task-env: + file: ./dolphinscheduler_env.sh services: @@ -136,6 +139,7 @@ services: volumes: - dolphinscheduler-frontend:/var/log/nginx networks: + - dolphinscheduler-frontend - dolphinscheduler-api dolphinscheduler-alert: @@ -209,7 +213,8 @@ services: - dolphinscheduler-zookeeper volumes: - dolphinscheduler-master:/opt/dolphinscheduler/logs - networks: + networks: + - dolphinscheduler-master - dolphinscheduler-postgresql - dolphinscheduler-zookeeper @@ -247,7 +252,10 @@ services: volumes: - dolphinscheduler-worker-data:/tmp/dolphinscheduler - dolphinscheduler-worker-logs:/opt/dolphinscheduler/logs - - dolphinscheduler-worker-task-env:/opt/dolphinscheduler/conf/env/ - networks: + configs: + - source: dolphinscheduler-worker-task-env + target: /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh + networks: + - dolphinscheduler-worker - dolphinscheduler-postgresql - dolphinscheduler-zookeeper \ No newline at end of file diff --git a/docker/docker-swarm/dolphinscheduler_env.sh b/docker/docker-swarm/dolphinscheduler_env.sh new file mode 100644 index 0000000000..790e30636e --- /dev/null +++ b/docker/docker-swarm/dolphinscheduler_env.sh @@ -0,0 +1,26 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +export HADOOP_HOME=/opt/soft/hadoop +export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop +export SPARK_HOME1=/opt/soft/spark1 +export SPARK_HOME2=/opt/soft/spark2 +export PYTHON_HOME=/opt/soft/python +export JAVA_HOME=/opt/soft/java +export HIVE_HOME=/opt/soft/hive +export FLINK_HOME=/opt/soft/flink +export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$PATH \ No newline at end of file diff --git a/kubernetes/dolphinscheduler/Chart.yaml b/docker/kubernetes/dolphinscheduler/Chart.yaml similarity index 100% rename from kubernetes/dolphinscheduler/Chart.yaml rename to docker/kubernetes/dolphinscheduler/Chart.yaml diff --git a/kubernetes/dolphinscheduler/README.md b/docker/kubernetes/dolphinscheduler/README.md similarity index 100% rename from kubernetes/dolphinscheduler/README.md rename to docker/kubernetes/dolphinscheduler/README.md diff --git a/kubernetes/dolphinscheduler/requirements.yaml b/docker/kubernetes/dolphinscheduler/requirements.yaml similarity index 100% rename from kubernetes/dolphinscheduler/requirements.yaml rename to docker/kubernetes/dolphinscheduler/requirements.yaml diff --git a/kubernetes/dolphinscheduler/templates/NOTES.txt b/docker/kubernetes/dolphinscheduler/templates/NOTES.txt similarity index 100% rename from kubernetes/dolphinscheduler/templates/NOTES.txt rename to docker/kubernetes/dolphinscheduler/templates/NOTES.txt diff --git a/kubernetes/dolphinscheduler/templates/_helpers.tpl b/docker/kubernetes/dolphinscheduler/templates/_helpers.tpl similarity index 100% rename from kubernetes/dolphinscheduler/templates/_helpers.tpl rename to docker/kubernetes/dolphinscheduler/templates/_helpers.tpl diff --git a/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml similarity index 100% rename from kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml rename to docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml diff --git a/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml similarity index 100% rename from kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml rename to docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml diff --git a/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml similarity index 100% rename from kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml rename to docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml diff --git a/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml similarity index 96% rename from kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml rename to docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml index f3bb1dd880..69662e95d9 100644 --- a/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml @@ -166,19 +166,19 @@ spec: value: {{ template "dolphinscheduler.postgresql.fullname" . }} {{- else }} value: {{ .Values.externalDatabase.host | quote }} - {{- end }} + {{- end }} - name: POSTGRESQL_PORT {{- if .Values.postgresql.enabled }} value: "5432" {{- else }} value: {{ .Values.externalDatabase.port }} - {{- end }} + {{- end }} - name: POSTGRESQL_USERNAME {{- if .Values.postgresql.enabled }} value: {{ .Values.postgresql.postgresqlUsername }} {{- else }} value: {{ .Values.externalDatabase.username | quote }} - {{- end }} + {{- end }} - name: POSTGRESQL_PASSWORD valueFrom: secretKeyRef: @@ -188,7 +188,13 @@ spec: {{- else }} name: {{ printf "%s-%s" .Release.Name "externaldb" }} key: db-password - {{- end }} + {{- end }} + - name: POSTGRESQL_DATABASE + {{- if .Values.postgresql.enabled }} + value: {{ .Values.postgresql.postgresqlDatabase }} + {{- else }} + value: {{ .Values.externalDatabase.database | quote }} + {{- end }} {{- if .Values.alert.livenessProbe.enabled }} livenessProbe: exec: diff --git a/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml similarity index 94% rename from kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml rename to docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml index 926ce3c062..487ede0b8f 100644 --- a/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml @@ -99,19 +99,19 @@ spec: value: {{ template "dolphinscheduler.postgresql.fullname" . }} {{- else }} value: {{ .Values.externalDatabase.host | quote }} - {{- end }} + {{- end }} - name: POSTGRESQL_PORT {{- if .Values.postgresql.enabled }} value: "5432" {{- else }} value: {{ .Values.externalDatabase.port }} - {{- end }} + {{- end }} - name: POSTGRESQL_USERNAME {{- if .Values.postgresql.enabled }} value: {{ .Values.postgresql.postgresqlUsername }} {{- else }} value: {{ .Values.externalDatabase.username | quote }} - {{- end }} + {{- end }} - name: POSTGRESQL_PASSWORD valueFrom: secretKeyRef: @@ -122,6 +122,12 @@ spec: name: {{ printf "%s-%s" .Release.Name "externaldb" }} key: db-password {{- end }} + - name: POSTGRESQL_DATABASE + {{- if .Values.postgresql.enabled }} + value: {{ .Values.postgresql.postgresqlDatabase }} + {{- else }} + value: {{ .Values.externalDatabase.database | quote }} + {{- end }} - name: ZOOKEEPER_QUORUM {{- if .Values.zookeeper.enabled }} value: "{{ template "dolphinscheduler.zookeeper.quorum" . }}" diff --git a/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-frontend.yaml b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-frontend.yaml similarity index 100% rename from kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-frontend.yaml rename to docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-frontend.yaml diff --git a/kubernetes/dolphinscheduler/templates/ingress.yaml b/docker/kubernetes/dolphinscheduler/templates/ingress.yaml similarity index 100% rename from kubernetes/dolphinscheduler/templates/ingress.yaml rename to docker/kubernetes/dolphinscheduler/templates/ingress.yaml diff --git a/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-alert.yaml b/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-alert.yaml similarity index 100% rename from kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-alert.yaml rename to docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-alert.yaml diff --git a/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-api.yaml b/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-api.yaml similarity index 100% rename from kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-api.yaml rename to docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-api.yaml diff --git a/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-frontend.yaml b/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-frontend.yaml similarity index 100% rename from kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-frontend.yaml rename to docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-frontend.yaml diff --git a/kubernetes/dolphinscheduler/templates/secret-external-postgresql.yaml b/docker/kubernetes/dolphinscheduler/templates/secret-external-postgresql.yaml similarity index 100% rename from kubernetes/dolphinscheduler/templates/secret-external-postgresql.yaml rename to docker/kubernetes/dolphinscheduler/templates/secret-external-postgresql.yaml diff --git a/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml similarity index 97% rename from kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml rename to docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml index adc35ce804..b142fac4fe 100644 --- a/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml @@ -188,6 +188,12 @@ spec: name: {{ printf "%s-%s" .Release.Name "externaldb" }} key: db-password {{- end }} + - name: POSTGRESQL_DATABASE + {{- if .Values.postgresql.enabled }} + value: {{ .Values.postgresql.postgresqlDatabase }} + {{- else }} + value: {{ .Values.externalDatabase.database | quote }} + {{- end }} - name: ZOOKEEPER_QUORUM {{- if .Values.zookeeper.enabled }} value: {{ template "dolphinscheduler.zookeeper.quorum" . }} diff --git a/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml similarity index 97% rename from kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml rename to docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml index 5a03c52775..198cef43b6 100644 --- a/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml @@ -184,7 +184,13 @@ spec: {{- else }} name: {{ printf "%s-%s" .Release.Name "externaldb" }} key: db-password - {{- end }} + {{- end }} + - name: POSTGRESQL_DATABASE + {{- if .Values.postgresql.enabled }} + value: {{ .Values.postgresql.postgresqlDatabase }} + {{- else }} + value: {{ .Values.externalDatabase.database | quote }} + {{- end }} - name: ZOOKEEPER_QUORUM {{- if .Values.zookeeper.enabled }} value: "{{ template "dolphinscheduler.zookeeper.quorum" . }}" diff --git a/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-api.yaml b/docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-api.yaml similarity index 100% rename from kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-api.yaml rename to docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-api.yaml diff --git a/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-frontend.yaml b/docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-frontend.yaml similarity index 100% rename from kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-frontend.yaml rename to docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-frontend.yaml diff --git a/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-master-headless.yaml b/docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-master-headless.yaml similarity index 100% rename from kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-master-headless.yaml rename to docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-master-headless.yaml diff --git a/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml b/docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml similarity index 100% rename from kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml rename to docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml diff --git a/kubernetes/dolphinscheduler/values.yaml b/docker/kubernetes/dolphinscheduler/values.yaml similarity index 100% rename from kubernetes/dolphinscheduler/values.yaml rename to docker/kubernetes/dolphinscheduler/values.yaml diff --git a/dockerfile/startup.sh b/dockerfile/startup.sh index 30439c2321..af3c456116 100644 --- a/dockerfile/startup.sh +++ b/dockerfile/startup.sh @@ -70,10 +70,10 @@ initZK() { while ! nc -z ${line%:*} ${line#*:}; do counter=$((counter+1)) if [ $counter == 30 ]; then - log "Error: Couldn't connect to zookeeper." + echo "Error: Couldn't connect to zookeeper." exit 1 fi - log "Trying to connect to zookeeper at ${line}. Attempt $counter." + echo "Trying to connect to zookeeper at ${line}. Attempt $counter." sleep 5 done done From 9f32f2db66a5004da2aaf62aa62aa15427333fba Mon Sep 17 00:00:00 2001 From: qiaozhanwei Date: Sun, 26 Apr 2020 15:58:05 +0800 Subject: [PATCH 49/55] #2499 bug fix (#2527) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * dispatch task fail will set task status failed * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，task status statistics and process status statistics bug fix (#2357) 2，worker group bug fix * 1，task status statistics and process status statistics bug fix (#2357) 2，worker group bug fix * 1，task status statistics and process status statistics bug fix (#2357) 2，worker group bug fix * 1，task status statistics and process status statistics bug fix (#2357) 2，worker group bug fix * send mail error, #2466 bug fix * send mail error, #2466 bug fix * send mail error, #2466 bug fix * send mail error, #2466 bug fix * #2486 bug fix * host and workergroup compatible * EnterpriseWeChatUtils modify * EnterpriseWeChatUtils modify * EnterpriseWeChatUtils modify * #2499 bug fix * add comment * revert comment * revert comment * #2499 buf fix * #2499 bug fix * #2499 bug fix * #2499 bug fix * #2499 bug fix * #2499 bug fix * #2499 bug fix Co-authored-by: qiaozhanwei --- .../dolphinscheduler/service/process/ProcessService.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java index c69ea34c5c..26462d2337 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java @@ -119,6 +119,10 @@ public class ProcessService { logger.info("there is not enough thread for this command: {}", command); return setWaitingThreadProcess(command, processInstance); } + if (processInstance.getCommandType().equals(CommandType.RECOVER_TOLERANCE_FAULT_PROCESS)){ + delCommandByid(command.getId()); + return null; + } processInstance.setCommandType(command.getCommandType()); processInstance.addHistoryCmd(command.getCommandType()); saveProcessInstance(processInstance); From 1185cdafb69ccf8978c2e24e453c76ea49aff3dc Mon Sep 17 00:00:00 2001 From: zhangchunyang Date: Sun, 26 Apr 2020 18:50:38 +0800 Subject: [PATCH 50/55] update version 2.0.0 backup to 1.3.0 (#2531) * Ambari plugin development for the dolphin scheduler in version 2.0.0 is complete * Update the Ambari plugin of DS usage documentation * delete readme whic is not completed * update version 2.0.0 backup to 1.3.0 * update ambari plugin version to 1.3.0 Co-authored-by: zhangchunyang Co-authored-by: zhangchunyang <18910529250@163.com> --- .../common-services/DOLPHIN/1.3.0/alerts.json | 164 +++++ .../1.3.0/configuration/dolphin-alert.xml | 143 ++++ .../configuration/dolphin-application-api.xml | 87 +++ .../1.3.0/configuration/dolphin-common.xml | 158 +++++ .../configuration/dolphin-datasource.xml | 467 +++++++++++++ .../1.3.0/configuration/dolphin-env.xml | 123 ++++ .../1.3.0/configuration/dolphin-master.xml | 88 +++ .../1.3.0/configuration/dolphin-quartz.xml | 126 ++++ .../1.3.0/configuration/dolphin-worker.xml | 76 ++ .../1.3.0/configuration/dolphin-zookeeper.xml | 84 +++ .../DOLPHIN/1.3.0/metainfo.xml | 137 ++++ .../alerts/alert_dolphin_scheduler_status.py | 124 ++++ .../package/scripts/dolphin_alert_service.py | 61 ++ .../package/scripts/dolphin_api_service.py | 70 ++ .../1.3.0/package/scripts/dolphin_env.py | 123 ++++ .../package/scripts/dolphin_logger_service.py | 61 ++ .../package/scripts/dolphin_master_service.py | 61 ++ .../package/scripts/dolphin_worker_service.py | 60 ++ .../DOLPHIN/1.3.0/package/scripts/params.py | 154 ++++ .../1.3.0/package/scripts/service_check.py | 31 + .../1.3.0/package/scripts/status_params.py | 23 + .../package/templates/alert.properties.j2 | 20 + .../templates/application-api.properties.j2 | 20 + .../package/templates/common.properties.j2 | 20 + .../templates/datasource.properties.j2 | 20 + .../package/templates/dolphin-daemon.sh.j2 | 116 +++ .../package/templates/master.properties.j2 | 20 + .../package/templates/quartz.properties.j2 | 20 + .../package/templates/worker.properties.j2 | 20 + .../package/templates/zookeeper.properties.j2 | 20 + .../DOLPHIN/1.3.0/quicklinks/quicklinks.json | 26 + .../DOLPHIN/1.3.0/themes/theme.json | 661 ++++++++++++++++++ ambari_plugin/statcks/DOLPHIN/metainfo.xml | 2 +- 33 files changed, 3385 insertions(+), 1 deletion(-) create mode 100644 ambari_plugin/common-services/DOLPHIN/1.3.0/alerts.json create mode 100644 ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-alert.xml create mode 100644 ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-application-api.xml create mode 100644 ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-common.xml create mode 100644 ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-datasource.xml create mode 100644 ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-env.xml create mode 100644 ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-master.xml create mode 100644 ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-quartz.xml create mode 100644 ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-worker.xml create mode 100644 ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-zookeeper.xml create mode 100644 ambari_plugin/common-services/DOLPHIN/1.3.0/metainfo.xml create mode 100644 ambari_plugin/common-services/DOLPHIN/1.3.0/package/alerts/alert_dolphin_scheduler_status.py create mode 100644 ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_alert_service.py create mode 100644 ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_api_service.py create mode 100644 ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_env.py create mode 100644 ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_logger_service.py create mode 100644 ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_master_service.py create mode 100644 ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_worker_service.py create mode 100644 ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/params.py create mode 100644 ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/service_check.py create mode 100644 ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/status_params.py create mode 100644 ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/alert.properties.j2 create mode 100644 ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/application-api.properties.j2 create mode 100644 ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/common.properties.j2 create mode 100644 ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/datasource.properties.j2 create mode 100644 ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/dolphin-daemon.sh.j2 create mode 100644 ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/master.properties.j2 create mode 100644 ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/quartz.properties.j2 create mode 100644 ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/worker.properties.j2 create mode 100644 ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/zookeeper.properties.j2 create mode 100755 ambari_plugin/common-services/DOLPHIN/1.3.0/quicklinks/quicklinks.json create mode 100644 ambari_plugin/common-services/DOLPHIN/1.3.0/themes/theme.json diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/alerts.json b/ambari_plugin/common-services/DOLPHIN/1.3.0/alerts.json new file mode 100644 index 0000000000..2a9800e59a --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/alerts.json @@ -0,0 +1,164 @@ +{ + "DOLPHIN": { + "service": [], + "DOLPHIN_API": [ + { + "name": "dolphin_api_port_check", + "label": "dolphin_api_port_check", + "description": "dolphin_api_port_check.", + "interval": 10, + "scope": "ANY", + "source": { + "type": "PORT", + "uri": "{{dolphin-application-api/server.port}}", + "default_port": 12345, + "reporting": { + "ok": { + "text": "TCP OK - {0:.3f}s response on port {1}" + }, + "warning": { + "text": "TCP OK - {0:.3f}s response on port {1}", + "value": 1.5 + }, + "critical": { + "text": "Connection failed: {0} to {1}:{2}", + "value": 5.0 + } + } + } + } + ], + "DOLPHIN_MASTER": [ + { + "name": "DOLPHIN_MASTER_CHECK", + "label": "check dolphin scheduler master status", + "description": "", + "interval":10, + "scope": "HOST", + "enabled": true, + "source": { + "type": "SCRIPT", + "path": "DOLPHIN/1.3.0/package/alerts/alert_dolphin_scheduler_status.py", + "parameters": [ + + { + "name": "connection.timeout", + "display_name": "Connection Timeout", + "value": 5.0, + "type": "NUMERIC", + "description": "The maximum time before this alert is considered to be CRITICAL", + "units": "seconds", + "threshold": "CRITICAL" + }, + { + "name": "alertName", + "display_name": "alertName", + "value": "DOLPHIN_MASTER", + "type": "STRING", + "description": "alert name" + } + ] + } + } + ], + "DOLPHIN_WORKER": [ + { + "name": "DOLPHIN_WORKER_CHECK", + "label": "check dolphin scheduler worker status", + "description": "", + "interval":10, + "scope": "HOST", + "enabled": true, + "source": { + "type": "SCRIPT", + "path": "DOLPHIN/1.3.0/package/alerts/alert_dolphin_scheduler_status.py", + "parameters": [ + + { + "name": "connection.timeout", + "display_name": "Connection Timeout", + "value": 5.0, + "type": "NUMERIC", + "description": "The maximum time before this alert is considered to be CRITICAL", + "units": "seconds", + "threshold": "CRITICAL" + }, + { + "name": "alertName", + "display_name": "alertName", + "value": "DOLPHIN_WORKER", + "type": "STRING", + "description": "alert name" + } + ] + } + } + ], + "DOLPHIN_ALERT": [ + { + "name": "DOLPHIN_DOLPHIN_ALERT_CHECK", + "label": "check dolphin scheduler alert status", + "description": "", + "interval":10, + "scope": "HOST", + "enabled": true, + "source": { + "type": "SCRIPT", + "path": "DOLPHIN/1.3.0/package/alerts/alert_dolphin_scheduler_status.py", + "parameters": [ + + { + "name": "connection.timeout", + "display_name": "Connection Timeout", + "value": 5.0, + "type": "NUMERIC", + "description": "The maximum time before this alert is considered to be CRITICAL", + "units": "seconds", + "threshold": "CRITICAL" + }, + { + "name": "alertName", + "display_name": "alertName", + "value": "DOLPHIN_ALERT", + "type": "STRING", + "description": "alert name" + } + ] + } + } + ], + "DOLPHIN_ALERT": [ + { + "name": "DOLPHIN_DOLPHIN_LOGGER_CHECK", + "label": "check dolphin scheduler alert status", + "description": "", + "interval":10, + "scope": "HOST", + "enabled": true, + "source": { + "type": "SCRIPT", + "path": "DOLPHIN/1.3.0/package/alerts/alert_dolphin_scheduler_status.py", + "parameters": [ + + { + "name": "connection.timeout", + "display_name": "Connection Timeout", + "value": 5.0, + "type": "NUMERIC", + "description": "The maximum time before this alert is considered to be CRITICAL", + "units": "seconds", + "threshold": "CRITICAL" + }, + { + "name": "alertName", + "display_name": "alertName", + "value": "DOLPHIN_LOGGER", + "type": "STRING", + "description": "alert name" + } + ] + } + } + ] + } +} \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-alert.xml b/ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-alert.xml new file mode 100644 index 0000000000..5f44a1a4c8 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-alert.xml @@ -0,0 +1,143 @@ + + + + alert.type + EMAIL + alert type is EMAIL/SMS + + + + alert.template + html + alter msg template, default is html template + + + + mail.protocol + SMTP + + + + + mail.server.host + xxx.xxx.com + + + + + mail.server.port + 25 + + int + + + + + + mail.sender + admin + + + + + mail.user + admin + + + + + mail.passwd + 000000 + + PASSWORD + + password + + + + + + mail.smtp.starttls.enable + true + + boolean + + + + + + mail.smtp.ssl.enable + true + + boolean + + + + + + mail.smtp.ssl.trust + xxx.xxx.com + + + + + + enterprise.wechat.enable + false + + + value-list + + + true + + + + false + + + + 1 + + + + + enterprise.wechat.corp.id + wechatId + + + + + enterprise.wechat.secret + secret + + + + + enterprise.wechat.agent.id + agentId + + + + + enterprise.wechat.users + wechatUsers + + + + \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-application-api.xml b/ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-application-api.xml new file mode 100644 index 0000000000..766c0f477d --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-application-api.xml @@ -0,0 +1,87 @@ + + + + server.port + 12345 + + server port + + + int + + + + server.servlet.session.timeout + 7200 + + int + + + + + + server.servlet.context-path + /dolphinscheduler/ + + + + + spring.servlet.multipart.max-file-size + 1024 + + MB + int + + + + + + spring.servlet.multipart.max-request-size + 1024 + + MB + int + + + + + + server.jetty.max-http-post-size + 5000000 + + int + + + + + + spring.messages.encoding + UTF-8 + + + + spring.messages.basename + i18n/messages + + + + security.authentication.type + PASSWORD + + + \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-common.xml b/ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-common.xml new file mode 100644 index 0000000000..439e21188a --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-common.xml @@ -0,0 +1,158 @@ + + + + resource.storage.type + Choose Resource Upload Startup Type + + Resource upload startup type : HDFS,S3,NONE + + NONE + + value-list + + + HDFS + + + + S3 + + + + NONE + + + + 1 + + + + + resource.upload.path + /dolphinscheduler + + resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended + + + + + data.basedir.path + /tmp/dolphinscheduler + + user data local directory path, please make sure the directory exists and have read write permissions + + + + + + hadoop.security.authentication.startup.state + false + + value-list + + + true + + + + false + + + + 1 + + whether kerberos starts + + + java.security.krb5.conf.path + /opt/krb5.conf + + java.security.krb5.conf path + + + + + login.user.keytab.username + hdfs-mycluster@ESZ.COM + + LoginUserFromKeytab user + + + + + login.user.keytab.path + /opt/hdfs.headless.keytab + + LoginUserFromKeytab path + + + + + resource.view.suffixs + txt,log,sh,conf,cfg,py,java,sql,hql,xml,properties + + + + hdfs.root.user + hdfs + + Users who have permission to create directories under the HDFS root path + + + + + fs.defaultFS + hdfs://mycluster:8020 + + HA or single namenode, + If namenode ha needs to copy core-site.xml and hdfs-site.xml to the conf directory， + support s3，for example : s3a://dolphinscheduler + + + + + fs.s3a.endpoint + http://host:9010 + + s3 need，s3 endpoint + + + + + fs.s3a.access.key + A3DXS30FO22544RE + + s3 need，s3 access key + + + + + fs.s3a.secret.key + OloCLq3n+8+sdPHUhJ21XrSxTC+JK + + s3 need，s3 secret key + + + + + kerberos.expire.time + 7 + + + + \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-datasource.xml b/ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-datasource.xml new file mode 100644 index 0000000000..6e50a1b649 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-datasource.xml @@ -0,0 +1,467 @@ + + + + spring.datasource.initialSize + 5 + + Init connection number + + + int + + + + + spring.datasource.minIdle + 5 + + Min connection number + + + int + + + + + spring.datasource.maxActive + 50 + + Max connection number + + + int + + + + + spring.datasource.maxWait + 60000 + + Max wait time for get a connection in milliseconds. + If configuring maxWait, fair locks are enabled by default and concurrency efficiency decreases. + If necessary, unfair locks can be used by configuring the useUnfairLock attribute to true. + + + int + + + + + spring.datasource.timeBetweenEvictionRunsMillis + 60000 + + Milliseconds for check to close free connections + + + int + + + + + spring.datasource.timeBetweenConnectErrorMillis + 60000 + + The Destroy thread detects the connection interval and closes the physical connection in milliseconds + if the connection idle time is greater than or equal to minEvictableIdleTimeMillis. + + + int + + + + + spring.datasource.minEvictableIdleTimeMillis + 300000 + + The longest time a connection remains idle without being evicted, in milliseconds + + + int + + + + + spring.datasource.validationQuery + SELECT 1 + + The SQL used to check whether the connection is valid requires a query statement. + If validation Query is null, testOnBorrow, testOnReturn, and testWhileIdle will not work. + + + + + spring.datasource.validationQueryTimeout + 3 + + int + + + Check whether the connection is valid for timeout, in seconds + + + + + spring.datasource.testWhileIdle + true + + boolean + + + When applying for a connection, + if it is detected that the connection is idle longer than time Between Eviction Runs Millis, + validation Query is performed to check whether the connection is valid + + + + + spring.datasource.testOnBorrow + true + + boolean + + + Execute validation to check if the connection is valid when applying for a connection + + + + + spring.datasource.testOnReturn + false + + boolean + + + Execute validation to check if the connection is valid when the connection is returned + + + + + spring.datasource.defaultAutoCommit + true + + boolean + + + + + + + spring.datasource.keepAlive + false + + boolean + + + + + + + + spring.datasource.poolPreparedStatements + true + + boolean + + + Open PSCache, specify count PSCache for every connection + + + + + spring.datasource.maxPoolPreparedStatementPerConnectionSize + 20 + + int + + + + + + spring.datasource.spring.datasource.filters + stat,wall,log4j + + + + + spring.datasource.connectionProperties + druid.stat.mergeSql=true;druid.stat.slowSqlMillis=5000 + + + + + + mybatis-plus.mapper-locations + classpath*:/org.apache.dolphinscheduler.dao.mapper/*.xml + + + + + mybatis-plus.typeEnumsPackage + org.apache.dolphinscheduler.*.enums + + + + + mybatis-plus.typeAliasesPackage + org.apache.dolphinscheduler.dao.entity + + Entity scan, where multiple packages are separated by a comma or semicolon + + + + + mybatis-plus.global-config.db-config.id-type + AUTO + + value-list + + + AUTO + + + + INPUT + + + + ID_WORKER + + + + UUID + + + + 1 + + + Primary key type AUTO:" database ID AUTO ", + INPUT:" user INPUT ID", + ID_WORKER:" global unique ID (numeric type unique ID)", + UUID:" global unique ID UUID"; + + + + + mybatis-plus.global-config.db-config.field-strategy + NOT_NULL + + value-list + + + IGNORED + + + + NOT_NULL + + + + NOT_EMPTY + + + + 1 + + + Field policy IGNORED:" ignore judgment ", + NOT_NULL:" not NULL judgment "), + NOT_EMPTY:" not NULL judgment" + + + + + mybatis-plus.global-config.db-config.column-underline + true + + boolean + + + + + + mybatis-plus.global-config.db-config.logic-delete-value + 1 + + int + + + + + + mybatis-plus.global-config.db-config.logic-not-delete-value + 0 + + int + + + + + + mybatis-plus.global-config.db-config.banner + true + + boolean + + + + + + + mybatis-plus.configuration.map-underscore-to-camel-case + true + + boolean + + + + + + mybatis-plus.configuration.cache-enabled + false + + boolean + + + + + + mybatis-plus.configuration.call-setters-on-nulls + true + + boolean + + + + + + mybatis-plus.configuration.jdbc-type-for-null + null + + + + + master.exec.threads + 100 + + int + + + + + + master.exec.task.num + 20 + + int + + + + + + master.heartbeat.interval + 10 + + int + + + + + + master.task.commit.retryTimes + 5 + + int + + + + + + master.task.commit.interval + 1000 + + int + + + + + + master.max.cpuload.avg + 100 + + int + + + + + + master.reserved.memory + 0.1 + + float + + + + + + worker.exec.threads + 100 + + int + + + + + + worker.heartbeat.interval + 10 + + int + + + + + + worker.fetch.task.num + 3 + + int + + + + + + worker.max.cpuload.avg + 100 + + int + + + + + + worker.reserved.memory + 0.1 + + float + + + + + + \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-env.xml b/ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-env.xml new file mode 100644 index 0000000000..8e14716d05 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-env.xml @@ -0,0 +1,123 @@ + + + + dolphin.database.type + mysql + Dolphin Scheduler DataBase Type Which Is Select + Dolphin Database Type + + value-list + + + mysql + + + + postgresql + + + + 1 + + + + + + dolphin.database.host + + Dolphin Database Host + + + + + dolphin.database.port + + Dolphin Database Port + + + + + dolphin.database.username + + Dolphin Database Username + + + + + dolphin.database.password + + Dolphin Database Password + PASSWORD + + password + + + + + + dolphin.user + + Which user to install and admin dolphin scheduler + Deploy User + + + + dolphin.group + + Which user to install and admin dolphin scheduler + Deploy Group + + + + + dolphinscheduler-env-content + Dolphinscheduler Env template + This is the jinja template for dolphinscheduler.env.sh file + # +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +export HADOOP_HOME=/opt/soft/hadoop +export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop +export SPARK_HOME1=/opt/soft/spark1 +export SPARK_HOME2=/opt/soft/spark2 +export PYTHON_HOME=/opt/soft/python +export JAVA_HOME=/opt/soft/java +export HIVE_HOME=/opt/soft/hive +export FLINK_HOME=/opt/soft/flink + + content + false + false + + + + \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-master.xml b/ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-master.xml new file mode 100644 index 0000000000..c8eec047fc --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-master.xml @@ -0,0 +1,88 @@ + + + + master.exec.threads + 100 + + int + + master execute thread num + + + + master.exec.task.num + 20 + + int + + master execute task number in parallel + + + + master.heartbeat.interval + 10 + + int + + master heartbeat interval + + + + master.task.commit.retryTimes + 5 + + int + + master commit task retry times + + + + master.task.commit.interval + 1000 + + int + + master commit task interval + + + + master.max.cpuload.avg + 100 + + int + + only less than cpu avg load, master server can work. default value : the number of cpu cores * 2 + + + + master.reserved.memory + 0.3 + only larger than reserved memory, master server can work. default value : physical memory * 1/10, unit is G. + + + + + master.listen.port + 5678 + + int + + master listen port + + + \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-quartz.xml b/ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-quartz.xml new file mode 100644 index 0000000000..7a0c68b051 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-quartz.xml @@ -0,0 +1,126 @@ + + + + org.quartz.scheduler.instanceName + DolphinScheduler + + + + + org.quartz.scheduler.instanceId + AUTO + + + + org.quartz.scheduler.makeSchedulerThreadDaemon + true + + boolean + + + + + org.quartz.jobStore.useProperties + false + + boolean + + + + + org.quartz.threadPool.class + org.quartz.simpl.SimpleThreadPool + + + + org.quartz.threadPool.makeThreadsDaemons + true + + boolean + + + + + org.quartz.threadPool.threadCount + 25 + + int + + + + + org.quartz.threadPool.threadPriority + 5 + + int + + + + + org.quartz.jobStore.class + org.quartz.impl.jdbcjobstore.JobStoreTX + + + + org.quartz.jobStore.tablePrefix + QRTZ_ + + + + org.quartz.jobStore.isClustered + true + + boolean + + + + + org.quartz.jobStore.misfireThreshold + 60000 + + int + + + + + org.quartz.jobStore.clusterCheckinInterval + 5000 + + int + + + + + org.quartz.jobStore.acquireTriggersWithinLock + true + + boolean + + + + + org.quartz.jobStore.dataSource + myDs + + + + org.quartz.dataSource.myDs.connectionProvider.class + org.apache.dolphinscheduler.service.quartz.DruidConnectionProvider + + + \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-worker.xml b/ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-worker.xml new file mode 100644 index 0000000000..97beade1bc --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-worker.xml @@ -0,0 +1,76 @@ + + + + worker.exec.threads + 100 + + int + + worker execute thread num + + + + worker.heartbeat.interval + 10 + + int + + worker heartbeat interval + + + + worker.fetch.task.num + 3 + + int + + submit the number of tasks at a time + + + + worker.max.cpuload.avg + 100 + + int + + only less than cpu avg load, worker server can work. default value : the number of cpu cores * 2 + + + + worker.reserved.memory + 0.3 + only larger than reserved memory, worker server can work. default value : physical memory * 1/10, unit is G. + + + + + worker.listen.port + 1234 + + int + + worker listen port + + + + worker.group + default + default worker group + + + \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-zookeeper.xml b/ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-zookeeper.xml new file mode 100644 index 0000000000..5882162254 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-zookeeper.xml @@ -0,0 +1,84 @@ + + + + dolphinscheduler.queue.impl + zookeeper + + Task queue implementation, default "zookeeper" + + + + + zookeeper.dolphinscheduler.root + /dolphinscheduler + + dolphinscheduler root directory + + + + + zookeeper.session.timeout + 300 + + int + + + + + + + zookeeper.connection.timeout + 300 + + int + + + + + + + zookeeper.retry.base.sleep + 100 + + int + + + + + + + zookeeper.retry.max.sleep + 30000 + + int + + + + + + + zookeeper.retry.maxtime + 5 + + int + + + + + + \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/metainfo.xml b/ambari_plugin/common-services/DOLPHIN/1.3.0/metainfo.xml new file mode 100644 index 0000000000..a559085f03 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/metainfo.xml @@ -0,0 +1,137 @@ + + + + 2.0 + + + DOLPHIN + Dolphin Scheduler + 分布式易扩展的可视化DAG工作流任务调度系统 + 1.3.0 + + + DOLPHIN_MASTER + DS Master + MASTER + 1+ + + + PYTHON + 600 + + + + + DOLPHIN_LOGGER + DS Logger + SLAVE + 1+ + + + PYTHON + 600 + + + + + DOLPHIN_WORKER + DS Worker + SLAVE + 1+ + + + DOLPHIN/DOLPHIN_LOGGER + host + + true + + + + + + PYTHON + 600 + + + + + DOLPHIN_ALERT + DS Alert + SLAVE + 1 + + + PYTHON + 600 + + + + + DOLPHIN_API + DS_Api + SLAVE + 1 + + + PYTHON + 600 + + + + + + ZOOKEEPER + + + + + any + + + apache-dolphinscheduler-incubating-1.3.0* + + + + + + + dolphin-alert + dolphin-app-api + dolphin-app-dao + dolphin-common + dolphin-env + dolphin-quartz + + + + + theme.json + true + + + + quicklinks + + + quicklinks.json + true + + + + + diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/package/alerts/alert_dolphin_scheduler_status.py b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/alerts/alert_dolphin_scheduler_status.py new file mode 100644 index 0000000000..87cc7b453b --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/alerts/alert_dolphin_scheduler_status.py @@ -0,0 +1,124 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import socket +import urllib2 +import os +import logging +import ambari_simplejson as json +from resource_management.libraries.script.script import Script +import sys +reload(sys) +sys.setdefaultencoding('utf-8') + +logger = logging.getLogger('ambari_alerts') + +config = Script.get_config() + + +def get_tokens(): + """ + Returns a tuple of tokens in the format {{site/property}} that will be used + to build the dictionary passed into execute + + :rtype tuple + """ + +def get_info(url, connection_timeout): + response = None + + try: + response = urllib2.urlopen(url, timeout=connection_timeout) + json_data = response.read() + return json_data + finally: + if response is not None: + try: + response.close() + except: + pass + + +def execute(configurations={}, parameters={}, host_name=None): + """ + Returns a tuple containing the result code and a pre-formatted result label + + Keyword arguments: + configurations : a mapping of configuration key to value + parameters : a mapping of script parameter key to value + host_name : the name of this host where the alert is running + + :type configurations dict + :type parameters dict + :type host_name str + """ + + alert_name = parameters['alertName'] + + dolphin_pidfile_dir = "/opt/soft/run/dolphinscheduler" + + pid = "0" + + + from resource_management.core import sudo + + is_running = True + pid_file_path = "" + if alert_name == 'DOLPHIN_MASTER': + pid_file_path = dolphin_pidfile_dir + "/master-server.pid" + elif alert_name == 'DOLPHIN_WORKER': + pid_file_path = dolphin_pidfile_dir + "/worker-server.pid" + elif alert_name == 'DOLPHIN_ALERT': + pid_file_path = dolphin_pidfile_dir + "/alert-server.pid" + elif alert_name == 'DOLPHIN_LOGGER': + pid_file_path = dolphin_pidfile_dir + "/logger-server.pid" + elif alert_name == 'DOLPHIN_API': + pid_file_path = dolphin_pidfile_dir + "/api-server.pid" + + if not pid_file_path or not os.path.isfile(pid_file_path): + is_running = False + + try: + pid = int(sudo.read_file(pid_file_path)) + except: + is_running = False + + try: + # Kill will not actually kill the process + # From the doc: + # If sig is 0, then no signal is sent, but error checking is still + # performed; this can be used to check for the existence of a + # process ID or process group ID. + sudo.kill(pid, 0) + except OSError: + is_running = False + + if host_name is None: + host_name = socket.getfqdn() + + if not is_running: + result_code = "CRITICAL" + else: + result_code = "OK" + + label = "The comment {0} of DOLPHIN_SCHEDULER on {1} is {2}".format(alert_name, host_name, result_code) + + return ((result_code, [label])) + +if __name__ == "__main__": + pass diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_alert_service.py b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_alert_service.py new file mode 100644 index 0000000000..62255a3432 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_alert_service.py @@ -0,0 +1,61 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import time +from resource_management import * + +from dolphin_env import dolphin_env + + +class DolphinAlertService(Script): + def install(self, env): + import params + env.set_params(params) + self.install_packages(env) + Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True) + + def configure(self, env): + import params + params.pika_slave = True + env.set_params(params) + + dolphin_env() + + def start(self, env): + import params + env.set_params(params) + self.configure(env) + no_op_test = format("ls {dolphin_pidfile_dir}/alert-server.pid >/dev/null 2>&1 && ps `cat {dolphin_pidfile_dir}/alert-server.pid` | grep `cat {dolphin_pidfile_dir}/alert-server.pid` >/dev/null 2>&1") + + start_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh start alert-server") + Execute(start_cmd, user=params.dolphin_user, not_if=no_op_test) + + def stop(self, env): + import params + env.set_params(params) + stop_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh stop alert-server") + Execute(stop_cmd, user=params.dolphin_user) + time.sleep(5) + + def status(self, env): + import status_params + env.set_params(status_params) + check_process_status(status_params.dolphin_run_dir + "alert-server.pid") + + +if __name__ == "__main__": + DolphinAlertService().execute() diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_api_service.py b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_api_service.py new file mode 100644 index 0000000000..bdc18fb602 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_api_service.py @@ -0,0 +1,70 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import time +from resource_management import * + +from dolphin_env import dolphin_env + + +class DolphinApiService(Script): + def install(self, env): + import params + env.set_params(params) + self.install_packages(env) + Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True) + + def configure(self, env): + import params + params.pika_slave = True + env.set_params(params) + + dolphin_env() + + def start(self, env): + import params + env.set_params(params) + self.configure(env) + + #init + init_cmd=format("sh " + params.dolphin_home + "/script/create-dolphinscheduler.sh") + Execute(init_cmd, user=params.dolphin_user) + + #upgrade + upgrade_cmd=format("sh " + params.dolphin_home + "/script/upgrade-dolphinscheduler.sh") + Execute(upgrade_cmd, user=params.dolphin_user) + + no_op_test = format("ls {dolphin_pidfile_dir}/api-server.pid >/dev/null 2>&1 && ps `cat {dolphin_pidfile_dir}/api-server.pid` | grep `cat {dolphin_pidfile_dir}/api-server.pid` >/dev/null 2>&1") + + start_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh start api-server") + Execute(start_cmd, user=params.dolphin_user, not_if=no_op_test) + + def stop(self, env): + import params + env.set_params(params) + stop_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh stop api-server") + Execute(stop_cmd, user=params.dolphin_user) + time.sleep(5) + + def status(self, env): + import status_params + env.set_params(status_params) + check_process_status(status_params.dolphin_run_dir + "api-server.pid") + + +if __name__ == "__main__": + DolphinApiService().execute() diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_env.py b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_env.py new file mode 100644 index 0000000000..1661d76c75 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_env.py @@ -0,0 +1,123 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * + + +def dolphin_env(): + import params + + Directory(params.dolphin_pidfile_dir, + mode=0777, + owner=params.dolphin_user, + group=params.dolphin_group, + create_parents=True + ) + Directory(params.dolphin_log_dir, + mode=0777, + owner=params.dolphin_user, + group=params.dolphin_group, + create_parents=True + ) + Directory(params.dolphin_conf_dir, + mode=0777, + owner=params.dolphin_user, + group=params.dolphin_group, + create_parents=True + ) + + Directory(params.dolphin_common_map['data.basedir.path'], + mode=0777, + owner=params.dolphin_user, + group=params.dolphin_group, + create_parents=True + ) + + + File(format(params.dolphin_env_path), + mode=0777, + content=InlineTemplate(params.dolphin_env_content), + owner=params.dolphin_user, + group=params.dolphin_group + ) + + + File(format(params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh"), + mode=0755, + content=Template("dolphin-daemon.sh.j2"), + owner=params.dolphin_user, + group=params.dolphin_group + ) + + File(format(params.dolphin_conf_dir + "/master.properties"), + mode=0755, + content=Template("master.properties.j2"), + owner=params.dolphin_user, + group=params.dolphin_group + ) + + File(format(params.dolphin_conf_dir + "/worker.properties"), + mode=0755, + content=Template("worker.properties.j2"), + owner=params.dolphin_user, + group=params.dolphin_group + ) + + + File(format(params.dolphin_conf_dir + "/alert.properties"), + mode=0755, + content=Template("alert.properties.j2"), + owner=params.dolphin_user, + group=params.dolphin_group + ) + + File(format(params.dolphin_conf_dir + "/datasource.properties"), + mode=0755, + content=Template("datasource.properties.j2"), + owner=params.dolphin_user, + group=params.dolphin_group + ) + + File(format(params.dolphin_conf_dir + "/application-api.properties"), + mode=0755, + content=Template("application-api.properties.j2"), + owner=params.dolphin_user, + group=params.dolphin_group + ) + + File(format(params.dolphin_conf_dir + "/common.properties"), + mode=0755, + content=Template("common.properties.j2"), + owner=params.dolphin_user, + group=params.dolphin_group + ) + + File(format(params.dolphin_conf_dir + "/quartz.properties"), + mode=0755, + content=Template("quartz.properties.j2"), + owner=params.dolphin_user, + group=params.dolphin_group + ) + + File(format(params.dolphin_conf_dir + "/zookeeper.properties"), + mode=0755, + content=Template("zookeeper.properties.j2"), + owner=params.dolphin_user, + group=params.dolphin_group + ) diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_logger_service.py b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_logger_service.py new file mode 100644 index 0000000000..f1c19bd66f --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_logger_service.py @@ -0,0 +1,61 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import time +from resource_management import * + +from dolphin_env import dolphin_env + + +class DolphinLoggerService(Script): + def install(self, env): + import params + env.set_params(params) + self.install_packages(env) + Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True) + + def configure(self, env): + import params + params.pika_slave = True + env.set_params(params) + + dolphin_env() + + def start(self, env): + import params + env.set_params(params) + self.configure(env) + no_op_test = format("ls {dolphin_pidfile_dir}/logger-server.pid >/dev/null 2>&1 && ps `cat {dolphin_pidfile_dir}/logger-server.pid` | grep `cat {dolphin_pidfile_dir}/logger-server.pid` >/dev/null 2>&1") + + start_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh start logger-server") + Execute(start_cmd, user=params.dolphin_user, not_if=no_op_test) + + def stop(self, env): + import params + env.set_params(params) + stop_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh stop logger-server") + Execute(stop_cmd, user=params.dolphin_user) + time.sleep(5) + + def status(self, env): + import status_params + env.set_params(status_params) + check_process_status(status_params.dolphin_run_dir + "logger-server.pid") + + +if __name__ == "__main__": + DolphinLoggerService().execute() diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_master_service.py b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_master_service.py new file mode 100644 index 0000000000..6ee7ecfcf3 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_master_service.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import time +from resource_management import * + +from dolphin_env import dolphin_env + + +class DolphinMasterService(Script): + def install(self, env): + import params + env.set_params(params) + self.install_packages(env) + Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True) + + def configure(self, env): + import params + params.pika_slave = True + env.set_params(params) + + dolphin_env() + + def start(self, env): + import params + env.set_params(params) + self.configure(env) + no_op_test = format("ls {dolphin_pidfile_dir}/master-server.pid >/dev/null 2>&1 && ps `cat {dolphin_pidfile_dir}/master-server.pid` | grep `cat {dolphin_pidfile_dir}/master-server.pid` >/dev/null 2>&1") + start_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh start master-server") + Execute(start_cmd, user=params.dolphin_user, not_if=no_op_test) + + def stop(self, env): + import params + env.set_params(params) + stop_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh stop master-server") + Execute(stop_cmd, user=params.dolphin_user) + time.sleep(5) + + def status(self, env): + import status_params + env.set_params(status_params) + check_process_status(status_params.dolphin_run_dir + "master-server.pid") + + +if __name__ == "__main__": + DolphinMasterService().execute() diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_worker_service.py b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_worker_service.py new file mode 100644 index 0000000000..2d145ee730 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_worker_service.py @@ -0,0 +1,60 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import time +from resource_management import * + +from dolphin_env import dolphin_env + + +class DolphinWorkerService(Script): + def install(self, env): + import params + env.set_params(params) + self.install_packages(env) + Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True) + + def configure(self, env): + import params + params.pika_slave = True + env.set_params(params) + + dolphin_env() + + def start(self, env): + import params + env.set_params(params) + self.configure(env) + no_op_test = format("ls {dolphin_pidfile_dir}/worker-server.pid >/dev/null 2>&1 && ps `cat {dolphin_pidfile_dir}/worker-server.pid` | grep `cat {dolphin_pidfile_dir}/worker-server.pid` >/dev/null 2>&1") + start_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh start worker-server") + Execute(start_cmd, user=params.dolphin_user, not_if=no_op_test) + + def stop(self, env): + import params + env.set_params(params) + stop_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh stop worker-server") + Execute(stop_cmd, user=params.dolphin_user) + time.sleep(5) + + def status(self, env): + import status_params + env.set_params(status_params) + check_process_status(status_params.dolphin_run_dir + "worker-server.pid") + + +if __name__ == "__main__": + DolphinWorkerService().execute() diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/params.py b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/params.py new file mode 100644 index 0000000000..b09b2589f4 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/params.py @@ -0,0 +1,154 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + + +import sys +from resource_management import * +from resource_management.core.logger import Logger +from resource_management.libraries.functions import default + +Logger.initialize_logger() +reload(sys) +sys.setdefaultencoding('utf-8') + +# server configurations +config = Script.get_config() + +# conf_dir = "/etc/" +dolphin_home = "/opt/soft/dolphinscheduler" +dolphin_conf_dir = dolphin_home + "/conf" +dolphin_log_dir = dolphin_home + "/logs" +dolphin_bin_dir = dolphin_home + "/bin" +dolphin_lib_jars = dolphin_home + "/lib/*" +dolphin_pidfile_dir = "/opt/soft/run/dolphinscheduler" + +rmHosts = default("/clusterHostInfo/rm_host", []) + +# dolphin-env +dolphin_env_map = {} +dolphin_env_map.update(config['configurations']['dolphin-env']) + +# which user to install and admin dolphin scheduler +dolphin_user = dolphin_env_map['dolphin.user'] +dolphin_group = dolphin_env_map['dolphin.group'] + +# .dolphinscheduler_env.sh +dolphin_env_path = dolphin_conf_dir + '/env/dolphinscheduler_env.sh' +dolphin_env_content = dolphin_env_map['dolphinscheduler-env-content'] + +# database config +dolphin_database_config = {} +dolphin_database_config['dolphin_database_type'] = dolphin_env_map['dolphin.database.type'] +dolphin_database_config['dolphin_database_username'] = dolphin_env_map['dolphin.database.username'] +dolphin_database_config['dolphin_database_password'] = dolphin_env_map['dolphin.database.password'] +if 'mysql' == dolphin_database_config['dolphin_database_type']: + dolphin_database_config['dolphin_database_driver'] = 'com.mysql.jdbc.Driver' + dolphin_database_config['driverDelegateClass'] = 'org.quartz.impl.jdbcjobstore.StdJDBCDelegate' + dolphin_database_config['dolphin_database_url'] = 'jdbc:mysql://' + dolphin_env_map['dolphin.database.host'] \ + + ':' + dolphin_env_map['dolphin.database.port'] \ + + '/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8' +else: + dolphin_database_config['dolphin_database_driver'] = 'org.postgresql.Driver' + dolphin_database_config['driverDelegateClass'] = 'org.quartz.impl.jdbcjobstore.PostgreSQLDelegate' + dolphin_database_config['dolphin_database_url'] = 'jdbc:postgresql://' + dolphin_env_map['dolphin.database.host'] \ + + ':' + dolphin_env_map['dolphin.database.port'] \ + + '/dolphinscheduler' + + + + + +# application-alert.properties +dolphin_alert_map = {} +wechat_push_url = 'https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=$token' +wechat_token_url = 'https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid=$corpId&corpsecret=$secret' +wechat_team_send_msg = '{\"toparty\":\"$toParty\",\"agentid\":\"$agentId\",\"msgtype\":\"text\",\"text\":{\"content\":\"$msg\"},\"safe\":\"0\"}' +wechat_user_send_msg = '{\"touser\":\"$toUser\",\"agentid\":\"$agentId\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"$msg\"}}' + +dolphin_alert_config_map = config['configurations']['dolphin-alert'] + +if dolphin_alert_config_map['enterprise.wechat.enable']: + dolphin_alert_map['enterprise.wechat.push.ur'] = wechat_push_url + dolphin_alert_map['enterprise.wechat.token.url'] = wechat_token_url + dolphin_alert_map['enterprise.wechat.team.send.msg'] = wechat_team_send_msg + dolphin_alert_map['enterprise.wechat.user.send.msg'] = wechat_user_send_msg + +dolphin_alert_map.update(dolphin_alert_config_map) + + + +# application-api.properties +dolphin_app_api_map = {} +dolphin_app_api_map.update(config['configurations']['dolphin-application-api']) + + +# common.properties +dolphin_common_map = {} + +if 'yarn-site' in config['configurations'] and \ + 'yarn.resourcemanager.webapp.address' in config['configurations']['yarn-site']: + yarn_resourcemanager_webapp_address = config['configurations']['yarn-site']['yarn.resourcemanager.webapp.address'] + yarn_application_status_address = 'http://' + yarn_resourcemanager_webapp_address + '/ws/v1/cluster/apps/%s' + dolphin_common_map['yarn.application.status.address'] = yarn_application_status_address + +rmHosts = default("/clusterHostInfo/rm_host", []) +if len(rmHosts) > 1: + dolphin_common_map['yarn.resourcemanager.ha.rm.ids'] = ','.join(rmHosts) +else: + dolphin_common_map['yarn.resourcemanager.ha.rm.ids'] = '' + +dolphin_common_map_tmp = config['configurations']['dolphin-common'] +data_basedir_path = dolphin_common_map_tmp['data.basedir.path'] +process_exec_basepath = data_basedir_path + '/exec' +data_download_basedir_path = data_basedir_path + '/download' +dolphin_common_map['process.exec.basepath'] = process_exec_basepath +dolphin_common_map['data.download.basedir.path'] = data_download_basedir_path +dolphin_common_map['dolphinscheduler.env.path'] = dolphin_env_path +dolphin_common_map.update(config['configurations']['dolphin-common']) + +# datasource.properties +dolphin_datasource_map = {} +dolphin_datasource_map['spring.datasource.type'] = 'com.alibaba.druid.pool.DruidDataSource' +dolphin_datasource_map['spring.datasource.driver-class-name'] = dolphin_database_config['dolphin_database_driver'] +dolphin_datasource_map['spring.datasource.url'] = dolphin_database_config['dolphin_database_url'] +dolphin_datasource_map['spring.datasource.username'] = dolphin_database_config['dolphin_database_username'] +dolphin_datasource_map['spring.datasource.password'] = dolphin_database_config['dolphin_database_password'] +dolphin_datasource_map.update(config['configurations']['dolphin-datasource']) + +# master.properties +dolphin_master_map = config['configurations']['dolphin-master'] + +# quartz.properties +dolphin_quartz_map = {} +dolphin_quartz_map['org.quartz.jobStore.driverDelegateClass'] = dolphin_database_config['driverDelegateClass'] +dolphin_quartz_map.update(config['configurations']['dolphin-quartz']) + +# worker.properties +dolphin_worker_map = config['configurations']['dolphin-worker'] + +# zookeeper.properties +dolphin_zookeeper_map={} +zookeeperHosts = default("/clusterHostInfo/zookeeper_hosts", []) +if len(zookeeperHosts) > 0 and "clientPort" in config['configurations']['zoo.cfg']: + clientPort = config['configurations']['zoo.cfg']['clientPort'] + zookeeperPort = ":" + clientPort + "," + dolphin_zookeeper_map['zookeeper.quorum'] = zookeeperPort.join(zookeeperHosts) + ":" + clientPort +dolphin_zookeeper_map.update(config['configurations']['dolphin-zookeeper']) + + + diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/service_check.py b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/service_check.py new file mode 100644 index 0000000000..0e12f69932 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/service_check.py @@ -0,0 +1,31 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * +from resource_management.libraries.functions import get_unique_id_and_date + +class ServiceCheck(Script): + def service_check(self, env): + import params + #env.set_params(params) + + # Execute(format("which pika_server")) + +if __name__ == "__main__": + ServiceCheck().execute() diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/status_params.py b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/status_params.py new file mode 100644 index 0000000000..24b2c8b1bc --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/status_params.py @@ -0,0 +1,23 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from resource_management import * + +config = Script.get_config() + +dolphin_run_dir = "/opt/soft/run/dolphinscheduler/" diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/alert.properties.j2 b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/alert.properties.j2 new file mode 100644 index 0000000000..73840b8c18 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/alert.properties.j2 @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{% for key, value in dolphin_alert_map.iteritems() -%} + {{key}}={{value}} +{% endfor %} \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/application-api.properties.j2 b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/application-api.properties.j2 new file mode 100644 index 0000000000..70118003b9 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/application-api.properties.j2 @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{% for key, value in dolphin_app_api_map.iteritems() -%} + {{key}}={{value}} +{% endfor %} \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/common.properties.j2 b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/common.properties.j2 new file mode 100644 index 0000000000..2220c4effa --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/common.properties.j2 @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{% for key, value in dolphin_common_map.iteritems() -%} + {{key}}={{value}} +{% endfor %} \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/datasource.properties.j2 b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/datasource.properties.j2 new file mode 100644 index 0000000000..40aed83543 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/datasource.properties.j2 @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{% for key, value in dolphin_datasource_map.iteritems() -%} + {{key}}={{value}} +{% endfor %} \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/dolphin-daemon.sh.j2 b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/dolphin-daemon.sh.j2 new file mode 100644 index 0000000000..0802b74750 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/dolphin-daemon.sh.j2 @@ -0,0 +1,116 @@ +#!/bin/sh +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +usage="Usage: dolphinscheduler-daemon.sh (start|stop) " + +# if no args specified, show usage +if [ $# -le 1 ]; then + echo $usage + exit 1 +fi + +startStop=$1 +shift +command=$1 +shift + +echo "Begin $startStop $command......" + +BIN_DIR=`dirname $0` +BIN_DIR=`cd "$BIN_DIR"; pwd` +DOLPHINSCHEDULER_HOME=$BIN_DIR/.. + +export HOSTNAME=`hostname` + +DOLPHINSCHEDULER_LIB_JARS={{dolphin_lib_jars}} + +DOLPHINSCHEDULER_OPTS="-server -Xmx16g -Xms4g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70" +STOP_TIMEOUT=5 + +log={{dolphin_log_dir}}/dolphinscheduler-$command-$HOSTNAME.out +pid={{dolphin_pidfile_dir}}/$command.pid + +cd $DOLPHINSCHEDULER_HOME + +if [ "$command" = "api-server" ]; then + LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/logback-api.xml -Dspring.profiles.active=api" + CLASS=org.apache.dolphinscheduler.api.ApiApplicationServer +elif [ "$command" = "master-server" ]; then + LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/logback-master.xml -Ddruid.mysql.usePingMethod=false" + CLASS=org.apache.dolphinscheduler.server.master.MasterServer +elif [ "$command" = "worker-server" ]; then + LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/logback-worker.xml -Ddruid.mysql.usePingMethod=false" + CLASS=org.apache.dolphinscheduler.server.worker.WorkerServer +elif [ "$command" = "alert-server" ]; then + LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/logback-alert.xml" + CLASS=org.apache.dolphinscheduler.alert.AlertServer +elif [ "$command" = "logger-server" ]; then + CLASS=org.apache.dolphinscheduler.server.log.LoggerServer +else + echo "Error: No command named \`$command' was found." + exit 1 +fi + +case $startStop in + (start) + + if [ -f $pid ]; then + if kill -0 `cat $pid` > /dev/null 2>&1; then + echo $command running as process `cat $pid`. Stop it first. + exit 1 + fi + fi + + echo starting $command, logging to $log + + exec_command="$LOG_FILE $DOLPHINSCHEDULER_OPTS -classpath {{dolphin_conf_dir}}:{{dolphin_lib_jars}} $CLASS" + + echo "nohup java $exec_command > $log 2>&1 < /dev/null &" + nohup java $exec_command > $log 2>&1 < /dev/null & + echo $! > $pid + ;; + + (stop) + + if [ -f $pid ]; then + TARGET_PID=`cat $pid` + if kill -0 $TARGET_PID > /dev/null 2>&1; then + echo stopping $command + kill $TARGET_PID + sleep $STOP_TIMEOUT + if kill -0 $TARGET_PID > /dev/null 2>&1; then + echo "$command did not stop gracefully after $STOP_TIMEOUT seconds: killing with kill -9" + kill -9 $TARGET_PID + fi + else + echo no $command to stop + fi + rm -f $pid + else + echo no $command to stop + fi + ;; + + (*) + echo $usage + exit 1 + ;; + +esac + +echo "End $startStop $command." \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/master.properties.j2 b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/master.properties.j2 new file mode 100644 index 0000000000..d9b85e14cf --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/master.properties.j2 @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{% for key, value in dolphin_master_map.iteritems() -%} + {{key}}={{value}} +{% endfor %} \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/quartz.properties.j2 b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/quartz.properties.j2 new file mode 100644 index 0000000000..e027a263b5 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/quartz.properties.j2 @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{% for key, value in dolphin_quartz_map.iteritems() -%} + {{key}}={{value}} +{% endfor %} \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/worker.properties.j2 b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/worker.properties.j2 new file mode 100644 index 0000000000..a008b74084 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/worker.properties.j2 @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{% for key, value in dolphin_worker_map.iteritems() -%} + {{key}}={{value}} +{% endfor %} \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/zookeeper.properties.j2 b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/zookeeper.properties.j2 new file mode 100644 index 0000000000..9eb14eaef3 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/templates/zookeeper.properties.j2 @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{% for key, value in dolphin_zookeeper_map.iteritems() -%} + {{key}}={{value}} +{% endfor %} \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/quicklinks/quicklinks.json b/ambari_plugin/common-services/DOLPHIN/1.3.0/quicklinks/quicklinks.json new file mode 100755 index 0000000000..8753004fef --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/quicklinks/quicklinks.json @@ -0,0 +1,26 @@ +{ + "name": "default", + "description": "default quick links configuration", + "configuration": { + "protocol": + { + "type":"http" + }, + + "links": [ + { + "name": "dolphin-application-ui", + "label": "DolphinApplication UI", + "requires_user_name": "false", + "component_name": "DOLPHIN_API", + "url": "%@://%@:%@/dolphinscheduler/ui/view/login/index.html", + "port":{ + "http_property": "server.port", + "http_default_port": "12345", + "regex": "^(\\d+)$", + "site": "dolphin-application-api" + } + } + ] + } +} \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/themes/theme.json b/ambari_plugin/common-services/DOLPHIN/1.3.0/themes/theme.json new file mode 100644 index 0000000000..953e2323f8 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/themes/theme.json @@ -0,0 +1,661 @@ +{ + "name": "default", + "description": "Default theme for Dolphin Scheduler service", + "configuration": { + "layouts": [ + { + "name": "default", + "tabs": [ + { + "name": "settings", + "display-name": "Settings", + "layout": { + "tab-rows": "3", + "tab-columns": "3", + "sections": [ + { + "name": "dolphin-env-config", + "display-name": "Dolphin Env Config", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "2", + "section-rows": "1", + "section-columns": "2", + "subsections": [ + { + "name": "env-row1-col1", + "display-name": "Deploy User Info", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + }, + { + "name": "env-row1-col2", + "display-name": "System Env Optimization", + "row-index": "0", + "column-index": "1", + "row-span": "1", + "column-span": "1" + } + ] + }, + { + "name": "dolphin-database-config", + "display-name": "Database Config", + "row-index": "1", + "column-index": "0", + "row-span": "1", + "column-span": "2", + "section-rows": "1", + "section-columns": "3", + "subsections": [ + { + "name": "database-row1-col1", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + }, + { + "name": "database-row1-col2", + "row-index": "0", + "column-index": "1", + "row-span": "1", + "column-span": "1" + }, + { + "name": "database-row1-col3", + "row-index": "0", + "column-index": "2", + "row-span": "1", + "column-span": "1" + } + ] + }, + { + "name": "dynamic-config", + "row-index": "2", + "column-index": "0", + "row-span": "1", + "column-span": "2", + "section-rows": "1", + "section-columns": "3", + "subsections": [ + { + "name": "dynamic-row1-col1", + "display-name": "Resource FS Config", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + }, + { + "name": "dynamic-row1-col2", + "display-name": "Kerberos Info", + "row-index": "0", + "column-index": "1", + "row-span": "1", + "column-span": "1" + }, + { + "name": "dynamic-row1-col3", + "display-name": "Wechat Info", + "row-index": "0", + "column-index": "1", + "row-span": "1", + "column-span": "1" + } + ] + } + ] + } + } + ] + } + ], + "placement": { + "configuration-layout": "default", + "configs": [ + { + "config": "dolphin-env/dolphin.database.type", + "subsection-name": "database-row1-col1" + }, + { + "config": "dolphin-env/dolphin.database.host", + "subsection-name": "database-row1-col2" + }, + { + "config": "dolphin-env/dolphin.database.port", + "subsection-name": "database-row1-col2" + }, + { + "config": "dolphin-env/dolphin.database.username", + "subsection-name": "database-row1-col3" + }, + { + "config": "dolphin-env/dolphin.database.password", + "subsection-name": "database-row1-col3" + }, + { + "config": "dolphin-env/dolphin.user", + "subsection-name": "env-row1-col1" + }, + { + "config": "dolphin-env/dolphin.group", + "subsection-name": "env-row1-col1" + }, + { + "config": "dolphin-env/dolphinscheduler-env-content", + "subsection-name": "env-row1-col2" + }, + { + "config": "dolphin-common/resource.storage.type", + "subsection-name": "dynamic-row1-col1" + }, + { + "config": "dolphin-common/resource.upload.path", + "subsection-name": "dynamic-row1-col1", + "depends-on": [ + { + "configs":[ + "dolphin-common/resource.storage.type" + ], + "if": "${dolphin-common/resource.storage.type} === HDFS || ${dolphin-common/resource.storage.type} === S3", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "dolphin-common/hdfs.root.user", + "subsection-name": "dynamic-row1-col1", + "depends-on": [ + { + "configs":[ + "dolphin-common/resource.storage.type" + ], + "if": "${dolphin-common/resource.storage.type} === HDFS", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "dolphin-common/data.store2hdfs.basepath", + "subsection-name": "dynamic-row1-col1", + "depends-on": [ + { + "configs":[ + "dolphin-common/resource.storage.type" + ], + "if": "${dolphin-common/resource.storage.type} === HDFS", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "dolphin-common/fs.defaultFS", + "subsection-name": "dynamic-row1-col1", + "depends-on": [ + { + "configs":[ + "dolphin-common/resource.storage.type" + ], + "if": "${dolphin-common/resource.storage.type} === HDFS", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "dolphin-common/fs.s3a.endpoint", + "subsection-name": "dynamic-row1-col1", + "depends-on": [ + { + "configs":[ + "dolphin-common/resource.storage.type" + ], + "if": "${dolphin-common/resource.storage.type} === S3", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "dolphin-common/fs.s3a.access.key", + "subsection-name": "dynamic-row1-col1", + "depends-on": [ + { + "configs":[ + "dolphin-common/resource.storage.type" + ], + "if": "${dolphin-common/resource.storage.type} === S3", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "dolphin-common/fs.s3a.secret.key", + "subsection-name": "dynamic-row1-col1", + "depends-on": [ + { + "configs":[ + "dolphin-common/resource.storage.type" + ], + "if": "${dolphin-common/resource.storage.type} === S3", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "dolphin-common/hadoop.security.authentication.startup.state", + "subsection-name": "dynamic-row1-col2" + }, + { + "config": "dolphin-common/java.security.krb5.conf.path", + "subsection-name": "dynamic-row1-col2", + "depends-on": [ + { + "configs":[ + "dolphin-common/hadoop.security.authentication.startup.state" + ], + "if": "${dolphin-common/hadoop.security.authentication.startup.state}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "dolphin-common/login.user.keytab.username", + "subsection-name": "dynamic-row1-col2", + "depends-on": [ + { + "configs":[ + "dolphin-common/hadoop.security.authentication.startup.state" + ], + "if": "${dolphin-common/hadoop.security.authentication.startup.state}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "dolphin-common/login.user.keytab.path", + "subsection-name": "dynamic-row1-col2", + "depends-on": [ + { + "configs":[ + "dolphin-common/hadoop.security.authentication.startup.state" + ], + "if": "${dolphin-common/hadoop.security.authentication.startup.state}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "dolphin-common/kerberos.expire.time", + "subsection-name": "dynamic-row1-col2", + "depends-on": [ + { + "configs":[ + "dolphin-common/hadoop.security.authentication.startup.state" + ], + "if": "${dolphin-common/hadoop.security.authentication.startup.state}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "dolphin-alert/enterprise.wechat.enable", + "subsection-name": "dynamic-row1-col3" + }, + { + "config": "dolphin-alert/enterprise.wechat.corp.id", + "subsection-name": "dynamic-row1-col3", + "depends-on": [ + { + "configs":[ + "dolphin-alert/enterprise.wechat.enable" + ], + "if": "${dolphin-alert/enterprise.wechat.enable}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "dolphin-alert/enterprise.wechat.secret", + "subsection-name": "dynamic-row1-col3", + "depends-on": [ + { + "configs":[ + "dolphin-alert/enterprise.wechat.enable" + ], + "if": "${dolphin-alert/enterprise.wechat.enable}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "dolphin-alert/enterprise.wechat.agent.id", + "subsection-name": "dynamic-row1-col3", + "depends-on": [ + { + "configs":[ + "dolphin-alert/enterprise.wechat.enable" + ], + "if": "${dolphin-alert/enterprise.wechat.enable}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "dolphin-alert/enterprise.wechat.users", + "subsection-name": "dynamic-row1-col3", + "depends-on": [ + { + "configs":[ + "dolphin-alert/enterprise.wechat.enable" + ], + "if": "${dolphin-alert/enterprise.wechat.enable}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + } + ] + }, + "widgets": [ + { + "config": "dolphin-env/dolphin.database.type", + "widget": { + "type": "combo" + } + }, + { + "config": "dolphin-env/dolphin.database.host", + "widget": { + "type": "text-field" + } + }, + { + "config": "dolphin-env/dolphin.database.port", + "widget": { + "type": "text-field", + "units": [ + { + "unit-name": "int" + } + ] + } + }, + { + "config": "dolphin-env/dolphin.database.username", + "widget": { + "type": "text-field" + } + }, + { + "config": "dolphin-env/dolphin.database.password", + "widget": { + "type": "password" + } + }, + { + "config": "dolphin-env/dolphin.user", + "widget": { + "type": "text-field" + } + }, + { + "config": "dolphin-env/dolphin.group", + "widget": { + "type": "text-field" + } + }, + { + "config": "dolphin-env/dolphinscheduler-env-content", + "widget": { + "type": "text-area" + } + }, + { + "config": "dolphin-common/resource.storage.type", + "widget": { + "type": "combo" + } + }, + { + "config": "dolphin-common/resource.upload.path", + "widget": { + "type": "text-field" + } + }, + { + "config": "dolphin-common/hdfs.root.user", + "widget": { + "type": "text-field" + } + }, + { + "config": "dolphin-common/data.store2hdfs.basepath", + "widget": { + "type": "text-field" + } + }, + { + "config": "dolphin-common/fs.defaultFS", + "widget": { + "type": "text-field" + } + }, + { + "config": "dolphin-common/fs.s3a.endpoint", + "widget": { + "type": "text-field" + } + }, + { + "config": "dolphin-common/fs.s3a.access.key", + "widget": { + "type": "text-field" + } + }, + { + "config": "dolphin-common/fs.s3a.secret.key", + "widget": { + "type": "text-field" + } + }, + { + "config": "dolphin-common/hadoop.security.authentication.startup.state", + "widget": { + "type": "toggle" + } + }, + { + "config": "dolphin-common/java.security.krb5.conf.path", + "widget": { + "type": "text-field" + } + }, + { + "config": "dolphin-common/login.user.keytab.username", + "widget": { + "type": "text-field" + } + }, + { + "config": "dolphin-common/login.user.keytab.path", + "widget": { + "type": "text-field" + } + }, + { + "config": "dolphin-common/kerberos.expire.time", + "widget": { + "type": "text-field" + } + }, + { + "config": "dolphin-alert/enterprise.wechat.enable", + "widget": { + "type": "toggle" + } + }, + { + "config": "dolphin-alert/enterprise.wechat.corp.id", + "widget": { + "type": "text-field" + } + }, + { + "config": "dolphin-alert/enterprise.wechat.secret", + "widget": { + "type": "text-field" + } + }, + { + "config": "dolphin-alert/enterprise.wechat.agent.id", + "widget": { + "type": "text-field" + } + }, + { + "config": "dolphin-alert/enterprise.wechat.users", + "widget": { + "type": "text-field" + } + } + ] + } +} diff --git a/ambari_plugin/statcks/DOLPHIN/metainfo.xml b/ambari_plugin/statcks/DOLPHIN/metainfo.xml index 3af7f383eb..ea40cd304d 100755 --- a/ambari_plugin/statcks/DOLPHIN/metainfo.xml +++ b/ambari_plugin/statcks/DOLPHIN/metainfo.xml @@ -20,7 +20,7 @@ DOLPHIN - common-services/DOLPHIN/2.0.0 + common-services/DOLPHIN/1.3.0 \ No newline at end of file From c50ad1240c4789403e9518c855c0217fc9030b5b Mon Sep 17 00:00:00 2001 From: t1mon <178317391@qq.com> Date: Mon, 27 Apr 2020 10:23:29 +0800 Subject: [PATCH 51/55] Add query result set limit. (#2459) * Optimize PropertyUtils instantiation. * Fix info error. * Add query result set limit. * fix code smell Co-authored-by: dailidong Co-authored-by: qiaozhanwei --- .../server/worker/task/sql/SqlTask.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java index 22fa91dc1d..84e4e54a50 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java @@ -16,7 +16,6 @@ */ package org.apache.dolphinscheduler.server.worker.task.sql; -import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.serializer.SerializerFeature; @@ -24,7 +23,6 @@ import org.apache.commons.lang.StringUtils; import org.apache.dolphinscheduler.alert.utils.MailUtils; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.*; -import org.apache.dolphinscheduler.common.enums.AuthorizationType; import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.enums.ShowType; import org.apache.dolphinscheduler.common.enums.TaskTimeoutStrategy; @@ -37,7 +35,6 @@ import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.datasource.BaseDataSource; import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory; -import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.server.entity.SQLTaskExecutionContext; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; @@ -78,6 +75,10 @@ public class SqlTask extends AbstractTask { */ private TaskExecutionContext taskExecutionContext; + /** + * default query sql limit + */ + private static final int LIMIT = 10000; public SqlTask(TaskExecutionContext taskExecutionContext, Logger logger) { super(taskExecutionContext, logger); @@ -257,12 +258,15 @@ public class SqlTask extends AbstractTask { ResultSetMetaData md = resultSet.getMetaData(); int num = md.getColumnCount(); - while (resultSet.next()) { + int rowCount = 0; + + while (rowCount < LIMIT && resultSet.next()) { JSONObject mapOfColValues = new JSONObject(true); for (int i = 1; i <= num; i++) { mapOfColValues.put(md.getColumnName(i), resultSet.getObject(i)); } resultJSONArray.add(mapOfColValues); + rowCount++; } logger.debug("execute sql : {}", JSONObject.toJSONString(resultJSONArray, SerializerFeature.WriteMapNullValue)); From e6dec8d2a7a6d39280675d225ec818e0cc7012be Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Mon, 27 Apr 2020 14:16:21 +0800 Subject: [PATCH 52/55] fix #2442 when the resource name contains '$',need translate it to '\$' (#2524) * fix #2442 when the resource name contains '$',need translate it to '\$' * fix #2442 when the resource name contains '$',need translate it to '/$' Co-authored-by: qiaozhanwei Co-authored-by: dailidong --- .../api/service/ResourcesService.java | 37 ++++++++++------ .../api/service/ResourcesServiceTest.java | 43 +++++++++++++------ 2 files changed, 54 insertions(+), 26 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java index a2af47b2d5..8f3075476e 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java @@ -45,8 +45,10 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.springframework.web.multipart.MultipartFile; +import java.io.IOException; import java.text.MessageFormat; import java.util.*; +import java.util.regex.Matcher; import java.util.stream.Collectors; import static org.apache.dolphinscheduler.common.Constants.*; @@ -315,7 +317,6 @@ public class ResourcesService extends BaseService { return result; } - if (name.equals(resource.getAlias()) && desc.equals(resource.getDescription())) { putMsg(result, Status.SUCCESS); return result; @@ -323,9 +324,10 @@ public class ResourcesService extends BaseService { //check resource aleady exists String originFullName = resource.getFullName(); + String originResourceName = resource.getAlias(); String fullName = String.format("%s%s",originFullName.substring(0,originFullName.lastIndexOf("/")+1),name); - if (!resource.getAlias().equals(name) && checkResourceExists(fullName, 0, type.ordinal())) { + if (!originResourceName.equals(name) && checkResourceExists(fullName, 0, type.ordinal())) { logger.error("resource {} already exists, can't recreate", name); putMsg(result, Status.RESOURCE_EXIST); return result; @@ -336,8 +338,22 @@ public class ResourcesService extends BaseService { if (StringUtils.isEmpty(tenantCode)){ return result; } + // verify whether the resource exists in storage + // get the path of origin file in storage + String originHdfsFileName = HadoopUtils.getHdfsFileName(resource.getType(),tenantCode,originFullName); + try { + if (!HadoopUtils.getInstance().exists(originHdfsFileName)) { + logger.error("{} not exist", originHdfsFileName); + putMsg(result,Status.RESOURCE_NOT_EXIST); + return result; + } + } catch (IOException e) { + logger.error(e.getMessage(),e); + throw new ServiceException(Status.HDFS_OPERATION_ERROR); + } + String nameWithSuffix = name; - String originResourceName = resource.getAlias(); + if (!resource.isDirectory()) { //get the file suffix String suffix = originResourceName.substring(originResourceName.lastIndexOf(".")); @@ -361,10 +377,11 @@ public class ResourcesService extends BaseService { try { resourcesMapper.updateById(resource); if (resource.isDirectory() && CollectionUtils.isNotEmpty(childrenResource)) { + String matcherFullName = Matcher.quoteReplacement(fullName); List childResourceList = new ArrayList<>(); List resourceList = resourcesMapper.listResourceByIds(childrenResource.toArray(new Integer[childrenResource.size()])); childResourceList = resourceList.stream().map(t -> { - t.setFullName(t.getFullName().replaceFirst(oldFullName, fullName)); + t.setFullName(t.getFullName().replaceFirst(oldFullName, matcherFullName)); t.setUpdateTime(now); return t; }).collect(Collectors.toList()); @@ -389,19 +406,13 @@ public class ResourcesService extends BaseService { return result; } - // get the path of origin file in hdfs - String originHdfsFileName = HadoopUtils.getHdfsFileName(resource.getType(),tenantCode,originFullName); // get the path of dest file in hdfs String destHdfsFileName = HadoopUtils.getHdfsFileName(resource.getType(),tenantCode,fullName); + try { - if (HadoopUtils.getInstance().exists(originHdfsFileName)) { - logger.info("hdfs copy {} -> {}", originHdfsFileName, destHdfsFileName); - HadoopUtils.getInstance().copy(originHdfsFileName, destHdfsFileName, true, true); - } else { - logger.error("{} not exist", originHdfsFileName); - putMsg(result,Status.RESOURCE_NOT_EXIST); - } + logger.info("start hdfs copy {} -> {}", originHdfsFileName, destHdfsFileName); + HadoopUtils.getInstance().copy(originHdfsFileName, destHdfsFileName, true, true); } catch (Exception e) { logger.error(MessageFormat.format("hdfs copy {0} -> {1} fail", originHdfsFileName, destHdfsFileName), e); putMsg(result,Status.HDFS_COPY_FAIL); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java index 4f9176d699..e52f4670e2 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java @@ -19,12 +19,16 @@ package org.apache.dolphinscheduler.api.service; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.exceptions.ServiceException; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ResourceType; import org.apache.dolphinscheduler.common.enums.UserType; -import org.apache.dolphinscheduler.common.utils.*; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import org.apache.dolphinscheduler.common.utils.FileUtils; +import org.apache.dolphinscheduler.common.utils.HadoopUtils; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.dao.entity.Resource; import org.apache.dolphinscheduler.dao.entity.Tenant; import org.apache.dolphinscheduler.dao.entity.UdfFunc; @@ -37,7 +41,6 @@ import org.junit.runner.RunWith; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; -import org.omg.CORBA.Any; import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; @@ -172,10 +175,29 @@ public class ResourcesServiceTest { logger.info(result.toString()); Assert.assertEquals(Status.USER_NO_OPERATION_PERM.getMsg(),result.getMsg()); + //RESOURCE_NOT_EXIST + user.setId(1); + Mockito.when(userMapper.queryDetailsById(1)).thenReturn(getUser()); + Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); + PowerMockito.when(HadoopUtils.getHdfsFileName(Mockito.any(), Mockito.any(),Mockito.anyString())).thenReturn("test1"); + + try { + Mockito.when(HadoopUtils.getInstance().exists(Mockito.any())).thenReturn(false); + } catch (IOException e) { + logger.error(e.getMessage(),e); + } + result = resourcesService.updateResource(user, 1, "ResourcesServiceTest1.jar", "ResourcesServiceTest", ResourceType.UDF); + Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(),result.getMsg()); + //SUCCESS user.setId(1); Mockito.when(userMapper.queryDetailsById(1)).thenReturn(getUser()); Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); + try { + Mockito.when(HadoopUtils.getInstance().exists(Mockito.any())).thenReturn(true); + } catch (IOException e) { + logger.error(e.getMessage(),e); + } result = resourcesService.updateResource(user,1,"ResourcesServiceTest.jar","ResourcesServiceTest",ResourceType.FILE); logger.info(result.toString()); @@ -199,21 +221,16 @@ public class ResourcesServiceTest { logger.info(result.toString()); Assert.assertEquals(Status.TENANT_NOT_EXIST.getMsg(),result.getMsg()); - //RESOURCE_NOT_EXIST - Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); - PowerMockito.when(HadoopUtils.getHdfsResourceFileName(Mockito.any(), Mockito.any())).thenReturn("test1"); + //SUCCESS + Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); + PowerMockito.when(HadoopUtils.getHdfsResourceFileName(Mockito.any(), Mockito.any())).thenReturn("test"); try { - Mockito.when(hadoopUtils.exists("test")).thenReturn(true); - } catch (IOException e) { - e.printStackTrace(); + PowerMockito.when(HadoopUtils.getInstance().copy(Mockito.anyString(),Mockito.anyString(),true,true)).thenReturn(true); + } catch (Exception e) { + logger.error(e.getMessage(),e); } - result = resourcesService.updateResource(user,1,"ResourcesServiceTest1.jar","ResourcesServiceTest",ResourceType.UDF); - logger.info(result.toString()); - Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(),result.getMsg()); - //SUCCESS - PowerMockito.when(HadoopUtils.getHdfsResourceFileName(Mockito.any(), Mockito.any())).thenReturn("test"); result = resourcesService.updateResource(user,1,"ResourcesServiceTest1.jar","ResourcesServiceTest1.jar",ResourceType.UDF); logger.info(result.toString()); Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg()); From 4f9d8caeed66b978cbc03007e652a5b4e8d213c0 Mon Sep 17 00:00:00 2001 From: liwenhe1993 <411934049@qq.com> Date: Mon, 27 Apr 2020 16:18:21 +0800 Subject: [PATCH 53/55] Use docker-compose for deployment . Failed to execute python script (#2539) * Modify docker-compose and dockerfile 1. Add python3 in Dockerfile 2. Modify startup.sh file 3. Modify docker-compose.yml and add docker-stack.yml 4. Modify dolphinscheduler_env.sh * Add logback-xx.xml and Modify Dockerfile --- docker/docker-swarm/docker-compose.yml | 122 ++++------ docker/docker-swarm/docker-stack.yml | 230 ++++++++++++++++++ docker/docker-swarm/dolphinscheduler_env.sh | 12 +- dockerfile/Dockerfile | 3 +- .../env/dolphinscheduler_env.sh | 12 +- .../logback/logback-alert.xml | 52 ++++ .../dolphinscheduler/logback/logback-api.xml | 62 +++++ .../logback/logback-master.xml | 82 +++++++ .../logback-worker.xml} | 26 +- dockerfile/startup.sh | 19 +- 10 files changed, 512 insertions(+), 108 deletions(-) create mode 100644 docker/docker-swarm/docker-stack.yml create mode 100644 dockerfile/conf/dolphinscheduler/logback/logback-alert.xml create mode 100644 dockerfile/conf/dolphinscheduler/logback/logback-api.xml create mode 100644 dockerfile/conf/dolphinscheduler/logback/logback-master.xml rename dockerfile/conf/dolphinscheduler/{conf/worker_logback.xml => logback/logback-worker.xml} (84%) diff --git a/docker/docker-swarm/docker-compose.yml b/docker/docker-swarm/docker-compose.yml index 20fb0cced3..ffa91a0ba8 100644 --- a/docker/docker-swarm/docker-compose.yml +++ b/docker/docker-swarm/docker-compose.yml @@ -16,36 +16,6 @@ version: "3.4" -networks: - dolphinscheduler-postgresql: - driver: bridge - dolphinscheduler-zookeeper: - driver: bridge - dolphinscheduler-api: - driver: bridge - dolphinscheduler-frontend: - driver: bridge - dolphinscheduler-alert: - driver: bridge - dolphinscheduler-master: - driver: bridge - dolphinscheduler-worker: - driver: bridge - -volumes: - dolphinscheduler-postgresql: - dolphinscheduler-zookeeper: - dolphinscheduler-api: - dolphinscheduler-frontend: - dolphinscheduler-alert: - dolphinscheduler-master: - dolphinscheduler-worker-data: - dolphinscheduler-worker-logs: - -configs: - dolphinscheduler-worker-task-env: - file: ./dolphinscheduler_env.sh - services: dolphinscheduler-postgresql: @@ -58,16 +28,10 @@ services: POSTGRESQL_USERNAME: root POSTGRESQL_PASSWORD: root POSTGRESQL_DATABASE: dolphinscheduler - healthcheck: - test: ["CMD", "pg_isready", "-U", "${POSTGRESQL_USERNAME}", "-d", "{POSTGRESQL_PASSWORD}", "-h", "localhost", "5432"] - interval: 30s - timeout: 5s - retries: 3 - # start_period: 30s - volumes: + volumes: - dolphinscheduler-postgresql:/bitnami/postgresql networks: - - dolphinscheduler-postgresql + - dolphinscheduler dolphinscheduler-zookeeper: image: bitnami/zookeeper:latest @@ -77,19 +41,13 @@ services: environment: TZ: Asia/Shanghai ALLOW_ANONYMOUS_LOGIN: "yes" - healthcheck: - test: ["CMD-SHELL", "nc -z localhost 2181"] - interval: 30s - timeout: 5s - retries: 3 - # start_period: 30s volumes: - dolphinscheduler-zookeeper:/bitnami/zookeeper networks: - - dolphinscheduler-zookeeper + - dolphinscheduler dolphinscheduler-api: - image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:1.2.1 + image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev container_name: dolphinscheduler-api command: ["api-server"] ports: @@ -103,23 +61,21 @@ services: POSTGRESQL_DATABASE: dolphinscheduler ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 healthcheck: - test: ["CMD-SHELL", "curl -f http://localhost:12345"] + test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"] interval: 30s timeout: 5s retries: 3 - # start_period: 30s + start_period: 30s depends_on: - dolphinscheduler-postgresql - dolphinscheduler-zookeeper volumes: - dolphinscheduler-api:/opt/dolphinscheduler/logs networks: - - dolphinscheduler-api - - dolphinscheduler-postgresql - - dolphinscheduler-zookeeper + - dolphinscheduler dolphinscheduler-frontend: - image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:1.2.1 + image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev container_name: dolphinscheduler-frontend command: ["frontend"] ports: @@ -129,21 +85,20 @@ services: FRONTEND_API_SERVER_HOST: dolphinscheduler-api FRONTEND_API_SERVER_PORT: 12345 healthcheck: - test: ["CMD-SHELL", "curl -f http://localhost:8888"] + test: ["CMD", "nc", "-z", "localhost", "8888"] interval: 30s timeout: 5s retries: 3 - # start_period: 30s + start_period: 30s depends_on: - dolphinscheduler-api volumes: - dolphinscheduler-frontend:/var/log/nginx networks: - - dolphinscheduler-frontend - - dolphinscheduler-api + - dolphinscheduler dolphinscheduler-alert: - image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:1.2.1 + image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev container_name: dolphinscheduler-alert command: ["alert-server"] environment: @@ -172,17 +127,16 @@ services: interval: 30s timeout: 5s retries: 3 - # start_period: 30s + start_period: 30s depends_on: - dolphinscheduler-postgresql volumes: - dolphinscheduler-alert:/opt/dolphinscheduler/logs networks: - - dolphinscheduler-alert - - dolphinscheduler-postgresql + - dolphinscheduler dolphinscheduler-master: - image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:1.2.1 + image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev container_name: dolphinscheduler-master command: ["master-server"] ports: @@ -207,19 +161,17 @@ services: interval: 30s timeout: 5s retries: 3 - # start_period: 30s + start_period: 30s depends_on: - dolphinscheduler-postgresql - dolphinscheduler-zookeeper volumes: - dolphinscheduler-master:/opt/dolphinscheduler/logs networks: - - dolphinscheduler-master - - dolphinscheduler-postgresql - - dolphinscheduler-zookeeper + - dolphinscheduler dolphinscheduler-worker: - image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:1.2.1 + image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev container_name: dolphinscheduler-worker command: ["worker-server"] ports: @@ -245,17 +197,37 @@ services: interval: 30s timeout: 5s retries: 3 - # start_period: 30s + start_period: 30s depends_on: - dolphinscheduler-postgresql - dolphinscheduler-zookeeper - volumes: - - dolphinscheduler-worker-data:/tmp/dolphinscheduler - - dolphinscheduler-worker-logs:/opt/dolphinscheduler/logs - configs: - - source: dolphinscheduler-worker-task-env + volumes: + - type: bind + source: ./dolphinscheduler_env.sh target: /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh + - type: volume + source: dolphinscheduler-worker-data + target: /tmp/dolphinscheduler + - type: volume + source: dolphinscheduler-worker-logs + target: /opt/dolphinscheduler/logs networks: - - dolphinscheduler-worker - - dolphinscheduler-postgresql - - dolphinscheduler-zookeeper \ No newline at end of file + - dolphinscheduler + +networks: + dolphinscheduler: + driver: bridge + +volumes: + dolphinscheduler-postgresql: + dolphinscheduler-zookeeper: + dolphinscheduler-api: + dolphinscheduler-frontend: + dolphinscheduler-alert: + dolphinscheduler-master: + dolphinscheduler-worker-data: + dolphinscheduler-worker-logs: + +configs: + dolphinscheduler-worker-task-env: + file: ./dolphinscheduler_env.sh \ No newline at end of file diff --git a/docker/docker-swarm/docker-stack.yml b/docker/docker-swarm/docker-stack.yml new file mode 100644 index 0000000000..e628205ade --- /dev/null +++ b/docker/docker-swarm/docker-stack.yml @@ -0,0 +1,230 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "3.4" + +services: + + dolphinscheduler-postgresql: + image: bitnami/postgresql:latest + ports: + - 5432:5432 + environment: + TZ: Asia/Shanghai + POSTGRESQL_USERNAME: root + POSTGRESQL_PASSWORD: root + POSTGRESQL_DATABASE: dolphinscheduler + volumes: + - dolphinscheduler-postgresql:/bitnami/postgresql + networks: + - dolphinscheduler + deploy: + mode: replicated + replicas: 1 + + dolphinscheduler-zookeeper: + image: bitnami/zookeeper:latest + ports: + - 2181:2181 + environment: + TZ: Asia/Shanghai + ALLOW_ANONYMOUS_LOGIN: "yes" + volumes: + - dolphinscheduler-zookeeper:/bitnami/zookeeper + networks: + - dolphinscheduler + deploy: + mode: replicated + replicas: 1 + + dolphinscheduler-api: + image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev + command: ["api-server"] + ports: + - 12345:12345 + environment: + TZ: Asia/Shanghai + POSTGRESQL_HOST: dolphinscheduler-postgresql + POSTGRESQL_PORT: 5432 + POSTGRESQL_USERNAME: root + POSTGRESQL_PASSWORD: root + POSTGRESQL_DATABASE: dolphinscheduler + ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 + healthcheck: + test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"] + interval: 30 + timeout: 5s + retries: 3 + start_period: 30s + volumes: + - dolphinscheduler-api:/opt/dolphinscheduler/logs + networks: + - dolphinscheduler + deploy: + mode: replicated + replicas: 1 + + dolphinscheduler-frontend: + image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev + command: ["frontend"] + ports: + - 8888:8888 + environment: + TZ: Asia/Shanghai + FRONTEND_API_SERVER_HOST: dolphinscheduler-api + FRONTEND_API_SERVER_PORT: 12345 + healthcheck: + test: ["CMD", "nc", "-z", "localhost", "8888"] + interval: 30 + timeout: 5s + retries: 3 + start_period: 30s + volumes: + - dolphinscheduler-frontend:/var/log/nginx + networks: + - dolphinscheduler + deploy: + mode: replicated + replicas: 1 + + dolphinscheduler-alert: + image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev + command: ["alert-server"] + environment: + TZ: Asia/Shanghai + XLS_FILE_PATH: "/tmp/xls" + MAIL_SERVER_HOST: "" + MAIL_SERVER_PORT: "" + MAIL_SENDER: "" + MAIL_USER: "" + MAIL_PASSWD: "" + MAIL_SMTP_STARTTLS_ENABLE: "false" + MAIL_SMTP_SSL_ENABLE: "false" + MAIL_SMTP_SSL_TRUST: "" + ENTERPRISE_WECHAT_ENABLE: "false" + ENTERPRISE_WECHAT_CORP_ID: "" + ENTERPRISE_WECHAT_SECRET: "" + ENTERPRISE_WECHAT_AGENT_ID: "" + ENTERPRISE_WECHAT_USERS: "" + POSTGRESQL_HOST: dolphinscheduler-postgresql + POSTGRESQL_PORT: 5432 + POSTGRESQL_USERNAME: root + POSTGRESQL_PASSWORD: root + POSTGRESQL_DATABASE: dolphinscheduler + healthcheck: + test: ["CMD", "/root/checkpoint.sh", "AlertServer"] + interval: 30 + timeout: 5s + retries: 3 + start_period: 30s + volumes: + - dolphinscheduler-alert:/opt/dolphinscheduler/logs + networks: + - dolphinscheduler + deploy: + mode: replicated + replicas: 1 + + dolphinscheduler-master: + image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev + command: ["master-server"] + ports: + - 5678:5678 + environment: + TZ: Asia/Shanghai + MASTER_EXEC_THREADS: "100" + MASTER_EXEC_TASK_NUM: "20" + MASTER_HEARTBEAT_INTERVAL: "10" + MASTER_TASK_COMMIT_RETRYTIMES: "5" + MASTER_TASK_COMMIT_INTERVAL: "1000" + MASTER_MAX_CPULOAD_AVG: "100" + MASTER_RESERVED_MEMORY: "0.1" + POSTGRESQL_HOST: dolphinscheduler-postgresql + POSTGRESQL_PORT: 5432 + POSTGRESQL_USERNAME: root + POSTGRESQL_PASSWORD: root + POSTGRESQL_DATABASE: dolphinscheduler + ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 + healthcheck: + test: ["CMD", "/root/checkpoint.sh", "MasterServer"] + interval: 30 + timeout: 5s + retries: 3 + start_period: 30s + volumes: + - dolphinscheduler-master:/opt/dolphinscheduler/logs + networks: + - dolphinscheduler + deploy: + mode: replicated + replicas: 1 + + dolphinscheduler-worker: + image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev + command: ["worker-server"] + ports: + - 1234:1234 + - 50051:50051 + environment: + TZ: Asia/Shanghai + WORKER_EXEC_THREADS: "100" + WORKER_HEARTBEAT_INTERVAL: "10" + WORKER_FETCH_TASK_NUM: "3" + WORKER_MAX_CPULOAD_AVG: "100" + WORKER_RESERVED_MEMORY: "0.1" + WORKER_GROUP: "default" + DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler" + POSTGRESQL_HOST: dolphinscheduler-postgresql + POSTGRESQL_PORT: 5432 + POSTGRESQL_USERNAME: root + POSTGRESQL_PASSWORD: root + POSTGRESQL_DATABASE: dolphinscheduler + ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 + healthcheck: + test: ["CMD", "/root/checkpoint.sh", "WorkerServer"] + interval: 30 + timeout: 5s + retries: 3 + start_period: 30s + volumes: + - dolphinscheduler-worker-data:/tmp/dolphinscheduler + - dolphinscheduler-worker-logs:/opt/dolphinscheduler/logs + configs: + - source: dolphinscheduler-worker-task-env + target: /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh + networks: + - dolphinscheduler + deploy: + mode: replicated + replicas: 1 + +networks: + dolphinscheduler: + driver: overlay + +volumes: + dolphinscheduler-postgresql: + dolphinscheduler-zookeeper: + dolphinscheduler-api: + dolphinscheduler-frontend: + dolphinscheduler-alert: + dolphinscheduler-master: + dolphinscheduler-worker-data: + dolphinscheduler-worker-logs: + +configs: + dolphinscheduler-worker-task-env: + file: ./dolphinscheduler_env.sh \ No newline at end of file diff --git a/docker/docker-swarm/dolphinscheduler_env.sh b/docker/docker-swarm/dolphinscheduler_env.sh index 790e30636e..654318cb41 100644 --- a/docker/docker-swarm/dolphinscheduler_env.sh +++ b/docker/docker-swarm/dolphinscheduler_env.sh @@ -15,12 +15,6 @@ # limitations under the License. # -export HADOOP_HOME=/opt/soft/hadoop -export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop -export SPARK_HOME1=/opt/soft/spark1 -export SPARK_HOME2=/opt/soft/spark2 -export PYTHON_HOME=/opt/soft/python -export JAVA_HOME=/opt/soft/java -export HIVE_HOME=/opt/soft/hive -export FLINK_HOME=/opt/soft/flink -export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$PATH \ No newline at end of file +export PYTHON_HOME=/usr/bin/python2 +export JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk +export PATH=$PYTHON_HOME:$JAVA_HOME/bin:$PATH \ No newline at end of file diff --git a/dockerfile/Dockerfile b/dockerfile/Dockerfile index c48b51e377..bed8f6537c 100644 --- a/dockerfile/Dockerfile +++ b/dockerfile/Dockerfile @@ -27,7 +27,7 @@ ENV DEBIAN_FRONTEND noninteractive #If install slowly, you can replcae alpine's mirror with aliyun's mirror, Example: #RUN sed -i "s/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g" /etc/apk/repositories RUN apk update && \ - apk add dos2unix shadow bash openrc python sudo vim wget iputils net-tools openssh-server py2-pip tini && \ + apk add dos2unix shadow bash openrc python python3 sudo vim wget iputils net-tools openssh-server py2-pip tini && \ apk add --update procps && \ openrc boot && \ pip install kazoo @@ -67,6 +67,7 @@ ADD ./checkpoint.sh /root/checkpoint.sh ADD ./startup-init-conf.sh /root/startup-init-conf.sh ADD ./startup.sh /root/startup.sh ADD ./conf/dolphinscheduler/*.tpl /opt/dolphinscheduler/conf/ +ADD ./conf/dolphinscheduler/logback/* /opt/dolphinscheduler/conf/ ADD conf/dolphinscheduler/env/dolphinscheduler_env.sh /opt/dolphinscheduler/conf/env/ RUN chmod +x /root/checkpoint.sh && \ chmod +x /root/startup-init-conf.sh && \ diff --git a/dockerfile/conf/dolphinscheduler/env/dolphinscheduler_env.sh b/dockerfile/conf/dolphinscheduler/env/dolphinscheduler_env.sh index 070c438bb6..78c8f98bc1 100644 --- a/dockerfile/conf/dolphinscheduler/env/dolphinscheduler_env.sh +++ b/dockerfile/conf/dolphinscheduler/env/dolphinscheduler_env.sh @@ -15,12 +15,6 @@ # limitations under the License. # -export HADOOP_HOME=/opt/soft/hadoop -export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop -export SPARK_HOME1=/opt/soft/spark1 -export SPARK_HOME2=/opt/soft/spark2 -export PYTHON_HOME=/opt/soft/python -export JAVA_HOME=/opt/soft/java -export HIVE_HOME=/opt/soft/hive -export FLINK_HOME=/opt/soft/flink -export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$PATH +export PYTHON_HOME=/usr/bin/python2 +export JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk +export PATH=$PYTHON_HOME/bin:$JAVA_HOME/bin:$PATH diff --git a/dockerfile/conf/dolphinscheduler/logback/logback-alert.xml b/dockerfile/conf/dolphinscheduler/logback/logback-alert.xml new file mode 100644 index 0000000000..5d1c07858d --- /dev/null +++ b/dockerfile/conf/dolphinscheduler/logback/logback-alert.xml @@ -0,0 +1,52 @@ + + + + + + + + + + + [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n + + UTF-8 + + + + + ${log.base}/dolphinscheduler-alert.log + + ${log.base}/dolphinscheduler-alert.%d{yyyy-MM-dd_HH}.%i.log + 20 + 64MB + + + + [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n + + UTF-8 + + + + + + + + + \ No newline at end of file diff --git a/dockerfile/conf/dolphinscheduler/logback/logback-api.xml b/dockerfile/conf/dolphinscheduler/logback/logback-api.xml new file mode 100644 index 0000000000..2df90d8392 --- /dev/null +++ b/dockerfile/conf/dolphinscheduler/logback/logback-api.xml @@ -0,0 +1,62 @@ + + + + + + + + + + + [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n + + UTF-8 + + + + + + ${log.base}/dolphinscheduler-api-server.log + + INFO + + + ${log.base}/dolphinscheduler-api-server.%d{yyyy-MM-dd_HH}.%i.log + 168 + 64MB + + + + [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n + + UTF-8 + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/dockerfile/conf/dolphinscheduler/logback/logback-master.xml b/dockerfile/conf/dolphinscheduler/logback/logback-master.xml new file mode 100644 index 0000000000..7410c01f05 --- /dev/null +++ b/dockerfile/conf/dolphinscheduler/logback/logback-master.xml @@ -0,0 +1,82 @@ + + + + + + + + + + + [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n + + UTF-8 + + + + + + + INFO + + + + taskAppId + ${log.base} + + + + ${log.base}/${taskAppId}.log + + + [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %messsage%n + + UTF-8 + + true + + + + + + ${log.base}/dolphinscheduler-master.log + + + ${log.base}/dolphinscheduler-master.%d{yyyy-MM-dd_HH}.%i.log + 168 + 200MB + + + + [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n + + UTF-8 + + + + + + + + + + + \ No newline at end of file diff --git a/dockerfile/conf/dolphinscheduler/conf/worker_logback.xml b/dockerfile/conf/dolphinscheduler/logback/logback-worker.xml similarity index 84% rename from dockerfile/conf/dolphinscheduler/conf/worker_logback.xml rename to dockerfile/conf/dolphinscheduler/logback/logback-worker.xml index 1b09260334..be1d0acde5 100644 --- a/dockerfile/conf/dolphinscheduler/conf/worker_logback.xml +++ b/dockerfile/conf/dolphinscheduler/logback/logback-worker.xml @@ -1,4 +1,4 @@ - + - + + @@ -27,11 +28,15 @@ UTF-8 + + + INFO - + taskAppId ${log.base} @@ -41,7 +46,7 @@ ${log.base}/${taskAppId}.log - [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n + [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %messsage%n UTF-8 @@ -49,31 +54,30 @@ - ${log.base}/dolphinscheduler-worker.log - + INFO - + ${log.base}/dolphinscheduler-worker.%d{yyyy-MM-dd_HH}.%i.log 168 200MB - 　　　　　 - [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n + [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %messsage%n UTF-8 - 　　 - + + + \ No newline at end of file diff --git a/dockerfile/startup.sh b/dockerfile/startup.sh index af3c456116..71ab506777 100644 --- a/dockerfile/startup.sh +++ b/dockerfile/startup.sh @@ -25,7 +25,9 @@ DOLPHINSCHEDULER_LOGS=${DOLPHINSCHEDULER_HOME}/logs # start postgresql initPostgreSQL() { echo "checking postgresql" - if [ -n "$(ifconfig | grep ${POSTGRESQL_HOST})" ]; then + if [[ "${POSTGRESQL_HOST}" = "127.0.0.1" || "${POSTGRESQL_HOST}" = "localhost" ]]; then + export PGPORT=${POSTGRESQL_PORT} + echo "start postgresql service" rc-service postgresql restart @@ -47,10 +49,21 @@ initPostgreSQL() { sudo -u postgres psql -tAc "grant all privileges on database dolphinscheduler to ${POSTGRESQL_USERNAME}" fi + echo "test postgresql service" + while ! nc -z ${POSTGRESQL_HOST} ${POSTGRESQL_PORT}; do + counter=$((counter+1)) + if [ $counter == 30 ]; then + echo "Error: Couldn't connect to postgresql." + exit 1 + fi + echo "Trying to connect to postgresql at ${POSTGRESQL_HOST}:${POSTGRESQL_PORT}. Attempt $counter." + sleep 5 + done + echo "connect postgresql service" - v=$(sudo -u postgres PGPASSWORD=${POSTGRESQL_PASSWORD} psql -h ${POSTGRESQL_HOST} -U ${POSTGRESQL_USERNAME} -d dolphinscheduler -tAc "select 1") + v=$(sudo -u postgres PGPASSWORD=${POSTGRESQL_PASSWORD} psql -h ${POSTGRESQL_HOST} -p ${POSTGRESQL_PORT} -U ${POSTGRESQL_USERNAME} -d dolphinscheduler -tAc "select 1") if [ "$(echo '${v}' | grep 'FATAL' | wc -l)" -eq 1 ]; then - echo "Can't connect to database...${v}" + echo "Error: Can't connect to database...${v}" exit 1 fi From 76b61c1542e6605eecc4ee7260a79d70c1246e23 Mon Sep 17 00:00:00 2001 From: break60 <790061044@qq.com> Date: Tue, 28 Apr 2020 10:57:54 +0800 Subject: [PATCH 54/55] Fix page display incomplete --- .../projects/pages/definition/pages/list/index.vue | 13 +++++++++++-- .../projects/pages/instance/pages/list/index.vue | 11 +++++++++-- .../pages/projects/pages/taskInstance/index.vue | 11 +++++++++-- .../pages/resource/pages/file/pages/list/index.vue | 13 +++++++++++-- .../pages/file/pages/subdirectory/index.vue | 11 ++++++++++- .../resource/pages/udf/pages/function/index.vue | 14 +++++++++++--- .../resource/pages/udf/pages/resource/index.vue | 14 +++++++++++--- .../pages/udf/pages/subUdfDirectory/index.vue | 11 ++++++++++- .../conf/home/pages/security/pages/queue/index.vue | 11 ++++++++++- .../home/pages/security/pages/tenement/index.vue | 11 ++++++++++- .../conf/home/pages/security/pages/users/index.vue | 13 +++++++++++-- .../pages/security/pages/warningGroups/index.vue | 11 ++++++++++- .../js/conf/home/pages/user/pages/token/index.vue | 13 +++++++++++-- dolphinscheduler-ui/src/js/conf/login/App.vue | 1 + .../components/secondaryMenu/secondaryMenu.vue | 8 +++++++- .../src/js/module/components/spin/spin.vue | 8 ++++---- dolphinscheduler-ui/src/sass/conf/home/index.scss | 5 +++++ 17 files changed, 151 insertions(+), 28 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/index.vue index 1f28578504..5dd12b4355 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/index.vue @@ -35,7 +35,7 @@ - + @@ -64,7 +64,8 @@ pageNo: 1, searchVal: '', userId: '' - } + }, + isLeft: true } }, mixins: [listUrlParamHandle], @@ -98,6 +99,11 @@ * get data list */ _getList (flag) { + if(sessionStorage.getItem('isLeft')==0) { + this.isLeft = false + } else { + this.isLeft = true + } this.isLoading = !flag this.getProcessListP(this.searchParams).then(res => { if(this.searchParams.pageNo>1 && res.totalList.length == 0) { @@ -133,6 +139,9 @@ mounted() { this.$modal.destroy() }, + beforeDestroy () { + sessionStorage.setItem('isLeft',1) + }, components: { mList, mConditions, mSpin, mListConstruction, mSecondaryMenu, mNoData } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/index.vue index b95d4ed720..891dc2e281 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/index.vue @@ -30,7 +30,7 @@ - + @@ -74,7 +74,8 @@ endDate: '', // Exectuor Name executorName: '' - } + }, + isLeft: true } }, props: {}, @@ -136,6 +137,11 @@ * @desc Prevent functions from being called multiple times */ _debounceGET: _.debounce(function (flag) { + if(sessionStorage.getItem('isLeft')==0) { + this.isLeft = false + } else { + this.isLeft = true + } this._getProcessInstanceListP(flag) }, 100, { 'leading': false, @@ -183,6 +189,7 @@ beforeDestroy () { // Destruction wheel clearInterval(this.setIntervalP) + sessionStorage.setItem('isLeft',1) }, components: { mList, mInstanceConditions, mSpin, mListConstruction, mSecondaryMenu, mNoData } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/taskInstance/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/taskInstance/index.vue index 4cb166647e..594ffad14e 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/taskInstance/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/taskInstance/index.vue @@ -30,7 +30,7 @@ - + @@ -71,7 +71,8 @@ endDate: '', // Exectuor Name executorName: '' - } + }, + isLeft: true } }, mixins: [listUrlParamHandle], @@ -118,6 +119,11 @@ * @desc Prevent functions from being called multiple times */ _debounceGET: _.debounce(function (flag) { + if(sessionStorage.getItem('isLeft')==0) { + this.isLeft = false + } else { + this.isLeft = true + } this._getList(flag) }, 100, { 'leading': false, @@ -146,6 +152,7 @@ beforeDestroy () { // Destruction wheel clearInterval(this.setIntervalP) + sessionStorage.setItem('isLeft',1) }, components: { mList, mInstanceConditions, mSpin, mListConstruction, mSecondaryMenu, mNoData } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/list/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/list/index.vue index 73ce023ee7..5cf343ddda 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/list/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/list/index.vue @@ -38,7 +38,7 @@ - + @@ -67,7 +67,8 @@ pageNo: 1, searchVal: '', type: 'FILE' - } + }, + isLeft: true } }, mixins: [listUrlParamHandle], @@ -91,6 +92,11 @@ this.searchParams.pageSize = val }, _getList (flag) { + if(sessionStorage.getItem('isLeft')==0) { + this.isLeft = false + } else { + this.isLeft = true + } this.isLoading = !flag this.getResourcesListP(this.searchParams).then(res => { if(this.searchParams.pageNo>1 && res.totalList.length == 0) { @@ -125,6 +131,9 @@ mounted () { this.$modal.destroy() }, + beforeDestroy () { + sessionStorage.setItem('isLeft',1) + }, components: { mListConstruction, mConditions, mList, mSpin, mNoData } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/subdirectory/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/subdirectory/index.vue index 12be6b0bc8..dac5cc0a86 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/subdirectory/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/subdirectory/index.vue @@ -42,7 +42,7 @@ - + @@ -73,6 +73,7 @@ searchVal: '', type: 'FILE' }, + isLeft: true, breadList: [] } }, @@ -97,6 +98,11 @@ this.searchParams.pageSize = val }, _getList (flag) { + if(sessionStorage.getItem('isLeft')==0) { + this.isLeft = false + } else { + this.isLeft = true + } this.isLoading = !flag this.getResourcesListP(this.searchParams).then(res => { if(this.searchParams.pageNo>1 && res.totalList.length == 0) { @@ -159,6 +165,9 @@ this.breadList = dir this.$modal.destroy() }, + beforeDestroy () { + sessionStorage.setItem('isLeft',1) + }, components: { mListConstruction, mConditions, mList, mSpin, mNoData } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/function/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/function/index.vue index d6c79bd258..74b789fe93 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/function/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/function/index.vue @@ -36,8 +36,7 @@ - - + @@ -64,7 +63,8 @@ pageSize: 10, pageNo: 1, searchVal: '' - } + }, + isLeft: true } }, mixins: [listUrlParamHandle], @@ -110,6 +110,11 @@ this._debounceGET() }, _getList (flag) { + if(sessionStorage.getItem('isLeft')==0) { + this.isLeft = false + } else { + this.isLeft = true + } this.isLoading = !flag this.getUdfFuncListP(this.searchParams).then(res => { if(this.searchParams.pageNo>1 && res.totalList.length == 0) { @@ -137,6 +142,9 @@ mounted () { this.$modal.destroy() }, + beforeDestroy () { + sessionStorage.setItem('isLeft',1) + }, components: { mListConstruction, mConditions, mList, mSpin, mCreateUdf, mNoData } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/resource/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/resource/index.vue index b87b17800a..4058f267b7 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/resource/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/resource/index.vue @@ -37,8 +37,7 @@ - - + @@ -66,7 +65,8 @@ pageNo: 1, searchVal: '', type: 'UDF' - } + }, + isLeft: true } }, mixins: [listUrlParamHandle], @@ -98,6 +98,11 @@ this._debounceGET() }, _getList (flag) { + if(sessionStorage.getItem('isLeft')==0) { + this.isLeft = false + } else { + this.isLeft = true + } this.isLoading = !flag this.getResourcesListP(this.searchParams).then(res => { if(this.searchParams.pageNo>1 && res.totalList.length == 0) { @@ -125,6 +130,9 @@ mounted () { this.$modal.destroy() }, + beforeDestroy () { + sessionStorage.setItem('isLeft',1) + }, components: { mListConstruction, mConditions, mList, mSpin, mNoData } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/subUdfDirectory/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/subUdfDirectory/index.vue index dd39716f9a..ea8c4d838f 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/subUdfDirectory/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/subUdfDirectory/index.vue @@ -41,7 +41,7 @@ - + @@ -72,6 +72,7 @@ searchVal: '', type: 'UDF' }, + isLeft: true, breadList: [] } }, @@ -106,6 +107,11 @@ this._debounceGET() }, _getList (flag) { + if(sessionStorage.getItem('isLeft')==0) { + this.isLeft = false + } else { + this.isLeft = true + } this.isLoading = !flag this.getResourcesListP(this.searchParams).then(res => { if(this.searchParams.pageNo>1 && res.totalList.length == 0) { @@ -160,6 +166,9 @@ this.breadList = dir this.$modal.destroy() }, + beforeDestroy () { + sessionStorage.setItem('isLeft',1) + }, components: { mListConstruction, mConditions, mList, mSpin, mNoData } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/queue/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/queue/index.vue index 47ce14abd7..8e559e45a7 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/queue/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/queue/index.vue @@ -38,7 +38,7 @@ - + @@ -66,6 +66,7 @@ pageNo: 1, searchVal: '' }, + isLeft: true, isADMIN: store.state.user.userInfo.userType === 'ADMIN_USER' } }, @@ -116,6 +117,11 @@ }) }, _getList (flag) { + if(sessionStorage.getItem('isLeft')==0) { + this.isLeft = false + } else { + this.isLeft = true + } this.isLoading = !flag this.getQueueListP(this.searchParams).then(res => { if(this.searchParams.pageNo>1 && res.totalList.length == 0) { @@ -143,6 +149,9 @@ mounted () { this.$modal.destroy() }, + beforeDestroy () { + sessionStorage.setItem('isLeft',1) + }, components: { mList, mListConstruction, mConditions, mSpin, mNoData } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/tenement/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/tenement/index.vue index 0c38f0a911..ca180b1718 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/tenement/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/tenement/index.vue @@ -39,7 +39,7 @@ - + @@ -67,6 +67,7 @@ pageNo: 1, searchVal: '' }, + isLeft: true, isADMIN: store.state.user.userInfo.userType === 'ADMIN_USER' } }, @@ -120,6 +121,11 @@ }) }, _getList (flag) { + if(sessionStorage.getItem('isLeft')==0) { + this.isLeft = false + } else { + this.isLeft = true + } this.isLoading = !flag this.getTenantListP(this.searchParams).then(res => { if(this.searchParams.pageNo>1 && res.totalList.length == 0) { @@ -147,6 +153,9 @@ mounted () { this.$modal.destroy() }, + beforeDestroy () { + sessionStorage.setItem('isLeft',1) + }, components: { mList, mListConstruction, mConditions, mSpin, mNoData } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/users/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/users/index.vue index f8ad40df85..b09b08bcf6 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/users/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/users/index.vue @@ -39,7 +39,7 @@ - + @@ -65,7 +65,8 @@ pageSize: 10, pageNo: 1, searchVal: '' - } + }, + isLeft: true } }, mixins: [listUrlParamHandle], @@ -118,6 +119,11 @@ }) }, _getList (flag) { + if(sessionStorage.getItem('isLeft')==0) { + this.isLeft = false + } else { + this.isLeft = true + } this.isLoading = !flag this.getUsersListP(this.searchParams).then(res => { if(this.searchParams.pageNo>1 && res.totalList.length == 0) { @@ -145,6 +151,9 @@ mounted () { this.$modal.destroy() }, + beforeDestroy () { + sessionStorage.setItem('isLeft',1) + }, components: { mList, mListConstruction, mConditions, mSpin, mNoData } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/warningGroups/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/warningGroups/index.vue index e70ead46ce..ad604ba3a6 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/warningGroups/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/warningGroups/index.vue @@ -39,7 +39,7 @@ - + @@ -67,6 +67,7 @@ pageNo: 1, searchVal: '' }, + isLeft: true, isADMIN: store.state.user.userInfo.userType === 'ADMIN_USER' } }, @@ -120,6 +121,11 @@ }) }, _getList (flag) { + if(sessionStorage.getItem('isLeft')==0) { + this.isLeft = false + } else { + this.isLeft = true + } this.isLoading = !flag this.getAlertgroupP(this.searchParams).then(res => { if(this.searchParams.pageNo>1 && res.totalList.length == 0) { @@ -147,6 +153,9 @@ mounted () { this.$modal.destroy() }, + beforeDestroy () { + sessionStorage.setItem('isLeft',1) + }, components: { mList, mListConstruction, mConditions, mSpin, mNoData } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/user/pages/token/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/user/pages/token/index.vue index 21d2becd32..3398acca13 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/user/pages/token/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/user/pages/token/index.vue @@ -40,7 +40,7 @@ - + @@ -67,7 +67,8 @@ pageSize: 10, pageNo: 1, searchVal: '' - } + }, + isLeft: true } }, mixins: [listUrlParamHandle], @@ -120,6 +121,11 @@ }) }, _getList (flag) { + if(sessionStorage.getItem('isLeft')==0) { + this.isLeft = false + } else { + this.isLeft = true + } this.isLoading = !flag this.getTokenListP(this.searchParams).then(res => { if(this.searchParams.pageNo>1 && res.totalList.length == 0) { @@ -147,6 +153,9 @@ mounted () { this.$modal.destroy() }, + beforeDestroy () { + sessionStorage.setItem('isLeft',1) + }, components: { mSecondaryMenu, mList, mListConstruction, mConditions, mSpin, mNoData } } diff --git a/dolphinscheduler-ui/src/js/conf/login/App.vue b/dolphinscheduler-ui/src/js/conf/login/App.vue index a79743a262..8d065920a8 100644 --- a/dolphinscheduler-ui/src/js/conf/login/App.vue +++ b/dolphinscheduler-ui/src/js/conf/login/App.vue @@ -84,6 +84,7 @@ this._gLogin().then(res => { setTimeout(() => { this.spinnerLoading = false + sessionStorage.setItem('isLeft',1); if (res.data.hasOwnProperty("sessionId")) { let sessionId=res.data.sessionId sessionStorage.setItem("sessionId", sessionId) diff --git a/dolphinscheduler-ui/src/js/module/components/secondaryMenu/secondaryMenu.vue b/dolphinscheduler-ui/src/js/module/components/secondaryMenu/secondaryMenu.vue index 102df8f910..5336555c21 100644 --- a/dolphinscheduler-ui/src/js/module/components/secondaryMenu/secondaryMenu.vue +++ b/dolphinscheduler-ui/src/js/module/components/secondaryMenu/secondaryMenu.vue @@ -63,7 +63,8 @@ menuList: menu(this.type), index: 0, id: this.$route.params.id, - isTogHide: false + isTogHide: false, + isLeft: true } }, props: { @@ -82,6 +83,11 @@ }, _toggleMenu () { this.isTogHide = !this.isTogHide + if(this.isTogHide) { + sessionStorage.setItem('isLeft',0) + } else { + sessionStorage.setItem('isLeft',1) + } } }, mounted () { diff --git a/dolphinscheduler-ui/src/js/module/components/spin/spin.vue b/dolphinscheduler-ui/src/js/module/components/spin/spin.vue index 95fc8e924e..7c6a9c3acf 100644 --- a/dolphinscheduler-ui/src/js/module/components/spin/spin.vue +++ b/dolphinscheduler-ui/src/js/module/components/spin/spin.vue @@ -47,7 +47,7 @@ #spin-model { position: fixed; left: 20px; - top: 80px; + top: 60px; background: #fff; z-index: 99; border-radius: 3px; @@ -69,11 +69,11 @@ } &.spin-sp1 { width: calc(100% - 40px); - height: calc(100% - 100px); + height: calc(100% - 60px); } &.spin-sp2 { - width: calc(100% - 240px); - height: calc(100% - 100px); + width: calc(100% - 220px); + height: calc(100% - 60px); left: 220px; } } diff --git a/dolphinscheduler-ui/src/sass/conf/home/index.scss b/dolphinscheduler-ui/src/sass/conf/home/index.scss index 76b9f6da01..16f588363d 100644 --- a/dolphinscheduler-ui/src/sass/conf/home/index.scss +++ b/dolphinscheduler-ui/src/sass/conf/home/index.scss @@ -24,8 +24,13 @@ body { .ans-message-box,.ans-message-wrapper { z-index: 121 !important; } + &::-webkit-scrollbar { + width: 0; + height: 0; + } } + @media screen and (max-width: 960px){ .nav-model { .logo-box, From fd79be39a936027bdfe7ab43ae9b9606605c5f69 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Tue, 28 Apr 2020 11:23:28 +0800 Subject: [PATCH 55/55] verify whether the filename start with '/' (#2544) * fix #2442 when the resource name contains '$',need translate it to '\$' * fix #2442 when the resource name contains '$',need translate it to '/$' * verify whether the filename start with '/' Co-authored-by: qiaozhanwei Co-authored-by: dailidong --- .../dolphinscheduler/common/utils/HadoopUtils.java | 9 +++++++++ .../common/utils/HadoopUtilsTest.java | 12 ++++++++++++ 2 files changed, 21 insertions(+) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java index 02f00ce330..963aff5f31 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java @@ -503,6 +503,9 @@ public class HadoopUtils implements Closeable { * @return hdfs file name */ public static String getHdfsFileName(ResourceType resourceType, String tenantCode, String fileName) { + if (fileName.startsWith("/")) { + fileName = fileName.replaceFirst("/",""); + } return String.format("%s/%s", getHdfsDir(resourceType,tenantCode), fileName); } @@ -514,6 +517,9 @@ public class HadoopUtils implements Closeable { * @return get absolute path and name for file on hdfs */ public static String getHdfsResourceFileName(String tenantCode, String fileName) { + if (fileName.startsWith("/")) { + fileName = fileName.replaceFirst("/",""); + } return String.format("%s/%s", getHdfsResDir(tenantCode), fileName); } @@ -525,6 +531,9 @@ public class HadoopUtils implements Closeable { * @return get absolute path and name for udf file on hdfs */ public static String getHdfsUdfFileName(String tenantCode, String fileName) { + if (fileName.startsWith("/")) { + fileName = fileName.replaceFirst("/",""); + } return String.format("%s/%s", getHdfsUdfDir(tenantCode), fileName); } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HadoopUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HadoopUtilsTest.java index 00b8f1c5c6..e239fe7cb0 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HadoopUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HadoopUtilsTest.java @@ -127,6 +127,18 @@ public class HadoopUtilsTest { Assert.assertEquals("/dolphinscheduler/11000/resources/aa.txt", result); } + @Test + public void getHdfsResourceFileName() { + String result = hadoopUtils.getHdfsResourceFileName("11000","aa.txt"); + Assert.assertEquals("/dolphinscheduler/11000/resources/aa.txt", result); + } + + @Test + public void getHdfsUdfFileName() { + String result = hadoopUtils.getHdfsFileName(ResourceType.UDF,"11000","aa.txt"); + Assert.assertEquals("/dolphinscheduler/11000/udfs/aa.txt", result); + } + @Test public void isYarnEnabled() { boolean result = hadoopUtils.isYarnEnabled();