From b854d6b77af6168e6fe300180dc151185d667319 Mon Sep 17 00:00:00 2001 From: dailidong Date: Fri, 10 Apr 2020 11:10:37 +0800 Subject: [PATCH] [Refactor worker] simplify and optimize config (#2386) * simplify config * simplify config * simplify and optimize config --- .../alert/template/AlertTemplateFactory.java | 17 +- .../alert/utils/Constants.java | 2 - .../alert/utils/MailUtils.java | 6 +- .../src/main/resources/alert.properties | 3 - .../template/AlertTemplateFactoryTest.java | 1 - .../common/utils/HadoopUtils.java | 56 +-- .../src/main/resources/common.properties | 24 +- .../src/main/resources/datasource.properties | 3 - .../main/resources/config/install_config.conf | 131 +++++- install.sh | 407 ++---------------- script/dolphinscheduler-daemon.sh | 6 +- script/scp-hosts.sh | 22 +- script/start-all.sh | 13 +- script/stop-all.sh | 14 +- 14 files changed, 228 insertions(+), 477 deletions(-) diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/AlertTemplateFactory.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/AlertTemplateFactory.java index 58e3800339..965677e7e1 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/AlertTemplateFactory.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/AlertTemplateFactory.java @@ -17,9 +17,6 @@ package org.apache.dolphinscheduler.alert.template; import org.apache.dolphinscheduler.alert.template.impl.DefaultHTMLTemplate; -import org.apache.dolphinscheduler.alert.utils.Constants; -import org.apache.dolphinscheduler.alert.utils.PropertyUtils; -import org.apache.dolphinscheduler.common.utils.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,8 +27,6 @@ public class AlertTemplateFactory { private static final Logger logger = LoggerFactory.getLogger(AlertTemplateFactory.class); - private static final String alertTemplate = PropertyUtils.getString(Constants.ALERT_TEMPLATE); - private AlertTemplateFactory(){} /** @@ -39,16 +34,6 @@ public class AlertTemplateFactory { * @return a template, default is DefaultHTMLTemplate */ public static AlertTemplate getMessageTemplate() { - - if(StringUtils.isEmpty(alertTemplate)){ - return new DefaultHTMLTemplate(); - } - - switch (alertTemplate){ - case "html": - return new DefaultHTMLTemplate(); - default: - throw new IllegalArgumentException(String.format("not support alert template: %s",alertTemplate)); - } + return new DefaultHTMLTemplate(); } } diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/Constants.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/Constants.java index 94d95b3c26..28be8aa195 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/Constants.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/Constants.java @@ -77,8 +77,6 @@ public class Constants { public static final int NUMBER_1000 = 1000; - public static final String ALERT_TEMPLATE = "alert.template"; - public static final String SPRING_DATASOURCE_DRIVER_CLASS_NAME = "spring.datasource.driver-class-name"; public static final String SPRING_DATASOURCE_URL = "spring.datasource.url"; diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java index b0aa418630..db96f8d2f3 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java @@ -262,9 +262,13 @@ public class MailUtils { part1.setContent(partContent, Constants.TEXT_HTML_CHARSET_UTF_8); // set attach file MimeBodyPart part2 = new MimeBodyPart(); + File file = new File(xlsFilePath + Constants.SINGLE_SLASH + title + Constants.EXCEL_SUFFIX_XLS); + if (!file.getParentFile().exists()) { + file.getParentFile().mkdirs(); + } // make excel file ExcelUtils.genExcelFile(content,title,xlsFilePath); - File file = new File(xlsFilePath + Constants.SINGLE_SLASH + title + Constants.EXCEL_SUFFIX_XLS); + part2.attachFile(file); part2.setFileName(MimeUtility.encodeText(title + Constants.EXCEL_SUFFIX_XLS,Constants.UTF_8,"B")); // add components to collection diff --git a/dolphinscheduler-alert/src/main/resources/alert.properties b/dolphinscheduler-alert/src/main/resources/alert.properties index db34452fb4..839eb61475 100644 --- a/dolphinscheduler-alert/src/main/resources/alert.properties +++ b/dolphinscheduler-alert/src/main/resources/alert.properties @@ -18,9 +18,6 @@ #alert type is EMAIL/SMS alert.type=EMAIL -# alter msg template, default is html template -#alert.template=html - # mail server configuration mail.protocol=SMTP mail.server.host=xxx.xxx.com diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/template/AlertTemplateFactoryTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/template/AlertTemplateFactoryTest.java index 6865b895e2..32201e6011 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/template/AlertTemplateFactoryTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/template/AlertTemplateFactoryTest.java @@ -47,7 +47,6 @@ public class AlertTemplateFactoryTest { public void testGetMessageTemplate(){ PowerMockito.mockStatic(PropertyUtils.class); - when(PropertyUtils.getString(Constants.ALERT_TEMPLATE)).thenReturn("html"); AlertTemplate defaultTemplate = AlertTemplateFactory.getMessageTemplate(); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java index 292568d60c..94c5cf8331 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java @@ -50,7 +50,8 @@ public class HadoopUtils implements Closeable { private static String hdfsUser = PropertyUtils.getString(Constants.HDFS_ROOT_USER); public static final String resourceUploadPath = PropertyUtils.getString(RESOURCE_UPLOAD_PATH, "/dolphinscheduler"); - + public static final String rmHaIds = PropertyUtils.getString(Constants.YARN_RESOURCEMANAGER_HA_RM_IDS); + public static final String appAddress = PropertyUtils.getString(Constants.YARN_APPLICATION_STATUS_ADDRESS); private static volatile HadoopUtils instance = new HadoopUtils(); private static volatile Configuration configuration; private static volatile boolean yarnEnabled = false; @@ -58,9 +59,6 @@ public class HadoopUtils implements Closeable { private HadoopUtils(){ - if(StringUtils.isEmpty(hdfsUser)){ - hdfsUser = PropertyUtils.getString(Constants.HDFS_ROOT_USER); - } init(); initHdfsPath(); } @@ -99,14 +97,15 @@ public class HadoopUtils implements Closeable { try { configuration = new Configuration(); - String resUploadStartupType = PropertyUtils.getString(Constants.RESOURCE_STORAGE_TYPE); - ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType); + String resourceStorageType = PropertyUtils.getString(Constants.RESOURCE_STORAGE_TYPE); + ResUploadType resUploadType = ResUploadType.valueOf(resourceStorageType); if (resUploadType == ResUploadType.HDFS){ if (PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE,false)){ System.setProperty(Constants.JAVA_SECURITY_KRB5_CONF, PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH)); configuration.set(Constants.HADOOP_SECURITY_AUTHENTICATION,"kerberos"); + hdfsUser = ""; UserGroupInformation.setConfiguration(configuration); UserGroupInformation.loginUserFromKeytab(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME), PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH)); @@ -155,27 +154,6 @@ public class HadoopUtils implements Closeable { fs = FileSystem.get(configuration); } - /** - * if rmHaIds includes xx, it signs not use resourcemanager - * otherwise: - * if rmHaIds is empty, single resourcemanager enabled - * if rmHaIds not empty: resourcemanager HA enabled - */ - String rmHaIds = PropertyUtils.getString(Constants.YARN_RESOURCEMANAGER_HA_RM_IDS); - String appAddress = PropertyUtils.getString(Constants.YARN_APPLICATION_STATUS_ADDRESS); - //not use resourcemanager - if (rmHaIds.contains(Constants.YARN_RESOURCEMANAGER_HA_XX)){ - yarnEnabled = false; - } else if (!StringUtils.isEmpty(rmHaIds)) { - //resourcemanager HA enabled - appAddress = getAppAddress(appAddress, rmHaIds); - yarnEnabled = true; - logger.info("appAddress : {}", appAddress); - } else { - //single resourcemanager enabled - yarnEnabled = true; - } - configuration.set(Constants.YARN_APPLICATION_STATUS_ADDRESS, appAddress); } catch (Exception e) { logger.error(e.getMessage(), e); @@ -200,7 +178,29 @@ public class HadoopUtils implements Closeable { * @return url of application */ public String getApplicationUrl(String applicationId) { - return String.format(configuration.get(Constants.YARN_APPLICATION_STATUS_ADDRESS), applicationId); + /** + * if rmHaIds contains xx, it signs not use resourcemanager + * otherwise: + * if rmHaIds is empty, single resourcemanager enabled + * if rmHaIds not empty: resourcemanager HA enabled + */ + String appUrl = ""; + //not use resourcemanager + if (rmHaIds.contains(Constants.YARN_RESOURCEMANAGER_HA_XX)){ + + yarnEnabled = false; + logger.warn("should not step here"); + } else if (!StringUtils.isEmpty(rmHaIds)) { + //resourcemanager HA enabled + appUrl = getAppAddress(appAddress, rmHaIds); + yarnEnabled = true; + logger.info("application url : {}", appUrl); + } else { + //single resourcemanager enabled + yarnEnabled = true; + } + + return String.format(appUrl, applicationId); } /** diff --git a/dolphinscheduler-common/src/main/resources/common.properties b/dolphinscheduler-common/src/main/resources/common.properties index baead1ba42..5b883b7468 100644 --- a/dolphinscheduler-common/src/main/resources/common.properties +++ b/dolphinscheduler-common/src/main/resources/common.properties @@ -19,22 +19,22 @@ resource.storage.type=NONE # resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended -#resource.upload.path=/dolphinscheduler +resource.upload.path=/dolphinscheduler # user data local directory path, please make sure the directory exists and have read write permissions #data.basedir.path=/tmp/dolphinscheduler # whether kerberos starts -#hadoop.security.authentication.startup.state=false +hadoop.security.authentication.startup.state=false # java.security.krb5.conf path -#java.security.krb5.conf.path=/opt/krb5.conf +java.security.krb5.conf.path=/opt/krb5.conf -# loginUserFromKeytab user -#login.user.keytab.username=hdfs-mycluster@ESZ.COM +# login user from keytab username +login.user.keytab.username=hdfs-mycluster@ESZ.COM # loginUserFromKeytab path -#login.user.keytab.path=/opt/hdfs.headless.keytab +login.user.keytab.path=/opt/hdfs.headless.keytab #resource.view.suffixs #resource.view.suffixs=txt,log,sh,conf,cfg,py,java,sql,hql,xml,properties @@ -46,19 +46,19 @@ hdfs.root.user=hdfs fs.defaultFS=hdfs://mycluster:8020 # if resource.storage.type=S3，s3 endpoint -#fs.s3a.endpoint=http://192.168.199.91:9010 +fs.s3a.endpoint=http://192.168.199.91:9010 # if resource.storage.type=S3，s3 access key -#fs.s3a.access.key=A3DXS30FO22544RE +fs.s3a.access.key=A3DXS30FO22544RE # if resource.storage.type=S3，s3 secret key -#fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK +fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK -# if not use hadoop resourcemanager, please keep default value; if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty TODO +# if not use hadoop resourcemanager, please keep default value; if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx -# If resourcemanager HA enable or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ark1 to actual resourcemanager hostname. +# if resourcemanager HA enable or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ark1 to actual resourcemanager hostname. yarn.application.status.address=http://ark1:8088/ws/v1/cluster/apps/%s -# system env path. self configuration, please make sure the directory and file exists and have read write execute permissions, TODO +# system env path #dolphinscheduler.env.path=env/dolphinscheduler_env.sh diff --git a/dolphinscheduler-dao/src/main/resources/datasource.properties b/dolphinscheduler-dao/src/main/resources/datasource.properties index cc7efe3b4e..8dca4ca095 100644 --- a/dolphinscheduler-dao/src/main/resources/datasource.properties +++ b/dolphinscheduler-dao/src/main/resources/datasource.properties @@ -25,9 +25,6 @@ spring.datasource.url=jdbc:mysql://localhost:3306/dolphinscheduler?useUnicode=tr spring.datasource.username=root spring.datasource.password=root@123 -## base spring data source configuration todo need to remove -#spring.datasource.type=com.alibaba.druid.pool.DruidDataSource - # connection configuration #spring.datasource.initialSize=5 # min connection number diff --git a/dolphinscheduler-server/src/main/resources/config/install_config.conf b/dolphinscheduler-server/src/main/resources/config/install_config.conf index 0378490abb..4671be7371 100644 --- a/dolphinscheduler-server/src/main/resources/config/install_config.conf +++ b/dolphinscheduler-server/src/main/resources/config/install_config.conf @@ -15,11 +15,126 @@ # limitations under the License. # -installPath=/data1_1T/dolphinscheduler -deployUser=dolphinscheduler -ips=ark0,ark1,ark2,ark3,ark4 -sshPort=22 -masters=ark0,ark1 -workers=ark2,ark3,ark4 -alertServer=ark3 -apiServers=ark1 + +# NOTICE : If the following config has special characters in the variable `.*[]^${}\+?|()@#&`, Please escape, for example, `[` escape to `\[` +# postgresql or mysql +dbtype="mysql" + +# db config +# db address and port +dbhost="192.168.xx.xx:3306" + +# db username +username="xx" + +# db passwprd +# NOTICE: if there are special characters, please use the \ to escape, for example, `[` escape to `\[` +password="xx" + +# zk cluster +zkQuorum="192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181" + +# Note: the target installation path for dolphinscheduler, please not config as the same as the current path (pwd) +installPath="/data1_1T/dolphinscheduler" + +# deployment user +# Note: the deployment user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled, the root directory needs to be created by itself +deployUser="dolphinscheduler" + + +# alert config +# mail server host +mailServerHost="smtp.exmail.qq.com" + +# mail server port +# note: Different protocols and encryption methods correspond to different ports, when SSL/TLS is enabled, make sure the port is correct. +mailServerPort="25" + +# sender +mailSender="xxxxxxxxxx" + +# user +mailUser="xxxxxxxxxx" + +# sender password +# note: The mail.passwd is email service authorization code, not the email login password. +mailPassword="xxxxxxxxxx" + +# TLS mail protocol support +starttlsEnable="false" + +sslTrust="xxxxxxxxxx" + +# SSL mail protocol support +# note: The SSL protocol is enabled by default. +# only one of TLS and SSL can be in the true state. +sslEnable="true" + + +# resource storage type：HDFS,S3,NONE +resourceStorageType="NONE" + +# if resourceStorageType is HDFS，defaultFS write namenode address，HA you need to put core-site.xml and hdfs-site.xml in the conf directory. +# if S3，write S3 address，HA，for example ：s3a://dolphinscheduler， +# Note，s3 be sure to create the root directory /dolphinscheduler +defaultFS="hdfs://mycluster:8020" + +# if resourceStorageType is S3, the following three configuration is required, otherwise please ignore +s3Endpoint="http://192.168.xx.xx:9010" +s3AccessKey="xxxxxxxxxx" +s3SecretKey="xxxxxxxxxx" + +# if not use hadoop resourcemanager, please keep default value; if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty +yarnHaIps="192.168.xx.xx,192.168.xx.xx" + +# if resourcemanager HA enable or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ark1 to actual resourcemanager hostname. +singleYarnIp="ark1" + +# resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。/dolphinscheduler is recommended +resourceUploadPath="/dolphinscheduler" + +# who have permissions to create directory under HDFS/S3 root path +# Note: if kerberos is enabled, please config hdfsRootUser= +hdfsRootUser="hdfs" + +# kerberos config +# whether kerberos starts, if kerberos starts, following four items need to config, otherwise please ignore +kerberosStartUp="false" +# kdc krb5 config file path +krb5ConfPath="$installPath/conf/krb5.conf" +# keytab username +keytabUserName="hdfs-mycluster@ESZ.COM" +# username keytab path +keytabPath="$installPath/conf/hdfs.headless.keytab" + + +# api server port +apiServerPort="12345" + + +# install hosts +# Note: install the scheduled hostname list. If it is pseudo-distributed, just write a pseudo-distributed hostname +ips="ark0,ark1,ark2,ark3,ark4" + +# ssh port, default 22 +# Note: if ssh port is not default, modify here +sshPort="22" + +# run master machine +# Note: list of hosts hostname for deploying master +masters="ark0,ark1" + +# run worker machine +# note: list of machine hostnames for deploying workers +workers="ark2,ark3,ark4" + +# run alert machine +# note: list of machine hostnames for deploying alert server +alertServer="ark3" + +# run api machine +# note: list of machine hostnames for deploying api server +apiServers="ark1" + +# whether to start monitoring self-starting scripts +monitorServerState="false" diff --git a/install.sh b/install.sh index 354cdd2be3..20b293f697 100644 --- a/install.sh +++ b/install.sh @@ -19,307 +19,25 @@ workDir=`dirname $0` workDir=`cd ${workDir};pwd` -#To be compatible with MacOS and Linux +source ${workDir}/conf/config/install_config.conf + +# 1.replace file +echo "1.replace file" + txt="" if [[ "$OSTYPE" == "darwin"* ]]; then # Mac OSX txt="''" -elif [[ "$OSTYPE" == "linux-gnu" ]]; then - # linux - txt="" -elif [[ "$OSTYPE" == "cygwin" ]]; then - # POSIX compatibility layer and Linux environment emulation for Windows - echo "DolphinScheduler not support Windows operating system" - exit 1 -elif [[ "$OSTYPE" == "msys" ]]; then - # Lightweight shell and GNU utilities compiled for Windows (part of MinGW) - echo "DolphinScheduler not support Windows operating system" - exit 1 -elif [[ "$OSTYPE" == "win32" ]]; then - echo "DolphinScheduler not support Windows operating system" - exit 1 -elif [[ "$OSTYPE" == "freebsd"* ]]; then - # ... - txt="" -else - # Unknown. - echo "Operating system unknown, please tell us(submit issue) for better service" - exit 1 -fi - -source ${workDir}/conf/config/install_config.conf - -# for example postgresql or mysql ... -dbtype="postgresql" - -# db config -# db address and port -dbhost="192.168.xx.xx:5432" - -# db name -dbname="dolphinscheduler" - -# db username -username="xx" - -# db passwprd -# Note: if there are special characters, please use the \ transfer character to transfer -passowrd="xx" - -# conf/config/install_config.conf config -# Note: the installation path is not the same as the current path (pwd) -installPath="/data1_1T/dolphinscheduler" - -# deployment user -# Note: the deployment user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled, the root directory needs to be created by itself -deployUser="dolphinscheduler" - -# zk cluster -zkQuorum="192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181" - -# install hosts -# Note: install the scheduled hostname list. If it is pseudo-distributed, just write a pseudo-distributed hostname -ips="ark0,ark1,ark2,ark3,ark4" - -# ssh port, default 22 -# Note: if ssh port is not default, modify here -sshPort=22 - -# run master machine -# Note: list of hosts hostname for deploying master -masters="ark0,ark1" - -# run worker machine -# note: list of machine hostnames for deploying workers -workers="ark2,ark3,ark4" - -# run alert machine -# note: list of machine hostnames for deploying alert server -alertServer="ark3" - -# run api machine -# note: list of machine hostnames for deploying api server -apiServers="ark1" - -# alert config -# mail protocol -mailProtocol="SMTP" - -# mail server host -mailServerHost="smtp.exmail.qq.com" - -# mail server port -mailServerPort="25" - -# sender -mailSender="xxxxxxxxxx" - -# user -mailUser="xxxxxxxxxx" - -# sender password -mailPassword="xxxxxxxxxx" - -# TLS mail protocol support -starttlsEnable="false" - -sslTrust="xxxxxxxxxx" - -# SSL mail protocol support -# note: The SSL protocol is enabled by default. -# only one of TLS and SSL can be in the true state. -sslEnable="true" - -# download excel path -xlsFilePath="/tmp/xls" - -# Enterprise WeChat Enterprise ID Configuration -enterpriseWechatCorpId="xxxxxxxxxx" - -# Enterprise WeChat application Secret configuration -enterpriseWechatSecret="xxxxxxxxxx" - -# Enterprise WeChat Application AgentId Configuration -enterpriseWechatAgentId="xxxxxxxxxx" - -# Enterprise WeChat user configuration, multiple users to , split -enterpriseWechatUsers="xxxxx,xxxxx" - - -# whether to start monitoring self-starting scripts -monitorServerState="false" - -# resource Center upload and select storage method：HDFS,S3,NONE -resUploadStartupType="NONE" - -# if resUploadStartupType is HDFS，defaultFS write namenode address，HA you need to put core-site.xml and hdfs-site.xml in the conf directory. -# if S3，write S3 address，HA，for example ：s3a://dolphinscheduler， -# Note，s3 be sure to create the root directory /dolphinscheduler -defaultFS="hdfs://mycluster:8020" - -# if S3 is configured, the following configuration is required. -s3Endpoint="http://192.168.xx.xx:9010" -s3AccessKey="xxxxxxxxxx" -s3SecretKey="xxxxxxxxxx" - -# resourcemanager HA configuration, if it is a single resourcemanager, here is yarnHaIps="" -yarnHaIps="192.168.xx.xx,192.168.xx.xx" - -# if it is a single resourcemanager, you only need to configure one host name. If it is resourcemanager HA, the default configuration is fine. -singleYarnIp="ark1" - -# hdfs root path, the owner of the root path must be the deployment user. -# versions prior to 1.1.0 do not automatically create the hdfs root directory, you need to create it yourself. -hdfsPath="/dolphinscheduler" - -# have users who create directory permissions under hdfs root path / -# Note: if kerberos is enabled, hdfsRootUser="" can be used directly. -hdfsRootUser="hdfs" - -# common config -# Program root path -programPath="/tmp/dolphinscheduler" - -# download path -downloadPath="/tmp/dolphinscheduler/download" - -# task execute path -execPath="/tmp/dolphinscheduler/exec" - -# SHELL environmental variable path -shellEnvPath="$installPath/conf/env/dolphinscheduler_env.sh" - -# suffix of the resource file -resSuffixs="txt,log,sh,conf,cfg,py,java,sql,hql,xml" - -# development status, if true, for the SHELL script, you can view the encapsulated SHELL script in the execPath directory. -# If it is false, execute the direct delete -devState="true" - -# kerberos config -# kerberos whether to start -kerberosStartUp="false" - -# kdc krb5 config file path -krb5ConfPath="$installPath/conf/krb5.conf" - -# keytab username -keytabUserName="hdfs-mycluster@ESZ.COM" - -# username keytab path -keytabPath="$installPath/conf/hdfs.headless.keytab" - -# zk config -# zk root directory -zkRoot="/dolphinscheduler" - -# zk session timeout -zkSessionTimeout="300" - -# zk connection timeout -zkConnectionTimeout="300" - -# zk retry interval -zkRetryMaxSleep="100" - -# zk retry maximum number of times -zkRetryMaxtime="5" - - -# master config -# master execution thread maximum number, maximum parallelism of process instance -masterExecThreads="100" - -# the maximum number of master task execution threads, the maximum degree of parallelism for each process instance -masterExecTaskNum="20" - -# master heartbeat interval -masterHeartbeatInterval="10" - -# master task submission retries -masterTaskCommitRetryTimes="5" - -# master task submission retry interval -masterTaskCommitInterval="1000" - -# master maximum cpu average load, used to determine whether the master has execution capability -masterMaxCpuLoadAvg="100" - -# master reserve memory to determine if the master has execution capability -masterReservedMemory="0.1" - -# worker config -# worker execution thread -workerExecThreads="100" - -# worker heartbeat interval -workerHeartbeatInterval="10" - -# worker number of fetch tasks -workerFetchTaskNum="3" - -# worker reserve memory to determine if the master has execution capability -workerReservedMemory="0.1" - -# api config -# api server port -apiServerPort="12345" - -# api session timeout -apiServerSessionTimeout="7200" - -# api server context path -apiServerContextPath="/dolphinscheduler/" - -# spring max file size -springMaxFileSize="1024MB" - -# spring max request size -springMaxRequestSize="1024MB" - -# api max http post size -apiMaxHttpPostSize="5000000" - - -# 1,replace file -echo "1,replace file" -if [ $dbtype == "mysql" ];then - sed -i ${txt} "s#spring.datasource.url.*#spring.datasource.url=jdbc:mysql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/application.properties - sed -i ${txt} "s#spring.datasource.username.*#spring.datasource.username=${username}#g" conf/application.properties - sed -i ${txt} "s#spring.datasource.password.*#spring.datasource.password=${passowrd}#g" conf/application.properties - sed -i ${txt} "s#spring.datasource.driver-class-name.*#spring.datasource.driver-class-name=com.mysql.jdbc.Driver#g" conf/application.properties - - sed -i ${txt} "s#org.quartz.dataSource.myDs.URL.*#org.quartz.dataSource.myDs.URL=jdbc:mysql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/quartz.properties - sed -i ${txt} "s#org.quartz.dataSource.myDs.user.*#org.quartz.dataSource.myDs.user=${username}#g" conf/quartz.properties - sed -i ${txt} "s#org.quartz.dataSource.myDs.password.*#org.quartz.dataSource.myDs.password=${passowrd}#g" conf/quartz.properties - sed -i ${txt} "s#org.quartz.dataSource.myDs.driver.*#org.quartz.dataSource.myDs.driver=com.mysql.jdbc.Driver#g" conf/quartz.properties - sed -i ${txt} "s#org.quartz.jobStore.driverDelegateClass.*#org.quartz.jobStore.driverDelegateClass=org.quartz.impl.jdbcjobstore.StdJDBCDelegate#g" conf/quartz.properties fi +datasourceDriverClassname="com.mysql.jdbc.Driver" if [ $dbtype == "postgresql" ];then - sed -i ${txt} "s#spring.datasource.url.*#spring.datasource.url=jdbc:postgresql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/application.properties - sed -i ${txt} "s#spring.datasource.username.*#spring.datasource.username=${username}#g" conf/application.properties - sed -i ${txt} "s#spring.datasource.password.*#spring.datasource.password=${passowrd}#g" conf/application.properties - sed -i ${txt} "s#spring.datasource.driver-class-name.*#spring.datasource.driver-class-name=org.postgresql.Driver#g" conf/application.properties - - sed -i ${txt} "s#org.quartz.dataSource.myDs.URL.*#org.quartz.dataSource.myDs.URL=jdbc:postgresql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/quartz.properties - sed -i ${txt} "s#org.quartz.dataSource.myDs.user.*#org.quartz.dataSource.myDs.user=${username}#g" conf/quartz.properties - sed -i ${txt} "s#org.quartz.dataSource.myDs.password.*#org.quartz.dataSource.myDs.password=${passowrd}#g" conf/quartz.properties - sed -i ${txt} "s#org.quartz.dataSource.myDs.driver.*#org.quartz.dataSource.myDs.driver=org.postgresql.Driver#g" conf/quartz.properties - sed -i ${txt} "s#org.quartz.jobStore.driverDelegateClass.*#org.quartz.jobStore.driverDelegateClass=org.quartz.impl.jdbcjobstore.PostgreSQLDelegate#g" conf/quartz.properties + datasourceDriverClassname="org.postgresql.Driver" fi - -sed -i ${txt} "s#master.exec.threads.*#master.exec.threads=${masterExecThreads}#g" conf/application.properties -sed -i ${txt} "s#master.exec.task.num.*#master.exec.task.num=${masterExecTaskNum}#g" conf/application.properties -sed -i ${txt} "s#master.heartbeat.interval.*#master.heartbeat.interval=${masterHeartbeatInterval}#g" conf/application.properties -sed -i ${txt} "s#master.task.commit.retryTimes.*#master.task.commit.retryTimes=${masterTaskCommitRetryTimes}#g" conf/application.properties -sed -i ${txt} "s#master.task.commit.interval.*#master.task.commit.interval=${masterTaskCommitInterval}#g" conf/application.properties -sed -i ${txt} "s#master.reserved.memory.*#master.reserved.memory=${masterReservedMemory}#g" conf/application.properties - -sed -i ${txt} "s#worker.exec.threads.*#worker.exec.threads=${workerExecThreads}#g" conf/application.properties -sed -i ${txt} "s#worker.heartbeat.interval.*#worker.heartbeat.interval=${workerHeartbeatInterval}#g" conf/application.properties -sed -i ${txt} "s#worker.fetch.task.num.*#worker.fetch.task.num=${workerFetchTaskNum}#g" conf/application.properties -sed -i ${txt} "s#worker.reserved.memory.*#worker.reserved.memory=${workerReservedMemory}#g" conf/application.properties +sed -i ${txt} "s#spring.datasource.driver-class-name.*#spring.datasource.driver-class-name=${datasourceDriverClassname}#g" conf/datasource.properties +sed -i ${txt} "s#spring.datasource.url.*#spring.datasource.url=jdbc:${dbtype}://${dbhost}/dolphinscheduler?characterEncoding=UTF-8#g" conf/datasource.properties +sed -i ${txt} "s#spring.datasource.username.*#spring.datasource.username=${username}#g" conf/datasource.properties +sed -i ${txt} "s#spring.datasource.password.*#spring.datasource.password=${password}#g" conf/datasource.properties sed -i ${txt} "s#fs.defaultFS.*#fs.defaultFS=${defaultFS}#g" conf/common.properties sed -i ${txt} "s#fs.s3a.endpoint.*#fs.s3a.endpoint=${s3Endpoint}#g" conf/common.properties @@ -327,37 +45,18 @@ sed -i ${txt} "s#fs.s3a.access.key.*#fs.s3a.access.key=${s3AccessKey}#g" conf/co sed -i ${txt} "s#fs.s3a.secret.key.*#fs.s3a.secret.key=${s3SecretKey}#g" conf/common.properties sed -i ${txt} "s#yarn.resourcemanager.ha.rm.ids.*#yarn.resourcemanager.ha.rm.ids=${yarnHaIps}#g" conf/common.properties sed -i ${txt} "s#yarn.application.status.address.*#yarn.application.status.address=http://${singleYarnIp}:8088/ws/v1/cluster/apps/%s#g" conf/common.properties - -sed -i ${txt} "s#data.basedir.path.*#data.basedir.path=${programPath}#g" conf/common.properties -sed -i ${txt} "s#data.download.basedir.path.*#data.download.basedir.path=${downloadPath}#g" conf/common.properties -sed -i ${txt} "s#process.exec.basepath.*#process.exec.basepath=${execPath}#g" conf/common.properties sed -i ${txt} "s#hdfs.root.user.*#hdfs.root.user=${hdfsRootUser}#g" conf/common.properties -sed -i ${txt} "s#data.store2hdfs.basepath.*#data.store2hdfs.basepath=${hdfsPath}#g" conf/common.properties -sed -i ${txt} "s#res.upload.startup.type.*#res.upload.startup.type=${resUploadStartupType}#g" conf/common.properties -sed -i ${txt} "s#dolphinscheduler.env.path.*#dolphinscheduler.env.path=${shellEnvPath}#g" conf/common.properties -sed -i ${txt} "s#resource.view.suffixs.*#resource.view.suffixs=${resSuffixs}#g" conf/common.properties -sed -i ${txt} "s#development.state.*#development.state=${devState}#g" conf/common.properties +sed -i ${txt} "s#resource.upload.path.*#resource.upload.path=${resourceUploadPath}#g" conf/common.properties +sed -i ${txt} "s#resource.storage.type.*#resource.storage.type=${resourceStorageType}#g" conf/common.properties sed -i ${txt} "s#hadoop.security.authentication.startup.state.*#hadoop.security.authentication.startup.state=${kerberosStartUp}#g" conf/common.properties sed -i ${txt} "s#java.security.krb5.conf.path.*#java.security.krb5.conf.path=${krb5ConfPath}#g" conf/common.properties sed -i ${txt} "s#login.user.keytab.username.*#login.user.keytab.username=${keytabUserName}#g" conf/common.properties sed -i ${txt} "s#login.user.keytab.path.*#login.user.keytab.path=${keytabPath}#g" conf/common.properties - sed -i ${txt} "s#zookeeper.quorum.*#zookeeper.quorum=${zkQuorum}#g" conf/common.properties -sed -i ${txt} "s#zookeeper.dolphinscheduler.root.*#zookeeper.dolphinscheduler.root=${zkRoot}#g" conf/common.properties -sed -i ${txt} "s#zookeeper.session.timeout.*#zookeeper.session.timeout=${zkSessionTimeout}#g" conf/common.properties -sed -i ${txt} "s#zookeeper.connection.timeout.*#zookeeper.connection.timeout=${zkConnectionTimeout}#g" conf/common.properties -sed -i ${txt} "s#zookeeper.retry.max.sleep.*#zookeeper.retry.max.sleep=${zkRetryMaxSleep}#g" conf/common.properties -sed -i ${txt} "s#zookeeper.retry.maxtime.*#zookeeper.retry.maxtime=${zkRetryMaxtime}#g" conf/common.properties sed -i ${txt} "s#server.port.*#server.port=${apiServerPort}#g" conf/application-api.properties -sed -i ${txt} "s#server.servlet.session.timeout.*#server.servlet.session.timeout=${apiServerSessionTimeout}#g" conf/application-api.properties -sed -i ${txt} "s#server.servlet.context-path.*#server.servlet.context-path=${apiServerContextPath}#g" conf/application-api.properties -sed -i ${txt} "s#spring.servlet.multipart.max-file-size.*#spring.servlet.multipart.max-file-size=${springMaxFileSize}#g" conf/application-api.properties -sed -i ${txt} "s#spring.servlet.multipart.max-request-size.*#spring.servlet.multipart.max-request-size=${springMaxRequestSize}#g" conf/application-api.properties -sed -i ${txt} "s#server.jetty.max-http-post-size.*#server.jetty.max-http-post-size=${apiMaxHttpPostSize}#g" conf/application-api.properties -sed -i ${txt} "s#mail.protocol.*#mail.protocol=${mailProtocol}#g" conf/alert.properties sed -i ${txt} "s#mail.server.host.*#mail.server.host=${mailServerHost}#g" conf/alert.properties sed -i ${txt} "s#mail.server.port.*#mail.server.port=${mailServerPort}#g" conf/alert.properties sed -i ${txt} "s#mail.sender.*#mail.sender=${mailSender}#g" conf/alert.properties @@ -366,82 +65,38 @@ sed -i ${txt} "s#mail.passwd.*#mail.passwd=${mailPassword}#g" conf/alert.propert sed -i ${txt} "s#mail.smtp.starttls.enable.*#mail.smtp.starttls.enable=${starttlsEnable}#g" conf/alert.properties sed -i ${txt} "s#mail.smtp.ssl.trust.*#mail.smtp.ssl.trust=${sslTrust}#g" conf/alert.properties sed -i ${txt} "s#mail.smtp.ssl.enable.*#mail.smtp.ssl.enable=${sslEnable}#g" conf/alert.properties -sed -i ${txt} "s#xls.file.path.*#xls.file.path=${xlsFilePath}#g" conf/alert.properties -sed -i ${txt} "s#enterprise.wechat.corp.id.*#enterprise.wechat.corp.id=${enterpriseWechatCorpId}#g" conf/alert.properties -sed -i ${txt} "s#enterprise.wechat.secret.*#enterprise.wechat.secret=${enterpriseWechatSecret}#g" conf/alert.properties -sed -i ${txt} "s#enterprise.wechat.agent.id.*#enterprise.wechat.agent.id=${enterpriseWechatAgentId}#g" conf/alert.properties -sed -i ${txt} "s#enterprise.wechat.users.*#enterprise.wechat.users=${enterpriseWechatUsers}#g" conf/alert.properties - - -sed -i ${txt} "s#installPath.*#installPath=${installPath}#g" conf/config/install_config.conf -sed -i ${txt} "s#deployUser.*#deployUser=${deployUser}#g" conf/config/install_config.conf -sed -i ${txt} "s#ips.*#ips=${ips}#g" conf/config/install_config.conf -sed -i ${txt} "s#sshPort.*#sshPort=${sshPort}#g" conf/config/install_config.conf - - -sed -i ${txt} "s#masters.*#masters=${masters}#g" conf/config/install_config.conf -sed -i ${txt} "s#workers.*#workers=${workers}#g" conf/config/install_config.conf -sed -i ${txt} "s#alertServer.*#alertServer=${alertServer}#g" conf/config/install_config.conf -sed -i ${txt} "s#apiServers.*#apiServers=${apiServers}#g" conf/config/install_config.conf -sed -i ${txt} "s#sshPort.*#sshPort=${sshPort}#g" conf/config/install_config.conf - - -# 2,create directory -echo "2,create directory" +# 2.create directory +echo "2.create directory" if [ ! -d $installPath ];then sudo mkdir -p $installPath sudo chown -R $deployUser:$deployUser $installPath fi -hostsArr=(${ips//,/ }) -for host in ${hostsArr[@]} -do - -# create if programPath does not exist -if ! ssh -p $sshPort $host test -e $programPath; then - ssh -p $sshPort $host "sudo mkdir -p $programPath;sudo chown -R $deployUser:$deployUser $programPath" -fi - -# create if downloadPath does not exist -if ! ssh -p $sshPort $host test -e $downloadPath; then - ssh -p $sshPort $host "sudo mkdir -p $downloadPath;sudo chown -R $deployUser:$deployUser $downloadPath" -fi - -# create if execPath does not exist -if ! ssh -p $sshPort $host test -e $execPath; then - ssh -p $sshPort $host "sudo mkdir -p $execPath; sudo chown -R $deployUser:$deployUser $execPath" -fi - -# create if xlsFilePath does not exist -if ! ssh -p $sshPort $host test -e $xlsFilePath; then - ssh -p $sshPort $host "sudo mkdir -p $xlsFilePath; sudo chown -R $deployUser:$deployUser $xlsFilePath" +# 3.scp resources +echo "3.scp resources" +sh ${workDir}/script/scp-hosts.sh +if [ $? -eq 0 ] +then + echo 'scp copy completed' +else + echo 'scp copy failed to exit' + exit 1 fi -done - -# 3,stop server -echo "3,stop server" +# 4.stop server +echo "4.stop server" sh ${workDir}/script/stop-all.sh -# 4,delete zk node -echo "4,delete zk node" + +# 5.delete zk node +echo "5.delete zk node" sh ${workDir}/script/remove-zk-node.sh $zkRoot -# 5,scp resources -echo "5,scp resources" -sh ${workDir}/script/scp-hosts.sh -if [ $? -eq 0 ] -then - echo 'scp copy completed' -else - echo 'scp copy failed to exit' - exit -1 -fi -# 6,startup -echo "6,startup" +# 6.startup +echo "6.startup" sh ${workDir}/script/start-all.sh \ No newline at end of file diff --git a/script/dolphinscheduler-daemon.sh b/script/dolphinscheduler-daemon.sh index d7ef66f8bd..5fb50b86ca 100644 --- a/script/dolphinscheduler-daemon.sh +++ b/script/dolphinscheduler-daemon.sh @@ -35,6 +35,8 @@ BIN_DIR=`dirname $0` BIN_DIR=`cd "$BIN_DIR"; pwd` DOLPHINSCHEDULER_HOME=$BIN_DIR/.. +source /etc/profile + export JAVA_HOME=$JAVA_HOME #export JAVA_HOME=/opt/soft/jdk export HOSTNAME=`hostname` @@ -90,8 +92,8 @@ case $startStop in exec_command="$LOG_FILE $DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS" - echo "nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 > /dev/null &" - nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 > /dev/null & + echo "nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 &" + nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 & echo $! > $pid ;; diff --git a/script/scp-hosts.sh b/script/scp-hosts.sh index 05878c3c51..f4949f311a 100644 --- a/script/scp-hosts.sh +++ b/script/scp-hosts.sh @@ -24,16 +24,18 @@ hostsArr=(${ips//,/ }) for host in ${hostsArr[@]} do - if ! ssh -p $sshPort $host test -e $installPath; then - ssh -p $sshPort $host "sudo mkdir -p $installPath; sudo chown -R $deployUser:$deployUser $installPath" - fi + if ! ssh -p $sshPort $host test -e $installPath; then + ssh -p $sshPort $host "sudo mkdir -p $installPath; sudo chown -R $deployUser:$deployUser $installPath" + fi + echo "scp dirs to $host/$installPath starting" ssh -p $sshPort $host "cd $installPath/; rm -rf bin/ conf/ lib/ script/ sql/ ui/" - scp -P $sshPort -r $workDir/../bin $host:$installPath - scp -P $sshPort -r $workDir/../conf $host:$installPath - scp -P $sshPort -r $workDir/../lib $host:$installPath - scp -P $sshPort -r $workDir/../script $host:$installPath - scp -P $sshPort -r $workDir/../sql $host:$installPath - scp -P $sshPort -r $workDir/../ui $host:$installPath - scp -P $sshPort $workDir/../install.sh $host:$installPath + + for dsDir in bin conf lib script sql ui install.sh + do + echo "start to scp $dsDir to $host/$installPath" + scp -P $sshPort -r $workDir/../$dsDir $host:$installPath + done + + echo "scp dirs to $host/$installPath complete" done diff --git a/script/start-all.sh b/script/start-all.sh index bb4b0a1fcb..11e4572059 100644 --- a/script/start-all.sh +++ b/script/start-all.sh @@ -23,7 +23,7 @@ source $workDir/../conf/config/install_config.conf mastersHost=(${masters//,/ }) for master in ${mastersHost[@]} do - echo $master + echo "$master master server is starting" ssh -p $sshPort $master "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start master-server;" done @@ -31,10 +31,10 @@ done workersHost=(${workers//,/ }) for worker in ${workersHost[@]} do - echo $worker + echo "$worker worker server is starting" - ssh -p $sshPort $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start worker-server;" - ssh -p $sshPort $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start logger-server;" + ssh -p $sshPort $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start worker-server;" + ssh -p $sshPort $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start logger-server;" done ssh -p $sshPort $alertServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start alert-server;" @@ -42,8 +42,7 @@ ssh -p $sshPort $alertServer "cd $installPath/; sh bin/dolphinscheduler-daemon. apiServersHost=(${apiServers//,/ }) for apiServer in ${apiServersHost[@]} do - echo $apiServer - - ssh -p $sshPort $apiServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start api-server;" + echo "$apiServer worker server is starting" + ssh -p $sshPort $apiServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start api-server;" done diff --git a/script/stop-all.sh b/script/stop-all.sh index c0c6f4dd45..f761579cc8 100644 --- a/script/stop-all.sh +++ b/script/stop-all.sh @@ -24,7 +24,7 @@ source $workDir/../conf/config/install_config.conf mastersHost=(${masters//,/ }) for master in ${mastersHost[@]} do - echo $master + echo "$master master server is stopping" ssh -p $sshPort $master "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop master-server;" done @@ -32,10 +32,9 @@ done workersHost=(${workers//,/ }) for worker in ${workersHost[@]} do - echo $worker - - ssh -p $sshPort $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop worker-server;" - ssh -p $sshPort $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop logger-server;" + echo "$worker worker server is stopping" + ssh -p $sshPort $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop worker-server;" + ssh -p $sshPort $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop logger-server;" done ssh -p $sshPort $alertServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop alert-server;" @@ -43,8 +42,7 @@ ssh -p $sshPort $alertServer "cd $installPath/; sh bin/dolphinscheduler-daemon. apiServersHost=(${apiServers//,/ }) for apiServer in ${apiServersHost[@]} do - echo $apiServer - - ssh -p $sshPort $apiServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop api-server;" + echo "$apiServer worker server is stopping" + ssh -p $sshPort $apiServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop api-server;" done