Fix install.sh for DS 2.0 and add comment to install_config.conf (#6506)

* Fix install.sh for DS 2.0

* Use #* make it work in macos

* Add comment for properties change

* Add JAVA HOME for only one file change to deploy

* Reduce config change by using default value of: plugin registry, alert, task as

* Change javaHome default value

* Delete config param zkQuorum

* Reorder param

* Add comment for install_config.conf

dolphinscheduler-server/src/main/resources/config/install_config.conf

@@ -15,54 +15,109 @@
 # limitations under the License.
 #
 
+# ---------------------------------------------------------
+# INSTALL MACHINE
+# ---------------------------------------------------------
+# A comma separated list of machine hostname or IP would be installed DolphinScheduler,
+# including master, worker, api, alert. If you want to deploy in pseudo-distributed
+# mode, just write a pseudo-distributed hostname
+# Example for hostnames: ips="ds1,ds2,ds3,ds4,ds5", Example for IP: ips="192.168.8.1,192.168.8.2,192.168.8.3,192.168.8.4,192.168.8.5"
+ips="ds1,ds2,ds3,ds4,ds5"
+
+# Port of SSH protocol, default value is 22. For now we only support same port in all `ips` machine
+# modify it if you use different ssh port
+sshPort="22"
+
+# A comma separated list of machine hostname or IP would be installed Master server, it
+# must be a subset of configuration `ips`.
+# Example for hostnames: ips="ds1,ds2", Example for IP: ips="192.168.8.1,192.168.8.2"
+masters="ds1,ds2"
+
+# A comma separated list of machine <hostname>:<workerGroup> or <IP>:<workerGroup>.All hostname or IP must be a
+# subset of configuration `ips`, And workerGroup have default value as `default`, but we recommend you declare behind the hosts
+# Example for hostnames: ips="ds1:default,ds2:default,ds3:default", Example for IP: ips="192.168.8.1:default,192.168.8.2:default,192.168.8.3:default"
+workers="ds1:default,ds2:default,ds3:default,ds4:default,ds5:default"
+
+# A comma separated list of machine hostname or IP would be installed Alert server, it
+# must be a subset of configuration `ips`.
+# Example for hostnames: ips="ds3", Example for IP: ips="192.168.8.3"
+alertServer="ds3"
 
-# NOTICE: If the following config has special characters in the variable `.*[]^${}\+?|()@#&`, Please escape, for example, `[` escape to `\[`
+# A comma separated list of machine hostname or IP would be installed API server, it
-# postgresql or mysql
+# must be a subset of configuration `ips`.
+# Example for hostnames: ips="ds1", Example for IP: ips="192.168.8.1"
+apiServers="ds1"
+
+# The directory to install DolphinScheduler for all machine we config above. It will automatically created by `install.sh` script if not exists.
+# **DO NOT** set this configuration same as the current path (pwd)
+installPath="/data1_1T/dolphinscheduler"
+
+# The user to deploy DolphinScheduler for all machine we config above. For now user must create by yourself before run `install.sh`
+# script. The user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled than the root directory needs
+# to be created by this user
+deployUser="dolphinscheduler"
+
+# The directory to store local data for all machine we config above. Make sure user `deployUser` have permissions to read and write this directory.
+dataBasedirPath="/tmp/dolphinscheduler"
+
+# ---------------------------------------------------------
+# DolphinScheduler ENV
+# ---------------------------------------------------------
+# JAVA_HOME, we recommend use same JAVA_HOME in all machine you going to install DolphinScheduler
+# and this configuration only support one parameter so far.
+javaHome="/your/java/home/here"
+
+# DolphinScheduler API service port, also this your DolphinScheduler UI component's URL port, default values is 12345
+apiServerPort="12345"
+
+# ---------------------------------------------------------
+# Database
+# NOTICE: If database value has special characters, such as `.*[]^${}\+?|()@#&`, Please add prefix `\` for escaping.
+# ---------------------------------------------------------
+# The type for the metadata database
+# Supported values: ``postgresql``, ``mysql``.
 dbtype="mysql"
 
-# db config
+# The <HOST>:<PORT> connection pair DolphinScheduler connect to the metadata database
-# db address and port
 dbhost="192.168.xx.xx:3306"
 
-# db username
+# The username DolphinScheduler connect to the metadata database
 username="xx"
 
-# db password
+# The password DolphinScheduler connect to the metadata database
-# NOTICE: if there are special characters, please use the \ to escape, for example, `[` escape to `\[`
 password="xx"
 
-# database name
+# The database DolphinScheduler connect to the metadata database
 dbname="dolphinscheduler"
 
+# ---------------------------------------------------------
+# Registry Server
+# ---------------------------------------------------------
+# Registry Server plugin dir. DolphinScheduler will find and load the registry plugin jar package from this dir.
+# For now default registry server is zookeeper, so the default value is `lib/plugin/registry/zookeeper`.
+# If you want to implement your own registry server, please see https://dolphinscheduler.apache.org/en-us/docs/dev/user_doc/registry_spi.html
+registryPluginDir="lib/plugin/registry/zookeeper"
 
-# zk cluster
+# Registry Server plugin name, should be a substring of `registryPluginDir`, DolphinScheduler use this for verifying configuration consistency
-zkQuorum="192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181"
-
-# zk root directory
-zkRoot="/dolphinscheduler"
-
-# registry config
-# registry plugin dir
-# Note: find and load the Registry Plugin Jar from this dir.
-registryPluginDir="/data1_1T/dolphinscheduler/lib/plugin/registry"
 registryPluginName="zookeeper"
-registryServers="192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181"
-
 
-# Note: the target installation path for dolphinscheduler, please not config as the same as the current path (pwd)
+# Registry Server address.
-installPath="/data1_1T/dolphinscheduler"
+registryServers="192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181"
 
-# deployment user
+# The root of zookeeper, for now DolphinScheduler default registry server is zookeeper.
-# Note: the deployment user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled, the root directory needs to be created by itself
+zkRoot="/dolphinscheduler"
-deployUser="dolphinscheduler"
 
-# alert config
+# ---------------------------------------------------------
-# alert plugin dir
+# Alert Server
-# Note: find and load the Alert Plugin Jar from this dir.
+# ---------------------------------------------------------
-alertPluginDir="/data1_1T/dolphinscheduler/lib/plugin/alert"
+# Alert Server plugin dir. DolphinScheduler will find and load the alert plugin jar package from this dir.
+alertPluginDir="lib/plugin/alert"
 
-# user data local directory path, please make sure the directory exists and have read write permissions
+# ---------------------------------------------------------
-dataBasedirPath="/tmp/dolphinscheduler"
+# Worker Task Server
+# ---------------------------------------------------------
+# Worker Task Server plugin dir. DolphinScheduler will find and load the worker task plugin jar package from this dir.
+taskPluginDir="lib/plugin/task"
 
 # resource storage type: HDFS, S3, NONE
 resourceStorageType="NONE"
@@ -110,34 +165,3 @@ sudoEnable="true"
 
 # worker tenant auto create
 workerTenantAutoCreate="false"
-
-# worker task plugin dir
-taskPluginDir="/data1_1T/dolphinscheduler/lib/plugin/task"
-
-# api server port
-apiServerPort="12345"
-
-
-# install hosts
-# Note: install the scheduled hostname list. If it is pseudo-distributed, just write a pseudo-distributed hostname
-ips="ds1,ds2,ds3,ds4,ds5"
-
-# ssh port, default 22
-# Note: if ssh port is not default, modify here
-sshPort="22"
-
-# run master machine
-# Note: list of hosts hostname for deploying master
-masters="ds1,ds2"
-
-# run worker machine
-# note: need to write the worker group name of each worker, the default value is "default"
-workers="ds1:default,ds2:default,ds3:default,ds4:default,ds5:default"
-
-# run alert machine
-# note: list of machine hostnames for deploying alert server
-alertServer="ds3"
-
-# run api machine
-# note: list of machine hostnames for deploying api server
-apiServers="ds1"

install.sh

@@ -34,11 +34,17 @@ datasourceDriverClassname="com.mysql.jdbc.Driver"
 if [ $dbtype == "postgresql" ];then
   datasourceDriverClassname="org.postgresql.Driver"
 fi
+
+# Change configuration in conf/config/dolphinscheduler_env.sh
+sed -i ${txt} "s@^export JAVA_HOME=.*@export JAVA_HOME=${javaHome}@g" conf/env/dolphinscheduler_env.sh
+
+# Change configuration in conf/datasource.properties
 sed -i ${txt} "s@^spring.datasource.driver-class-name=.*@spring.datasource.driver-class-name=${datasourceDriverClassname}@g" conf/datasource.properties
 sed -i ${txt} "s@^spring.datasource.url=.*@spring.datasource.url=jdbc:${dbtype}://${dbhost}/${dbname}?characterEncoding=UTF-8\&allowMultiQueries=true@g" conf/datasource.properties
 sed -i ${txt} "s@^spring.datasource.username=.*@spring.datasource.username=${username}@g" conf/datasource.properties
 sed -i ${txt} "s@^spring.datasource.password=.*@spring.datasource.password=${password}@g" conf/datasource.properties
 
+# Change configuration in conf/common.properties
 sed -i ${txt} "s@^data.basedir.path=.*@data.basedir.path=${dataBasedirPath}@g" conf/common.properties
 sed -i ${txt} "s@^resource.storage.type=.*@resource.storage.type=${resourceStorageType}@g" conf/common.properties
 sed -i ${txt} "s@^resource.upload.path=.*@resource.upload.path=${resourceUploadPath}@g" conf/common.properties
@@ -58,17 +64,22 @@ sed -i ${txt} "s@^yarn.application.status.address=.*@yarn.application.status.add
 sed -i ${txt} "s@^yarn.job.history.status.address=.*@yarn.job.history.status.address=http://${singleYarnIp}:19888/ws/v1/history/mapreduce/jobs/%s@g" conf/common.properties
 sed -i ${txt} "s@^sudo.enable=.*@sudo.enable=${sudoEnable}@g" conf/common.properties
 
-# the following configurations may be commented, so ddd #\? to ensure successful sed
+# The following configurations may be commented, so ddd #* to ensure sed work correct
-sed -i ${txt} "s@^#\?worker.tenant.auto.create=.*@worker.tenant.auto.create=${workerTenantAutoCreate}@g" conf/worker.properties
+# Change configuration in conf/worker.properties
-sed -i ${txt} "s@^#\?alert.listen.host=.*@alert.listen.host=${alertServer}@g" conf/worker.properties
+sed -i ${txt} "s@^#*worker.tenant.auto.create=.*@worker.tenant.auto.create=${workerTenantAutoCreate}@g" conf/worker.properties
-sed -i ${txt} "s@^#\?task.plugin.dir=.*@task.plugin.dir=${taskPluginDir}@g" conf/worker.properties
+sed -i ${txt} "s@^#*alert.listen.host=.*@alert.listen.host=${alertServer}@g" conf/worker.properties
+sed -i ${txt} "s@^#*task.plugin.dir=.*@task.plugin.dir=${installPath}/${taskPluginDir}@g" conf/worker.properties
+
+# Change configuration in conf/alert.properties
+sed -i ${txt} "s@^#*alert.plugin.dir=.*@alert.plugin.dir=${installPath}/${alertPluginDir}@g" conf/alert.properties
 
-sed -i ${txt} "s@^#\?alert.plugin.dir=.*@alert.plugin.dir=${alertPluginDir}@g" conf/alert.properties
+# Change configuration in conf/application-api.properties
-sed -i ${txt} "s@^#\?server.port=.*@server.port=${apiServerPort}@g" conf/application-api.properties
+sed -i ${txt} "s@^#*server.port=.*@server.port=${apiServerPort}@g" conf/application-api.properties
 
-sed -i ${txt} "s@^#\?registry.plugin.dir=.*@registry.plugin.dir=${registryPluginDir}@g" conf/registry.properties
+# Change configuration in conf/registry.properties
-sed -i ${txt} "s@^#\?registry.plugin.name=.*@registry.plugin.name=${registryPluginName}@g" conf/registry.properties
+sed -i ${txt} "s@^#*registry.plugin.dir=.*@registry.plugin.dir=${installPath}/${registryPluginDir}@g" conf/registry.properties
-sed -i ${txt} "s@^#\?registry.servers=.*@registry.servers=${registryServers}@g" conf/registry.properties
+sed -i ${txt} "s@^#*registry.plugin.name=.*@registry.plugin.name=${registryPluginName}@g" conf/registry.properties
+sed -i ${txt} "s@^#*registry.servers=.*@registry.servers=${registryServers}@g" conf/registry.properties
 
 # 2.create directory
 echo "2.create directory"

script/dolphinscheduler-daemon.sh

@@ -36,6 +36,7 @@ BIN_DIR=`cd "$BIN_DIR"; pwd`
 DOLPHINSCHEDULER_HOME=`cd "$BIN_DIR/.."; pwd`
 
 source /etc/profile
+source "${DOLPHINSCHEDULER_HOME}/conf/env/dolphinscheduler_env.sh"
 
 export JAVA_HOME=$JAVA_HOME
 #export JAVA_HOME=/opt/soft/jdk

script/remove-zk-node.sh

@@ -32,6 +32,7 @@ BIN_DIR=`cd "$BIN_DIR"; pwd`
 DOLPHINSCHEDULER_HOME=$BIN_DIR/..
 
 source ${BIN_DIR}/../conf/config/install_config.conf
+source ${BIN_DIR}/../conf/env/dolphinscheduler_env.sh
 
 export JAVA_HOME=$JAVA_HOME
 
@@ -44,7 +45,7 @@ export STOP_TIMEOUT=5
 
 CLASS=org.apache.zookeeper.ZooKeeperMain
 
-exec_command="$DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS -server $zkQuorum rmr $rootNode"
+exec_command="$DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS -server $registryServers rmr $rootNode"
 
 cd $DOLPHINSCHEDULER_HOME
 $JAVA_HOME/bin/java $exec_command

script/scp-hosts.sh

@@ -57,7 +57,8 @@ do
     fi
 
     echo "start to scp $dsDir to $host/$installPath"
-    scp -P $sshPort -r $workDir/../$dsDir  $host:$installPath
+    # Use quiet mode to reduce command line output
+    scp -q -P $sshPort -r $workDir/../$dsDir  $host:$installPath
   done
 
   echo "scp dirs to $host/$installPath complete"