diff --git a/dolphinscheduler-server/src/main/resources/config/install_config.conf b/dolphinscheduler-server/src/main/resources/config/install_config.conf index 9706897aa6..73282acc00 100755 --- a/dolphinscheduler-server/src/main/resources/config/install_config.conf +++ b/dolphinscheduler-server/src/main/resources/config/install_config.conf @@ -15,54 +15,109 @@ # limitations under the License. # +# --------------------------------------------------------- +# INSTALL MACHINE +# --------------------------------------------------------- +# A comma separated list of machine hostname or IP would be installed DolphinScheduler, +# including master, worker, api, alert. If you want to deploy in pseudo-distributed +# mode, just write a pseudo-distributed hostname +# Example for hostnames: ips="ds1,ds2,ds3,ds4,ds5", Example for IP: ips="192.168.8.1,192.168.8.2,192.168.8.3,192.168.8.4,192.168.8.5" +ips="ds1,ds2,ds3,ds4,ds5" + +# Port of SSH protocol, default value is 22. For now we only support same port in all `ips` machine +# modify it if you use different ssh port +sshPort="22" + +# A comma separated list of machine hostname or IP would be installed Master server, it +# must be a subset of configuration `ips`. +# Example for hostnames: ips="ds1,ds2", Example for IP: ips="192.168.8.1,192.168.8.2" +masters="ds1,ds2" + +# A comma separated list of machine : or :.All hostname or IP must be a +# subset of configuration `ips`, And workerGroup have default value as `default`, but we recommend you declare behind the hosts +# Example for hostnames: ips="ds1:default,ds2:default,ds3:default", Example for IP: ips="192.168.8.1:default,192.168.8.2:default,192.168.8.3:default" +workers="ds1:default,ds2:default,ds3:default,ds4:default,ds5:default" + +# A comma separated list of machine hostname or IP would be installed Alert server, it +# must be a subset of configuration `ips`. +# Example for hostnames: ips="ds3", Example for IP: ips="192.168.8.3" +alertServer="ds3" + +# A comma separated list of machine hostname or IP would be installed API server, it +# must be a subset of configuration `ips`. +# Example for hostnames: ips="ds1", Example for IP: ips="192.168.8.1" +apiServers="ds1" + +# The directory to install DolphinScheduler for all machine we config above. It will automatically created by `install.sh` script if not exists. +# **DO NOT** set this configuration same as the current path (pwd) +installPath="/data1_1T/dolphinscheduler" + +# The user to deploy DolphinScheduler for all machine we config above. For now user must create by yourself before run `install.sh` +# script. The user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled than the root directory needs +# to be created by this user +deployUser="dolphinscheduler" + +# The directory to store local data for all machine we config above. Make sure user `deployUser` have permissions to read and write this directory. +dataBasedirPath="/tmp/dolphinscheduler" + +# --------------------------------------------------------- +# DolphinScheduler ENV +# --------------------------------------------------------- +# JAVA_HOME, we recommend use same JAVA_HOME in all machine you going to install DolphinScheduler +# and this configuration only support one parameter so far. +javaHome="/your/java/home/here" -# NOTICE: If the following config has special characters in the variable `.*[]^${}\+?|()@#&`, Please escape, for example, `[` escape to `\[` -# postgresql or mysql +# DolphinScheduler API service port, also this your DolphinScheduler UI component's URL port, default values is 12345 +apiServerPort="12345" + +# --------------------------------------------------------- +# Database +# NOTICE: If database value has special characters, such as `.*[]^${}\+?|()@#&`, Please add prefix `\` for escaping. +# --------------------------------------------------------- +# The type for the metadata database +# Supported values: ``postgresql``, ``mysql``. dbtype="mysql" -# db config -# db address and port +# The : connection pair DolphinScheduler connect to the metadata database dbhost="192.168.xx.xx:3306" -# db username +# The username DolphinScheduler connect to the metadata database username="xx" -# db password -# NOTICE: if there are special characters, please use the \ to escape, for example, `[` escape to `\[` +# The password DolphinScheduler connect to the metadata database password="xx" -# database name +# The database DolphinScheduler connect to the metadata database dbname="dolphinscheduler" +# --------------------------------------------------------- +# Registry Server +# --------------------------------------------------------- +# Registry Server plugin dir. DolphinScheduler will find and load the registry plugin jar package from this dir. +# For now default registry server is zookeeper, so the default value is `lib/plugin/registry/zookeeper`. +# If you want to implement your own registry server, please see https://dolphinscheduler.apache.org/en-us/docs/dev/user_doc/registry_spi.html +registryPluginDir="lib/plugin/registry/zookeeper" -# zk cluster -zkQuorum="192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181" - -# zk root directory -zkRoot="/dolphinscheduler" - -# registry config -# registry plugin dir -# Note: find and load the Registry Plugin Jar from this dir. -registryPluginDir="/data1_1T/dolphinscheduler/lib/plugin/registry" +# Registry Server plugin name, should be a substring of `registryPluginDir`, DolphinScheduler use this for verifying configuration consistency registryPluginName="zookeeper" -registryServers="192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181" +# Registry Server address. +registryServers="192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181" -# Note: the target installation path for dolphinscheduler, please not config as the same as the current path (pwd) -installPath="/data1_1T/dolphinscheduler" +# The root of zookeeper, for now DolphinScheduler default registry server is zookeeper. +zkRoot="/dolphinscheduler" -# deployment user -# Note: the deployment user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled, the root directory needs to be created by itself -deployUser="dolphinscheduler" +# --------------------------------------------------------- +# Alert Server +# --------------------------------------------------------- +# Alert Server plugin dir. DolphinScheduler will find and load the alert plugin jar package from this dir. +alertPluginDir="lib/plugin/alert" -# alert config -# alert plugin dir -# Note: find and load the Alert Plugin Jar from this dir. -alertPluginDir="/data1_1T/dolphinscheduler/lib/plugin/alert" - -# user data local directory path, please make sure the directory exists and have read write permissions -dataBasedirPath="/tmp/dolphinscheduler" +# --------------------------------------------------------- +# Worker Task Server +# --------------------------------------------------------- +# Worker Task Server plugin dir. DolphinScheduler will find and load the worker task plugin jar package from this dir. +taskPluginDir="lib/plugin/task" # resource storage type: HDFS, S3, NONE resourceStorageType="NONE" @@ -110,34 +165,3 @@ sudoEnable="true" # worker tenant auto create workerTenantAutoCreate="false" - -# worker task plugin dir -taskPluginDir="/data1_1T/dolphinscheduler/lib/plugin/task" - -# api server port -apiServerPort="12345" - - -# install hosts -# Note: install the scheduled hostname list. If it is pseudo-distributed, just write a pseudo-distributed hostname -ips="ds1,ds2,ds3,ds4,ds5" - -# ssh port, default 22 -# Note: if ssh port is not default, modify here -sshPort="22" - -# run master machine -# Note: list of hosts hostname for deploying master -masters="ds1,ds2" - -# run worker machine -# note: need to write the worker group name of each worker, the default value is "default" -workers="ds1:default,ds2:default,ds3:default,ds4:default,ds5:default" - -# run alert machine -# note: list of machine hostnames for deploying alert server -alertServer="ds3" - -# run api machine -# note: list of machine hostnames for deploying api server -apiServers="ds1" \ No newline at end of file diff --git a/install.sh b/install.sh index 25e5d91e4d..c25d852b7b 100755 --- a/install.sh +++ b/install.sh @@ -34,11 +34,17 @@ datasourceDriverClassname="com.mysql.jdbc.Driver" if [ $dbtype == "postgresql" ];then datasourceDriverClassname="org.postgresql.Driver" fi + +# Change configuration in conf/config/dolphinscheduler_env.sh +sed -i ${txt} "s@^export JAVA_HOME=.*@export JAVA_HOME=${javaHome}@g" conf/env/dolphinscheduler_env.sh + +# Change configuration in conf/datasource.properties sed -i ${txt} "s@^spring.datasource.driver-class-name=.*@spring.datasource.driver-class-name=${datasourceDriverClassname}@g" conf/datasource.properties sed -i ${txt} "s@^spring.datasource.url=.*@spring.datasource.url=jdbc:${dbtype}://${dbhost}/${dbname}?characterEncoding=UTF-8\&allowMultiQueries=true@g" conf/datasource.properties sed -i ${txt} "s@^spring.datasource.username=.*@spring.datasource.username=${username}@g" conf/datasource.properties sed -i ${txt} "s@^spring.datasource.password=.*@spring.datasource.password=${password}@g" conf/datasource.properties +# Change configuration in conf/common.properties sed -i ${txt} "s@^data.basedir.path=.*@data.basedir.path=${dataBasedirPath}@g" conf/common.properties sed -i ${txt} "s@^resource.storage.type=.*@resource.storage.type=${resourceStorageType}@g" conf/common.properties sed -i ${txt} "s@^resource.upload.path=.*@resource.upload.path=${resourceUploadPath}@g" conf/common.properties @@ -58,17 +64,22 @@ sed -i ${txt} "s@^yarn.application.status.address=.*@yarn.application.status.add sed -i ${txt} "s@^yarn.job.history.status.address=.*@yarn.job.history.status.address=http://${singleYarnIp}:19888/ws/v1/history/mapreduce/jobs/%s@g" conf/common.properties sed -i ${txt} "s@^sudo.enable=.*@sudo.enable=${sudoEnable}@g" conf/common.properties -# the following configurations may be commented, so ddd #\? to ensure successful sed -sed -i ${txt} "s@^#\?worker.tenant.auto.create=.*@worker.tenant.auto.create=${workerTenantAutoCreate}@g" conf/worker.properties -sed -i ${txt} "s@^#\?alert.listen.host=.*@alert.listen.host=${alertServer}@g" conf/worker.properties -sed -i ${txt} "s@^#\?task.plugin.dir=.*@task.plugin.dir=${taskPluginDir}@g" conf/worker.properties +# The following configurations may be commented, so ddd #* to ensure sed work correct +# Change configuration in conf/worker.properties +sed -i ${txt} "s@^#*worker.tenant.auto.create=.*@worker.tenant.auto.create=${workerTenantAutoCreate}@g" conf/worker.properties +sed -i ${txt} "s@^#*alert.listen.host=.*@alert.listen.host=${alertServer}@g" conf/worker.properties +sed -i ${txt} "s@^#*task.plugin.dir=.*@task.plugin.dir=${installPath}/${taskPluginDir}@g" conf/worker.properties + +# Change configuration in conf/alert.properties +sed -i ${txt} "s@^#*alert.plugin.dir=.*@alert.plugin.dir=${installPath}/${alertPluginDir}@g" conf/alert.properties -sed -i ${txt} "s@^#\?alert.plugin.dir=.*@alert.plugin.dir=${alertPluginDir}@g" conf/alert.properties -sed -i ${txt} "s@^#\?server.port=.*@server.port=${apiServerPort}@g" conf/application-api.properties +# Change configuration in conf/application-api.properties +sed -i ${txt} "s@^#*server.port=.*@server.port=${apiServerPort}@g" conf/application-api.properties -sed -i ${txt} "s@^#\?registry.plugin.dir=.*@registry.plugin.dir=${registryPluginDir}@g" conf/registry.properties -sed -i ${txt} "s@^#\?registry.plugin.name=.*@registry.plugin.name=${registryPluginName}@g" conf/registry.properties -sed -i ${txt} "s@^#\?registry.servers=.*@registry.servers=${registryServers}@g" conf/registry.properties +# Change configuration in conf/registry.properties +sed -i ${txt} "s@^#*registry.plugin.dir=.*@registry.plugin.dir=${installPath}/${registryPluginDir}@g" conf/registry.properties +sed -i ${txt} "s@^#*registry.plugin.name=.*@registry.plugin.name=${registryPluginName}@g" conf/registry.properties +sed -i ${txt} "s@^#*registry.servers=.*@registry.servers=${registryServers}@g" conf/registry.properties # 2.create directory echo "2.create directory" diff --git a/script/dolphinscheduler-daemon.sh b/script/dolphinscheduler-daemon.sh index 81af5fd60c..4de812b979 100755 --- a/script/dolphinscheduler-daemon.sh +++ b/script/dolphinscheduler-daemon.sh @@ -36,6 +36,7 @@ BIN_DIR=`cd "$BIN_DIR"; pwd` DOLPHINSCHEDULER_HOME=`cd "$BIN_DIR/.."; pwd` source /etc/profile +source "${DOLPHINSCHEDULER_HOME}/conf/env/dolphinscheduler_env.sh" export JAVA_HOME=$JAVA_HOME #export JAVA_HOME=/opt/soft/jdk diff --git a/script/remove-zk-node.sh b/script/remove-zk-node.sh index 1129ca6a02..2cb8a2b284 100755 --- a/script/remove-zk-node.sh +++ b/script/remove-zk-node.sh @@ -32,6 +32,7 @@ BIN_DIR=`cd "$BIN_DIR"; pwd` DOLPHINSCHEDULER_HOME=$BIN_DIR/.. source ${BIN_DIR}/../conf/config/install_config.conf +source ${BIN_DIR}/../conf/env/dolphinscheduler_env.sh export JAVA_HOME=$JAVA_HOME @@ -44,7 +45,7 @@ export STOP_TIMEOUT=5 CLASS=org.apache.zookeeper.ZooKeeperMain -exec_command="$DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS -server $zkQuorum rmr $rootNode" +exec_command="$DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS -server $registryServers rmr $rootNode" cd $DOLPHINSCHEDULER_HOME $JAVA_HOME/bin/java $exec_command diff --git a/script/scp-hosts.sh b/script/scp-hosts.sh index bdcdc73fff..8def4d1bae 100755 --- a/script/scp-hosts.sh +++ b/script/scp-hosts.sh @@ -57,7 +57,8 @@ do fi echo "start to scp $dsDir to $host/$installPath" - scp -P $sshPort -r $workDir/../$dsDir $host:$installPath + # Use quiet mode to reduce command line output + scp -q -P $sshPort -r $workDir/../$dsDir $host:$installPath done echo "scp dirs to $host/$installPath complete"