Browse Source

Fix install.sh for DS 2.0 and add comment to install_config.conf (#6506)

* Fix install.sh for DS 2.0

* Use #* make it work in macos

* Add comment for properties change

* Add JAVA HOME for only one file change to deploy

* Reduce config change by using default value of: plugin registry, alert, task as

* Change javaHome default value

* Delete config param zkQuorum

* Reorder param

* Add comment for install_config.conf
3.0.0/version-upgrade
Jiajie Zhong 3 years ago committed by GitHub
parent
commit
6495a204f3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 148
      dolphinscheduler-server/src/main/resources/config/install_config.conf
  2. 29
      install.sh
  3. 1
      script/dolphinscheduler-daemon.sh
  4. 3
      script/remove-zk-node.sh
  5. 3
      script/scp-hosts.sh

148
dolphinscheduler-server/src/main/resources/config/install_config.conf

@ -15,54 +15,109 @@
# limitations under the License.
#
# ---------------------------------------------------------
# INSTALL MACHINE
# ---------------------------------------------------------
# A comma separated list of machine hostname or IP would be installed DolphinScheduler,
# including master, worker, api, alert. If you want to deploy in pseudo-distributed
# mode, just write a pseudo-distributed hostname
# Example for hostnames: ips="ds1,ds2,ds3,ds4,ds5", Example for IP: ips="192.168.8.1,192.168.8.2,192.168.8.3,192.168.8.4,192.168.8.5"
ips="ds1,ds2,ds3,ds4,ds5"
# Port of SSH protocol, default value is 22. For now we only support same port in all `ips` machine
# modify it if you use different ssh port
sshPort="22"
# A comma separated list of machine hostname or IP would be installed Master server, it
# must be a subset of configuration `ips`.
# Example for hostnames: ips="ds1,ds2", Example for IP: ips="192.168.8.1,192.168.8.2"
masters="ds1,ds2"
# A comma separated list of machine <hostname>:<workerGroup> or <IP>:<workerGroup>.All hostname or IP must be a
# subset of configuration `ips`, And workerGroup have default value as `default`, but we recommend you declare behind the hosts
# Example for hostnames: ips="ds1:default,ds2:default,ds3:default", Example for IP: ips="192.168.8.1:default,192.168.8.2:default,192.168.8.3:default"
workers="ds1:default,ds2:default,ds3:default,ds4:default,ds5:default"
# A comma separated list of machine hostname or IP would be installed Alert server, it
# must be a subset of configuration `ips`.
# Example for hostnames: ips="ds3", Example for IP: ips="192.168.8.3"
alertServer="ds3"
# A comma separated list of machine hostname or IP would be installed API server, it
# must be a subset of configuration `ips`.
# Example for hostnames: ips="ds1", Example for IP: ips="192.168.8.1"
apiServers="ds1"
# The directory to install DolphinScheduler for all machine we config above. It will automatically created by `install.sh` script if not exists.
# **DO NOT** set this configuration same as the current path (pwd)
installPath="/data1_1T/dolphinscheduler"
# The user to deploy DolphinScheduler for all machine we config above. For now user must create by yourself before run `install.sh`
# script. The user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled than the root directory needs
# to be created by this user
deployUser="dolphinscheduler"
# The directory to store local data for all machine we config above. Make sure user `deployUser` have permissions to read and write this directory.
dataBasedirPath="/tmp/dolphinscheduler"
# ---------------------------------------------------------
# DolphinScheduler ENV
# ---------------------------------------------------------
# JAVA_HOME, we recommend use same JAVA_HOME in all machine you going to install DolphinScheduler
# and this configuration only support one parameter so far.
javaHome="/your/java/home/here"
# NOTICE: If the following config has special characters in the variable `.*[]^${}\+?|()@#&`, Please escape, for example, `[` escape to `\[`
# postgresql or mysql
# DolphinScheduler API service port, also this your DolphinScheduler UI component's URL port, default values is 12345
apiServerPort="12345"
# ---------------------------------------------------------
# Database
# NOTICE: If database value has special characters, such as `.*[]^${}\+?|()@#&`, Please add prefix `\` for escaping.
# ---------------------------------------------------------
# The type for the metadata database
# Supported values: ``postgresql``, ``mysql``.
dbtype="mysql"
# db config
# db address and port
# The <HOST>:<PORT> connection pair DolphinScheduler connect to the metadata database
dbhost="192.168.xx.xx:3306"
# db username
# The username DolphinScheduler connect to the metadata database
username="xx"
# db password
# NOTICE: if there are special characters, please use the \ to escape, for example, `[` escape to `\[`
# The password DolphinScheduler connect to the metadata database
password="xx"
# database name
# The database DolphinScheduler connect to the metadata database
dbname="dolphinscheduler"
# ---------------------------------------------------------
# Registry Server
# ---------------------------------------------------------
# Registry Server plugin dir. DolphinScheduler will find and load the registry plugin jar package from this dir.
# For now default registry server is zookeeper, so the default value is `lib/plugin/registry/zookeeper`.
# If you want to implement your own registry server, please see https://dolphinscheduler.apache.org/en-us/docs/dev/user_doc/registry_spi.html
registryPluginDir="lib/plugin/registry/zookeeper"
# zk cluster
zkQuorum="192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181"
# zk root directory
zkRoot="/dolphinscheduler"
# registry config
# registry plugin dir
# Note: find and load the Registry Plugin Jar from this dir.
registryPluginDir="/data1_1T/dolphinscheduler/lib/plugin/registry"
# Registry Server plugin name, should be a substring of `registryPluginDir`, DolphinScheduler use this for verifying configuration consistency
registryPluginName="zookeeper"
registryServers="192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181"
# Registry Server address.
registryServers="192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181"
# Note: the target installation path for dolphinscheduler, please not config as the same as the current path (pwd)
installPath="/data1_1T/dolphinscheduler"
# The root of zookeeper, for now DolphinScheduler default registry server is zookeeper.
zkRoot="/dolphinscheduler"
# deployment user
# Note: the deployment user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled, the root directory needs to be created by itself
deployUser="dolphinscheduler"
# ---------------------------------------------------------
# Alert Server
# ---------------------------------------------------------
# Alert Server plugin dir. DolphinScheduler will find and load the alert plugin jar package from this dir.
alertPluginDir="lib/plugin/alert"
# alert config
# alert plugin dir
# Note: find and load the Alert Plugin Jar from this dir.
alertPluginDir="/data1_1T/dolphinscheduler/lib/plugin/alert"
# user data local directory path, please make sure the directory exists and have read write permissions
dataBasedirPath="/tmp/dolphinscheduler"
# ---------------------------------------------------------
# Worker Task Server
# ---------------------------------------------------------
# Worker Task Server plugin dir. DolphinScheduler will find and load the worker task plugin jar package from this dir.
taskPluginDir="lib/plugin/task"
# resource storage type: HDFS, S3, NONE
resourceStorageType="NONE"
@ -110,34 +165,3 @@ sudoEnable="true"
# worker tenant auto create
workerTenantAutoCreate="false"
# worker task plugin dir
taskPluginDir="/data1_1T/dolphinscheduler/lib/plugin/task"
# api server port
apiServerPort="12345"
# install hosts
# Note: install the scheduled hostname list. If it is pseudo-distributed, just write a pseudo-distributed hostname
ips="ds1,ds2,ds3,ds4,ds5"
# ssh port, default 22
# Note: if ssh port is not default, modify here
sshPort="22"
# run master machine
# Note: list of hosts hostname for deploying master
masters="ds1,ds2"
# run worker machine
# note: need to write the worker group name of each worker, the default value is "default"
workers="ds1:default,ds2:default,ds3:default,ds4:default,ds5:default"
# run alert machine
# note: list of machine hostnames for deploying alert server
alertServer="ds3"
# run api machine
# note: list of machine hostnames for deploying api server
apiServers="ds1"

29
install.sh

@ -34,11 +34,17 @@ datasourceDriverClassname="com.mysql.jdbc.Driver"
if [ $dbtype == "postgresql" ];then
datasourceDriverClassname="org.postgresql.Driver"
fi
# Change configuration in conf/config/dolphinscheduler_env.sh
sed -i ${txt} "s@^export JAVA_HOME=.*@export JAVA_HOME=${javaHome}@g" conf/env/dolphinscheduler_env.sh
# Change configuration in conf/datasource.properties
sed -i ${txt} "s@^spring.datasource.driver-class-name=.*@spring.datasource.driver-class-name=${datasourceDriverClassname}@g" conf/datasource.properties
sed -i ${txt} "s@^spring.datasource.url=.*@spring.datasource.url=jdbc:${dbtype}://${dbhost}/${dbname}?characterEncoding=UTF-8\&allowMultiQueries=true@g" conf/datasource.properties
sed -i ${txt} "s@^spring.datasource.username=.*@spring.datasource.username=${username}@g" conf/datasource.properties
sed -i ${txt} "s@^spring.datasource.password=.*@spring.datasource.password=${password}@g" conf/datasource.properties
# Change configuration in conf/common.properties
sed -i ${txt} "s@^data.basedir.path=.*@data.basedir.path=${dataBasedirPath}@g" conf/common.properties
sed -i ${txt} "s@^resource.storage.type=.*@resource.storage.type=${resourceStorageType}@g" conf/common.properties
sed -i ${txt} "s@^resource.upload.path=.*@resource.upload.path=${resourceUploadPath}@g" conf/common.properties
@ -58,17 +64,22 @@ sed -i ${txt} "s@^yarn.application.status.address=.*@yarn.application.status.add
sed -i ${txt} "s@^yarn.job.history.status.address=.*@yarn.job.history.status.address=http://${singleYarnIp}:19888/ws/v1/history/mapreduce/jobs/%s@g" conf/common.properties
sed -i ${txt} "s@^sudo.enable=.*@sudo.enable=${sudoEnable}@g" conf/common.properties
# the following configurations may be commented, so ddd #\? to ensure successful sed
sed -i ${txt} "s@^#\?worker.tenant.auto.create=.*@worker.tenant.auto.create=${workerTenantAutoCreate}@g" conf/worker.properties
sed -i ${txt} "s@^#\?alert.listen.host=.*@alert.listen.host=${alertServer}@g" conf/worker.properties
sed -i ${txt} "s@^#\?task.plugin.dir=.*@task.plugin.dir=${taskPluginDir}@g" conf/worker.properties
# The following configurations may be commented, so ddd #* to ensure sed work correct
# Change configuration in conf/worker.properties
sed -i ${txt} "s@^#*worker.tenant.auto.create=.*@worker.tenant.auto.create=${workerTenantAutoCreate}@g" conf/worker.properties
sed -i ${txt} "s@^#*alert.listen.host=.*@alert.listen.host=${alertServer}@g" conf/worker.properties
sed -i ${txt} "s@^#*task.plugin.dir=.*@task.plugin.dir=${installPath}/${taskPluginDir}@g" conf/worker.properties
# Change configuration in conf/alert.properties
sed -i ${txt} "s@^#*alert.plugin.dir=.*@alert.plugin.dir=${installPath}/${alertPluginDir}@g" conf/alert.properties
sed -i ${txt} "s@^#\?alert.plugin.dir=.*@alert.plugin.dir=${alertPluginDir}@g" conf/alert.properties
sed -i ${txt} "s@^#\?server.port=.*@server.port=${apiServerPort}@g" conf/application-api.properties
# Change configuration in conf/application-api.properties
sed -i ${txt} "s@^#*server.port=.*@server.port=${apiServerPort}@g" conf/application-api.properties
sed -i ${txt} "s@^#\?registry.plugin.dir=.*@registry.plugin.dir=${registryPluginDir}@g" conf/registry.properties
sed -i ${txt} "s@^#\?registry.plugin.name=.*@registry.plugin.name=${registryPluginName}@g" conf/registry.properties
sed -i ${txt} "s@^#\?registry.servers=.*@registry.servers=${registryServers}@g" conf/registry.properties
# Change configuration in conf/registry.properties
sed -i ${txt} "s@^#*registry.plugin.dir=.*@registry.plugin.dir=${installPath}/${registryPluginDir}@g" conf/registry.properties
sed -i ${txt} "s@^#*registry.plugin.name=.*@registry.plugin.name=${registryPluginName}@g" conf/registry.properties
sed -i ${txt} "s@^#*registry.servers=.*@registry.servers=${registryServers}@g" conf/registry.properties
# 2.create directory
echo "2.create directory"

1
script/dolphinscheduler-daemon.sh

@ -36,6 +36,7 @@ BIN_DIR=`cd "$BIN_DIR"; pwd`
DOLPHINSCHEDULER_HOME=`cd "$BIN_DIR/.."; pwd`
source /etc/profile
source "${DOLPHINSCHEDULER_HOME}/conf/env/dolphinscheduler_env.sh"
export JAVA_HOME=$JAVA_HOME
#export JAVA_HOME=/opt/soft/jdk

3
script/remove-zk-node.sh

@ -32,6 +32,7 @@ BIN_DIR=`cd "$BIN_DIR"; pwd`
DOLPHINSCHEDULER_HOME=$BIN_DIR/..
source ${BIN_DIR}/../conf/config/install_config.conf
source ${BIN_DIR}/../conf/env/dolphinscheduler_env.sh
export JAVA_HOME=$JAVA_HOME
@ -44,7 +45,7 @@ export STOP_TIMEOUT=5
CLASS=org.apache.zookeeper.ZooKeeperMain
exec_command="$DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS -server $zkQuorum rmr $rootNode"
exec_command="$DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS -server $registryServers rmr $rootNode"
cd $DOLPHINSCHEDULER_HOME
$JAVA_HOME/bin/java $exec_command

3
script/scp-hosts.sh

@ -57,7 +57,8 @@ do
fi
echo "start to scp $dsDir to $host/$installPath"
scp -P $sshPort -r $workDir/../$dsDir $host:$installPath
# Use quiet mode to reduce command line output
scp -q -P $sshPort -r $workDir/../$dsDir $host:$installPath
done
echo "scp dirs to $host/$installPath complete"

Loading…
Cancel
Save