From 06c4ddf8e36ca79b45484281e09a0bf9e0e1b969 Mon Sep 17 00:00:00 2001 From: dailidong Date: Tue, 12 May 2020 16:49:40 +0800 Subject: [PATCH] add worker group config (#2681) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * update README about DolphinScheduler * Update issue templates * update * regularize api pom xml update rpc maven compile to 1.8 * regularize api pom xml * change commons.lang3.StringUtils to common.utils.StringUtils * update pom.xml * update * correct equals method * jasper-runtime is needed when api server start * jasper-runtime jar is needed when api server start * combine logback config of master/worker/alert/api server to one logback.xml * remove tomcat runtime jar * add UT * add license * remove jasper-runtime jar, not need anymore * sovle jar conflict, remove servlet-api 2.5 jar * remove servlet 2.5 * add embedded database h2 and update embedded zookeeper * delete logs * add license * change log level to INFO * simplify and optimize config * add development state * simplify common config * add worker group config * add worker group config Co-authored-by: lidongdai Co-authored-by: dailidong --- .../src/main/resources/config/install_config.conf | 12 ++++++------ .../src/main/resources/worker.properties | 4 ++-- script/scp-hosts.sh | 11 +++++++++++ script/start-all.sh | 3 +-- script/stop-all.sh | 3 +-- 5 files changed, 21 insertions(+), 12 deletions(-) diff --git a/dolphinscheduler-server/src/main/resources/config/install_config.conf b/dolphinscheduler-server/src/main/resources/config/install_config.conf index fab6b32d51..f6b09b9cb8 100644 --- a/dolphinscheduler-server/src/main/resources/config/install_config.conf +++ b/dolphinscheduler-server/src/main/resources/config/install_config.conf @@ -118,7 +118,7 @@ apiServerPort="12345" # install hosts # Note: install the scheduled hostname list. If it is pseudo-distributed, just write a pseudo-distributed hostname -ips="ark0,ark1,ark2,ark3,ark4" +ips="ds1,ds2,ds3,ds4,ds5" # ssh port, default 22 # Note: if ssh port is not default, modify here @@ -126,19 +126,19 @@ sshPort="22" # run master machine # Note: list of hosts hostname for deploying master -masters="ark0,ark1" +masters="ds1,ds2" # run worker machine -# note: list of machine hostnames for deploying workers -workers="ark2,ark3,ark4" +# note: need to write the worker group name of each worker, the default value is "default" +workersGroup=(["ds1"]="default" ["ds2"]="default" ["ds3"]="default" ["ds4"]="default" ["ds5"]="default") # run alert machine # note: list of machine hostnames for deploying alert server -alertServer="ark3" +alertServer="ds3" # run api machine # note: list of machine hostnames for deploying api server -apiServers="ark1" +apiServers="ds1" # whether to start monitoring self-starting scripts monitorServerState="false" diff --git a/dolphinscheduler-server/src/main/resources/worker.properties b/dolphinscheduler-server/src/main/resources/worker.properties index d078f26ca6..36bc132743 100644 --- a/dolphinscheduler-server/src/main/resources/worker.properties +++ b/dolphinscheduler-server/src/main/resources/worker.properties @@ -21,7 +21,7 @@ # worker heartbeat interval #worker.heartbeat.interval=10 -# submit the number of tasks at a time +# submit the number of tasks at a time TODO #worker.fetch.task.num = 3 # only less than cpu avg load, worker server can work. default value : the number of cpu cores * 2 @@ -34,4 +34,4 @@ #worker.listen.port: 1234 # default worker group -#worker.group=default \ No newline at end of file +worker.group=default \ No newline at end of file diff --git a/script/scp-hosts.sh b/script/scp-hosts.sh index f4949f311a..adee7d76c7 100644 --- a/script/scp-hosts.sh +++ b/script/scp-hosts.sh @@ -20,6 +20,12 @@ workDir=`dirname $0` workDir=`cd ${workDir};pwd` source $workDir/../conf/config/install_config.conf +txt="" +if [[ "$OSTYPE" == "darwin"* ]]; then + # Mac OSX + txt="''" +fi + hostsArr=(${ips//,/ }) for host in ${hostsArr[@]} do @@ -33,6 +39,11 @@ do for dsDir in bin conf lib script sql ui install.sh do + # if worker in workersGroup + if [[ "${map[${host}]}" ]] && [[ "${dsDir}" -eq "conf" ]]; then + sed -i ${txt} "s#worker.group.*#worker.group=${map[${host}]}#g" $workDir/../conf/worker.properties + fi + echo "start to scp $dsDir to $host/$installPath" scp -P $sshPort -r $workDir/../$dsDir $host:$installPath done diff --git a/script/start-all.sh b/script/start-all.sh index 11e4572059..5579a7d87e 100644 --- a/script/start-all.sh +++ b/script/start-all.sh @@ -28,8 +28,7 @@ do done -workersHost=(${workers//,/ }) -for worker in ${workersHost[@]} +for worker in ${!workersGroup[*]} do echo "$worker worker server is starting" diff --git a/script/stop-all.sh b/script/stop-all.sh index f761579cc8..e4ccf75a29 100644 --- a/script/stop-all.sh +++ b/script/stop-all.sh @@ -29,8 +29,7 @@ do done -workersHost=(${workers//,/ }) -for worker in ${workersHost[@]} +for worker in ${!workersGroup[*]} do echo "$worker worker server is stopping" ssh -p $sshPort $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop worker-server;"