github
/
DolphinScheduler
mirror of https://gitee.com/dolphinscheduler/DolphinScheduler.git
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

Add config to override default ssh port for install scripts. #780 (#1388)

pull/2/head
elonlo 5 years ago committed by bao liang
parent
bf1fec74de
commit
e6395d0eef
7 changed files with 42 additions and 31 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 22
      install.sh
  2. 1
      script/config/install_config.conf
  3. 1
      script/config/run_config.conf
  4. 9
      script/monitor-server.py
  5. 18
      script/scp-hosts.sh
  6. 10
      script/start-all.sh
  7. 10
      script/stop-all.sh

22
install.sh
Unescape View File

@ -82,6 +82,10 @@ zkQuorum="192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181"
# Note: install the scheduled hostname list. If it is pseudo-distributed, just write a pseudo-distributed hostname # Note: install the scheduled hostname list. If it is pseudo-distributed, just write a pseudo-distributed hostname
ips="ark0,ark1,ark2,ark3,ark4" ips="ark0,ark1,ark2,ark3,ark4"
# ssh port, default 22
# Note: if ssh port is not default, modify here
sshPort=22
# conf/config/run_config.conf config # conf/config/run_config.conf config
# run master machine # run master machine
# Note: list of hosts hostname for deploying master # Note: list of hosts hostname for deploying master
@ -426,12 +430,14 @@ sed -i ${txt} "s#enterprise.wechat.users.*#enterprise.wechat.users=${enterpriseW
sed -i ${txt} "s#installPath.*#installPath=${installPath}#g" conf/config/install_config.conf sed -i ${txt} "s#installPath.*#installPath=${installPath}#g" conf/config/install_config.conf
sed -i ${txt} "s#deployUser.*#deployUser=${deployUser}#g" conf/config/install_config.conf sed -i ${txt} "s#deployUser.*#deployUser=${deployUser}#g" conf/config/install_config.conf
sed -i ${txt} "s#ips.*#ips=${ips}#g" conf/config/install_config.conf sed -i ${txt} "s#ips.*#ips=${ips}#g" conf/config/install_config.conf
sed -i ${txt} "s#sshPort.*#sshPort=${sshPort}#g" conf/config/install_config.conf
sed -i ${txt} "s#masters.*#masters=${masters}#g" conf/config/run_config.conf sed -i ${txt} "s#masters.*#masters=${masters}#g" conf/config/run_config.conf
sed -i ${txt} "s#workers.*#workers=${workers}#g" conf/config/run_config.conf sed -i ${txt} "s#workers.*#workers=${workers}#g" conf/config/run_config.conf
sed -i ${txt} "s#alertServer.*#alertServer=${alertServer}#g" conf/config/run_config.conf sed -i ${txt} "s#alertServer.*#alertServer=${alertServer}#g" conf/config/run_config.conf
sed -i ${txt} "s#apiServers.*#apiServers=${apiServers}#g" conf/config/run_config.conf sed -i ${txt} "s#apiServers.*#apiServers=${apiServers}#g" conf/config/run_config.conf
sed -i ${txt} "s#sshPort.*#sshPort=${sshPort}#g" conf/config/run_config.conf
# 2,create directory # 2,create directory
@ -447,23 +453,23 @@ for host in ${hostsArr[@]}
do do
# create if programPath does not exist # create if programPath does not exist
if ! ssh $host test -e $programPath; then if ! ssh -p $sshPort $host test -e $programPath; then
ssh $host "sudo mkdir -p $programPath;sudo chown -R $deployUser:$deployUser $programPath" ssh -p $sshPort $host "sudo mkdir -p $programPath;sudo chown -R $deployUser:$deployUser $programPath"
fi fi
# create if downloadPath does not exist # create if downloadPath does not exist
if ! ssh $host test -e $downloadPath; then if ! ssh -p $sshPort $host test -e $downloadPath; then
ssh $host "sudo mkdir -p $downloadPath;sudo chown -R $deployUser:$deployUser $downloadPath" ssh -p $sshPort $host "sudo mkdir -p $downloadPath;sudo chown -R $deployUser:$deployUser $downloadPath"
fi fi
# create if execPath does not exist # create if execPath does not exist
if ! ssh $host test -e $execPath; then if ! ssh -p $sshPort $host test -e $execPath; then
ssh $host "sudo mkdir -p $execPath; sudo chown -R $deployUser:$deployUser $execPath" ssh -p $sshPort $host "sudo mkdir -p $execPath; sudo chown -R $deployUser:$deployUser $execPath"
fi fi
# create if xlsFilePath does not exist # create if xlsFilePath does not exist
if ! ssh $host test -e $xlsFilePath; then if ! ssh -p $sshPort $host test -e $xlsFilePath; then
ssh $host "sudo mkdir -p $xlsFilePath; sudo chown -R $deployUser:$deployUser $xlsFilePath" ssh -p $sshPort $host "sudo mkdir -p $xlsFilePath; sudo chown -R $deployUser:$deployUser $xlsFilePath"
fi fi
done done

1
script/config/install_config.conf
Unescape View File

@ -18,3 +18,4 @@
installPath=/data1_1T/dolphinscheduler installPath=/data1_1T/dolphinscheduler
deployUser=dolphinscheduler deployUser=dolphinscheduler
ips=ark0,ark1,ark2,ark3,ark4 ips=ark0,ark1,ark2,ark3,ark4
sshPort=22

1
script/config/run_config.conf
Unescape View File

@ -19,3 +19,4 @@ masters=ark0,ark1
workers=ark2,ark3,ark4 workers=ark2,ark3,ark4
alertServer=ark3 alertServer=ark3
apiServers=ark1 apiServers=ark1
sshPort=22

9
script/monitor-server.py
Unescape View File

@ -77,9 +77,12 @@ class ZkClient:
master_list = list(map(lambda item : self.get_ip_by_hostname(item),master_list)) master_list = list(map(lambda item : self.get_ip_by_hostname(item),master_list))
worker_list = config_dict.get('workers').split(',') worker_list = config_dict.get('workers').split(',')
print worker_list print worker_list
worker_list = list(map(lambda item: self.get_ip_by_hostname(item), worker_list)) worker_list = list(map(lambda item: self.get_ip_by_hostname(item), worker_list))
ssh_port = config_dict.get("sshPort")
print ssh_port
if (self.zk.exists(masters_zk_path)): if (self.zk.exists(masters_zk_path)):
zk_master_list = [] zk_master_list = []
zk_master_nodes = self.zk.get_children(masters_zk_path) zk_master_nodes = self.zk.get_children(masters_zk_path)
@ -89,7 +92,7 @@ class ZkClient:
if (len(restart_master_list) != 0): if (len(restart_master_list) != 0):
for master in restart_master_list: for master in restart_master_list:
print("master " + self.get_ip_by_hostname(master) + " server has down") print("master " + self.get_ip_by_hostname(master) + " server has down")
os.system('ssh ' + self.get_ip_by_hostname(master) + ' sh ' + install_path + '/bin/dolphinscheduler-daemon.sh start master-server') os.system('ssh -p ' + ssh_port + ' ' + self.get_ip_by_hostname(master) + ' sh ' + install_path + '/bin/dolphinscheduler-daemon.sh start master-server')
if (self.zk.exists(workers_zk_path)): if (self.zk.exists(workers_zk_path)):
zk_worker_list = [] zk_worker_list = []
@ -100,7 +103,7 @@ class ZkClient:
if (len(restart_worker_list) != 0): if (len(restart_worker_list) != 0):
for worker in restart_worker_list: for worker in restart_worker_list:
print("worker " + self.get_ip_by_hostname(worker) + " server has down") print("worker " + self.get_ip_by_hostname(worker) + " server has down")
os.system('ssh ' + self.get_ip_by_hostname(worker) + ' sh ' + install_path + '/bin/dolphinscheduler-daemon.sh start worker-server') os.system('ssh -p ' + ssh_port + ' ' + self.get_ip_by_hostname(worker) + ' sh ' + install_path + '/bin/dolphinscheduler-daemon.sh start worker-server')
print(datetime.now().strftime("%Y-%m-%d %H:%M:%S")) print(datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
schedule.enter(inc, 0, self.restart_server, (inc,)) schedule.enter(inc, 0, self.restart_server, (inc,))

18
script/scp-hosts.sh
Unescape View File

@ -25,15 +25,15 @@ hostsArr=(${ips//,/ })
for host in ${hostsArr[@]} for host in ${hostsArr[@]}
do do
if ! ssh $host test -e $installPath; then if ! ssh -p $sshPort $host test -e $installPath; then
ssh $host "sudo mkdir -p $installPath; sudo chown -R $deployUser:$deployUser $installPath" ssh -p $sshPort $host "sudo mkdir -p $installPath; sudo chown -R $deployUser:$deployUser $installPath"
fi fi
ssh $host "cd $installPath/; rm -rf bin/ conf/ lib/ script/ sql/" ssh -p $sshPort $host "cd $installPath/; rm -rf bin/ conf/ lib/ script/ sql/"
scp -r $workDir/../bin $host:$installPath scp -P $sshPort -r $workDir/../bin $host:$installPath
scp -r $workDir/../conf $host:$installPath scp -P $sshPort -r $workDir/../conf $host:$installPath
scp -r $workDir/../lib $host:$installPath scp -P $sshPort -r $workDir/../lib $host:$installPath
scp -r $workDir/../script $host:$installPath scp -P $sshPort -r $workDir/../script $host:$installPath
scp -r $workDir/../sql $host:$installPath scp -P $sshPort -r $workDir/../sql $host:$installPath
scp $workDir/../install.sh $host:$installPath scp -P $sshPort $workDir/../install.sh $host:$installPath
done done

10
script/start-all.sh
Unescape View File

@ -25,7 +25,7 @@ mastersHost=(${masters//,/ })
for master in ${mastersHost[@]} for master in ${mastersHost[@]}
do do
echo $master echo $master
ssh $master "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start master-server;" ssh -p $sshPort $master "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start master-server;"
done done
@ -34,17 +34,17 @@ for worker in ${workersHost[@]}
do do
echo $worker echo $worker
ssh $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start worker-server;" ssh -p $sshPort $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start worker-server;"
ssh $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start logger-server;" ssh -p $sshPort $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start logger-server;"
done done
ssh $alertServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start alert-server;" ssh -p $sshPort $alertServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start alert-server;"
apiServersHost=(${apiServers//,/ }) apiServersHost=(${apiServers//,/ })
for apiServer in ${apiServersHost[@]} for apiServer in ${apiServersHost[@]}
do do
echo $apiServer echo $apiServer
ssh $apiServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start api-server;" ssh -p $sshPort $apiServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start api-server;"
done done

10
script/stop-all.sh
Unescape View File

@ -26,7 +26,7 @@ mastersHost=(${masters//,/ })
for master in ${mastersHost[@]} for master in ${mastersHost[@]}
do do
echo $master echo $master
ssh $master "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop master-server;" ssh -p $sshPort $master "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop master-server;"
done done
@ -35,17 +35,17 @@ for worker in ${workersHost[@]}
do do
echo $worker echo $worker
ssh $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop worker-server;" ssh -p $sshPort $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop worker-server;"
ssh $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop logger-server;" ssh -p $sshPort $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop logger-server;"
done done
ssh $alertServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop alert-server;" ssh -p $sshPort $alertServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop alert-server;"
apiServersHost=(${apiServers//,/ }) apiServersHost=(${apiServers//,/ })
for apiServer in ${apiServersHost[@]} for apiServer in ${apiServersHost[@]}
do do
echo $apiServer echo $apiServer
ssh $apiServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop api-server;" ssh -p $sshPort $apiServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop api-server;"
done done

Write Preview
Loading…
Cancel
Save