Browse Source

Add config to override default ssh port for install scripts. #780 (#1388)

pull/2/head
elonlo 5 years ago committed by bao liang
parent
commit
e6395d0eef
  1. 22
      install.sh
  2. 1
      script/config/install_config.conf
  3. 1
      script/config/run_config.conf
  4. 9
      script/monitor-server.py
  5. 18
      script/scp-hosts.sh
  6. 10
      script/start-all.sh
  7. 10
      script/stop-all.sh

22
install.sh

@ -82,6 +82,10 @@ zkQuorum="192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181"
# Note: install the scheduled hostname list. If it is pseudo-distributed, just write a pseudo-distributed hostname # Note: install the scheduled hostname list. If it is pseudo-distributed, just write a pseudo-distributed hostname
ips="ark0,ark1,ark2,ark3,ark4" ips="ark0,ark1,ark2,ark3,ark4"
# ssh port, default 22
# Note: if ssh port is not default, modify here
sshPort=22
# conf/config/run_config.conf config # conf/config/run_config.conf config
# run master machine # run master machine
# Note: list of hosts hostname for deploying master # Note: list of hosts hostname for deploying master
@ -426,12 +430,14 @@ sed -i ${txt} "s#enterprise.wechat.users.*#enterprise.wechat.users=${enterpriseW
sed -i ${txt} "s#installPath.*#installPath=${installPath}#g" conf/config/install_config.conf sed -i ${txt} "s#installPath.*#installPath=${installPath}#g" conf/config/install_config.conf
sed -i ${txt} "s#deployUser.*#deployUser=${deployUser}#g" conf/config/install_config.conf sed -i ${txt} "s#deployUser.*#deployUser=${deployUser}#g" conf/config/install_config.conf
sed -i ${txt} "s#ips.*#ips=${ips}#g" conf/config/install_config.conf sed -i ${txt} "s#ips.*#ips=${ips}#g" conf/config/install_config.conf
sed -i ${txt} "s#sshPort.*#sshPort=${sshPort}#g" conf/config/install_config.conf
sed -i ${txt} "s#masters.*#masters=${masters}#g" conf/config/run_config.conf sed -i ${txt} "s#masters.*#masters=${masters}#g" conf/config/run_config.conf
sed -i ${txt} "s#workers.*#workers=${workers}#g" conf/config/run_config.conf sed -i ${txt} "s#workers.*#workers=${workers}#g" conf/config/run_config.conf
sed -i ${txt} "s#alertServer.*#alertServer=${alertServer}#g" conf/config/run_config.conf sed -i ${txt} "s#alertServer.*#alertServer=${alertServer}#g" conf/config/run_config.conf
sed -i ${txt} "s#apiServers.*#apiServers=${apiServers}#g" conf/config/run_config.conf sed -i ${txt} "s#apiServers.*#apiServers=${apiServers}#g" conf/config/run_config.conf
sed -i ${txt} "s#sshPort.*#sshPort=${sshPort}#g" conf/config/run_config.conf
# 2,create directory # 2,create directory
@ -447,23 +453,23 @@ for host in ${hostsArr[@]}
do do
# create if programPath does not exist # create if programPath does not exist
if ! ssh $host test -e $programPath; then if ! ssh -p $sshPort $host test -e $programPath; then
ssh $host "sudo mkdir -p $programPath;sudo chown -R $deployUser:$deployUser $programPath" ssh -p $sshPort $host "sudo mkdir -p $programPath;sudo chown -R $deployUser:$deployUser $programPath"
fi fi
# create if downloadPath does not exist # create if downloadPath does not exist
if ! ssh $host test -e $downloadPath; then if ! ssh -p $sshPort $host test -e $downloadPath; then
ssh $host "sudo mkdir -p $downloadPath;sudo chown -R $deployUser:$deployUser $downloadPath" ssh -p $sshPort $host "sudo mkdir -p $downloadPath;sudo chown -R $deployUser:$deployUser $downloadPath"
fi fi
# create if execPath does not exist # create if execPath does not exist
if ! ssh $host test -e $execPath; then if ! ssh -p $sshPort $host test -e $execPath; then
ssh $host "sudo mkdir -p $execPath; sudo chown -R $deployUser:$deployUser $execPath" ssh -p $sshPort $host "sudo mkdir -p $execPath; sudo chown -R $deployUser:$deployUser $execPath"
fi fi
# create if xlsFilePath does not exist # create if xlsFilePath does not exist
if ! ssh $host test -e $xlsFilePath; then if ! ssh -p $sshPort $host test -e $xlsFilePath; then
ssh $host "sudo mkdir -p $xlsFilePath; sudo chown -R $deployUser:$deployUser $xlsFilePath" ssh -p $sshPort $host "sudo mkdir -p $xlsFilePath; sudo chown -R $deployUser:$deployUser $xlsFilePath"
fi fi
done done

1
script/config/install_config.conf

@ -18,3 +18,4 @@
installPath=/data1_1T/dolphinscheduler installPath=/data1_1T/dolphinscheduler
deployUser=dolphinscheduler deployUser=dolphinscheduler
ips=ark0,ark1,ark2,ark3,ark4 ips=ark0,ark1,ark2,ark3,ark4
sshPort=22

1
script/config/run_config.conf

@ -19,3 +19,4 @@ masters=ark0,ark1
workers=ark2,ark3,ark4 workers=ark2,ark3,ark4
alertServer=ark3 alertServer=ark3
apiServers=ark1 apiServers=ark1
sshPort=22

9
script/monitor-server.py

@ -77,9 +77,12 @@ class ZkClient:
master_list = list(map(lambda item : self.get_ip_by_hostname(item),master_list)) master_list = list(map(lambda item : self.get_ip_by_hostname(item),master_list))
worker_list = config_dict.get('workers').split(',') worker_list = config_dict.get('workers').split(',')
print worker_list print worker_list
worker_list = list(map(lambda item: self.get_ip_by_hostname(item), worker_list)) worker_list = list(map(lambda item: self.get_ip_by_hostname(item), worker_list))
ssh_port = config_dict.get("sshPort")
print ssh_port
if (self.zk.exists(masters_zk_path)): if (self.zk.exists(masters_zk_path)):
zk_master_list = [] zk_master_list = []
zk_master_nodes = self.zk.get_children(masters_zk_path) zk_master_nodes = self.zk.get_children(masters_zk_path)
@ -89,7 +92,7 @@ class ZkClient:
if (len(restart_master_list) != 0): if (len(restart_master_list) != 0):
for master in restart_master_list: for master in restart_master_list:
print("master " + self.get_ip_by_hostname(master) + " server has down") print("master " + self.get_ip_by_hostname(master) + " server has down")
os.system('ssh ' + self.get_ip_by_hostname(master) + ' sh ' + install_path + '/bin/dolphinscheduler-daemon.sh start master-server') os.system('ssh -p ' + ssh_port + ' ' + self.get_ip_by_hostname(master) + ' sh ' + install_path + '/bin/dolphinscheduler-daemon.sh start master-server')
if (self.zk.exists(workers_zk_path)): if (self.zk.exists(workers_zk_path)):
zk_worker_list = [] zk_worker_list = []
@ -100,7 +103,7 @@ class ZkClient:
if (len(restart_worker_list) != 0): if (len(restart_worker_list) != 0):
for worker in restart_worker_list: for worker in restart_worker_list:
print("worker " + self.get_ip_by_hostname(worker) + " server has down") print("worker " + self.get_ip_by_hostname(worker) + " server has down")
os.system('ssh ' + self.get_ip_by_hostname(worker) + ' sh ' + install_path + '/bin/dolphinscheduler-daemon.sh start worker-server') os.system('ssh -p ' + ssh_port + ' ' + self.get_ip_by_hostname(worker) + ' sh ' + install_path + '/bin/dolphinscheduler-daemon.sh start worker-server')
print(datetime.now().strftime("%Y-%m-%d %H:%M:%S")) print(datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
schedule.enter(inc, 0, self.restart_server, (inc,)) schedule.enter(inc, 0, self.restart_server, (inc,))

18
script/scp-hosts.sh

@ -25,15 +25,15 @@ hostsArr=(${ips//,/ })
for host in ${hostsArr[@]} for host in ${hostsArr[@]}
do do
if ! ssh $host test -e $installPath; then if ! ssh -p $sshPort $host test -e $installPath; then
ssh $host "sudo mkdir -p $installPath; sudo chown -R $deployUser:$deployUser $installPath" ssh -p $sshPort $host "sudo mkdir -p $installPath; sudo chown -R $deployUser:$deployUser $installPath"
fi fi
ssh $host "cd $installPath/; rm -rf bin/ conf/ lib/ script/ sql/" ssh -p $sshPort $host "cd $installPath/; rm -rf bin/ conf/ lib/ script/ sql/"
scp -r $workDir/../bin $host:$installPath scp -P $sshPort -r $workDir/../bin $host:$installPath
scp -r $workDir/../conf $host:$installPath scp -P $sshPort -r $workDir/../conf $host:$installPath
scp -r $workDir/../lib $host:$installPath scp -P $sshPort -r $workDir/../lib $host:$installPath
scp -r $workDir/../script $host:$installPath scp -P $sshPort -r $workDir/../script $host:$installPath
scp -r $workDir/../sql $host:$installPath scp -P $sshPort -r $workDir/../sql $host:$installPath
scp $workDir/../install.sh $host:$installPath scp -P $sshPort $workDir/../install.sh $host:$installPath
done done

10
script/start-all.sh

@ -25,7 +25,7 @@ mastersHost=(${masters//,/ })
for master in ${mastersHost[@]} for master in ${mastersHost[@]}
do do
echo $master echo $master
ssh $master "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start master-server;" ssh -p $sshPort $master "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start master-server;"
done done
@ -34,17 +34,17 @@ for worker in ${workersHost[@]}
do do
echo $worker echo $worker
ssh $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start worker-server;" ssh -p $sshPort $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start worker-server;"
ssh $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start logger-server;" ssh -p $sshPort $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start logger-server;"
done done
ssh $alertServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start alert-server;" ssh -p $sshPort $alertServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start alert-server;"
apiServersHost=(${apiServers//,/ }) apiServersHost=(${apiServers//,/ })
for apiServer in ${apiServersHost[@]} for apiServer in ${apiServersHost[@]}
do do
echo $apiServer echo $apiServer
ssh $apiServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start api-server;" ssh -p $sshPort $apiServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start api-server;"
done done

10
script/stop-all.sh

@ -26,7 +26,7 @@ mastersHost=(${masters//,/ })
for master in ${mastersHost[@]} for master in ${mastersHost[@]}
do do
echo $master echo $master
ssh $master "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop master-server;" ssh -p $sshPort $master "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop master-server;"
done done
@ -35,17 +35,17 @@ for worker in ${workersHost[@]}
do do
echo $worker echo $worker
ssh $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop worker-server;" ssh -p $sshPort $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop worker-server;"
ssh $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop logger-server;" ssh -p $sshPort $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop logger-server;"
done done
ssh $alertServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop alert-server;" ssh -p $sshPort $alertServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop alert-server;"
apiServersHost=(${apiServers//,/ }) apiServersHost=(${apiServers//,/ })
for apiServer in ${apiServersHost[@]} for apiServer in ${apiServersHost[@]}
do do
echo $apiServer echo $apiServer
ssh $apiServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop api-server;" ssh -p $sshPort $apiServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop api-server;"
done done

Loading…
Cancel
Save