github
/
DolphinScheduler
mirror of https://gitee.com/dolphinscheduler/DolphinScheduler.git
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

install-escheduler-ui.sh,monitor_server.py and install.sh scripts comment change to english and install-escheduler-ui.sh use escheduler change to dolphinscheduler (#812) 

* service start exception modify

* master，worker start modify

* .env update

* install-escheduler-ui.sh,monitor_server.py and install.sh scripts comment change to english and install-escheduler-ui.sh use escheduler change to dolphinscheduler
pull/2/head
qiaozhanwei 5 years ago committed by lgcareer
parent
78d4ef9662
commit
8ee233f082
3 changed files with 195 additions and 186 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 80
      escheduler-ui/install-escheduler-ui.sh
  2. 256
      install.sh
  3. 45
      script/monitor_server.py

80
escheduler-ui/install-escheduler-ui.sh
Unescape View File

@ -1,21 +1,21 @@
#!/bin/bash #!/bin/bash
# 当前路径 # current path
esc_basepath=$(cd `dirname $0`; pwd) esc_basepath=$(cd `dirname $0`; pwd)
menu(){ menu(){
cat <<END cat <<END
================================================= =================================================
1.CentOS6安装 1.CentOS6 Installation
2.CentOS7安装 2.CentOS7 Installation
3.Ubuntu安装 3.Ubuntu Installation
4.退出 4.Exit
================================================= =================================================
END END
} }
# 创建文件并配置nginx # create a file and configure nginx
eschedulerConf(){ dolphinschedulerConf(){
E_host='$host' E_host='$host'
E_remote_addr='$remote_addr' E_remote_addr='$remote_addr'
@ -23,16 +23,16 @@ eschedulerConf(){
E_http_upgrade='$http_upgrade' E_http_upgrade='$http_upgrade'
echo " echo "
server { server {
listen $1;# 访问端口 listen $1;# access port
server_name localhost; server_name localhost;
#charset koi8-r; #charset koi8-r;
#access_log /var/log/nginx/host.access.log main; #access_log /var/log/nginx/host.access.log main;
location / { location / {
root ${esc_basepath}/dist; # 静态文件目录 root ${esc_basepath}/dist; # static file directory
index index.html index.html; index index.html index.html;
} }
location /escheduler { location /escheduler {
proxy_pass $2; # 接口地址 proxy_pass $2; # interface address
proxy_set_header Host $E_host; proxy_set_header Host $E_host;
proxy_set_header X-Real-IP $E_remote_addr; proxy_set_header X-Real-IP $E_remote_addr;
proxy_set_header x_real_ipP $E_remote_addr; proxy_set_header x_real_ipP $E_remote_addr;
@ -53,21 +53,21 @@ eschedulerConf(){
root /usr/share/nginx/html; root /usr/share/nginx/html;
} }
} }
" >> /etc/nginx/conf.d/escheduler.conf " >> /etc/nginx/conf.d/dolphinscheduler.conf
} }
ubuntu(){ ubuntu(){
#更新源 # update source
apt-get update apt-get update
#安装nginx # install nginx
apt-get install -y nginx apt-get install -y nginx
# 配置nginx # config nginx
eschedulerConf $1 $2 dolphinschedulerConf $1 $2
# 启动nginx # startup nginx
/etc/init.d/nginx start /etc/init.d/nginx start
sleep 1 sleep 1
if [ $? -ne 0 ];then if [ $? -ne 0 ];then
@ -81,17 +81,17 @@ centos7(){
rpm -Uvh http://nginx.org/packages/centos/7/noarch/RPMS/nginx-release-centos-7-0.el7.ngx.noarch.rpm rpm -Uvh http://nginx.org/packages/centos/7/noarch/RPMS/nginx-release-centos-7-0.el7.ngx.noarch.rpm
yum install -y nginx yum install -y nginx
# 配置nginx # config nginx
eschedulerConf $1 $2 dolphinschedulerConf $1 $2
# 解决 0.0.0.0:8888 问题 # solve 0.0.0.0:8888 problem
yum -y install policycoreutils-python yum -y install policycoreutils-python
semanage port -a -t http_port_t -p tcp $esc_proxy semanage port -a -t http_port_t -p tcp $esc_proxy
# 开放前端访问端口 # open front access port
firewall-cmd --zone=public --add-port=$esc_proxy/tcp --permanent firewall-cmd --zone=public --add-port=$esc_proxy/tcp --permanent
# 启动nginx # startup nginx
systemctl start nginx systemctl start nginx
sleep 1 sleep 1
if [ $? -ne 0 ];then if [ $? -ne 0 ];then
@ -99,9 +99,9 @@ centos7(){
fi fi
nginx -s reload nginx -s reload
# 调整SELinux的参数 # set SELinux parameters
sed -i "s/SELINUX=enforcing/SELINUX=disabled/g" /etc/selinux/config sed -i "s/SELINUX=enforcing/SELINUX=disabled/g" /etc/selinux/config
# 临时生效 # temporary effect
setenforce 0 setenforce 0
} }
@ -114,10 +114,10 @@ centos6(){
# install nginx # install nginx
yum install nginx -y yum install nginx -y
# 配置nginx # config nginx
eschedulerConf $1 $2 dolphinschedulerConf $1 $2
# 启动nginx # startup nginx
/etc/init.d/nginx start /etc/init.d/nginx start
sleep 1 sleep 1
if [ $? -ne 0 ];then if [ $? -ne 0 ];then
@ -125,17 +125,17 @@ centos6(){
fi fi
nginx -s reload nginx -s reload
# 调整SELinux的参数 # set SELinux parameters
sed -i "s/SELINUX=enforcing/SELINUX=disabled/g" /etc/selinux/config sed -i "s/SELINUX=enforcing/SELINUX=disabled/g" /etc/selinux/config
# 临时生效 # temporary effect
setenforce 0 setenforce 0
} }
function main(){ function main(){
echo "欢迎使用easy scheduler前端部署脚本,目前前端部署脚本仅支持CentOS,Ubuntu" echo "Welcome to thedolphinscheduler front-end deployment script, which is currently only supported by front-end deployment scripts : CentOS and Ubuntu"
echo "请在 escheduler-ui 目录下执行" echo "Please execute in the dolphinscheduler-ui directory"
#To be compatible with MacOS and Linux #To be compatible with MacOS and Linux
if [[ "$OSTYPE" == "darwin"* ]]; then if [[ "$OSTYPE" == "darwin"* ]]; then
@ -166,33 +166,33 @@ function main(){
fi fi
# 配置前端访问端口 # config front-end access ports
read -p "请输入nginx代理端口，不输入，则默认8888 :" esc_proxy_port read -p "Please enter the nginx proxy port, do not enter, the default is 8888 :" esc_proxy_port
if [ -z "${esc_proxy_port}" ];then if [ -z "${esc_proxy_port}" ];then
esc_proxy_port="8888" esc_proxy_port="8888"
fi fi
read -p "请输入api server代理ip,必须输入，例如：192.168.xx.xx :" esc_api_server_ip read -p "Please enter the api server proxy ip, you must enter, for example: 192.168.xx.xx :" esc_api_server_ip
if [ -z "${esc_api_server_ip}" ];then if [ -z "${esc_api_server_ip}" ];then
echo "api server代理ip不能为空." echo "api server proxy ip can not be empty."
exit 1 exit 1
fi fi
read -p "请输入api server代理端口,不输入，则默认12345 :" esc_api_server_port read -p "Please enter the api server proxy port, do not enter, the default is 12345:" esc_api_server_port
if [ -z "${esc_api_server_port}" ];then if [ -z "${esc_api_server_port}" ];then
esc_api_server_port="12345" esc_api_server_port="12345"
fi fi
# api server后端地址 # api server backend address
esc_api_server="http://$esc_api_server_ip:$esc_api_server_port" esc_api_server="http://$esc_api_server_ip:$esc_api_server_port"
# 本机ip地址 # local ip address
esc_ipaddr=$(ip a | grep inet | grep -v inet6 | grep -v 127 | sed 's/^[ \t]*//g' | cut -d ' ' -f2 | head -n 1 | awk -F '/' '{print $1}') esc_ipaddr=$(ip a | grep inet | grep -v inet6 | grep -v 127 | sed 's/^[ \t]*//g' | cut -d ' ' -f2 | head -n 1 | awk -F '/' '{print $1}')
# 提示信息 # Prompt message
menu menu
read -p "请输入安装编号(1|2|3|4)：" num read -p "Please enter the installation number(1|2|3|4)：" num
case $num in case $num in
1) 1)
@ -212,7 +212,7 @@ function main(){
echo $"Usage :sh $0" echo $"Usage :sh $0"
exit 1 exit 1
esac esac
echo "请浏览器访问：http://${esc_ipaddr}:${esc_proxy_port}" echo "Please visit the browser：http://${esc_ipaddr}:${esc_proxy_port}"
} }

256
install.sh
Unescape View File

@ -34,252 +34,257 @@ fi
source ${workDir}/conf/config/run_config.conf source ${workDir}/conf/config/run_config.conf
source ${workDir}/conf/config/install_config.conf source ${workDir}/conf/config/install_config.conf
# mysql配置 # mysql config
# mysql 地址,端口 # mysql address and port
mysqlHost="192.168.xx.xx:3306" mysqlHost="192.168.xx.xx:3306"
# mysql 数据库名称 # mysql database
mysqlDb="escheduler" mysqlDb="escheduler"
# mysql 用户名 # mysql username
mysqlUserName="xx" mysqlUserName="xx"
# mysql 密码 # mysql passwprd
# 注意：如果有特殊字符，请用 \ 转移符进行转移 # Note: if there are special characters, please use the \ transfer character to transfer
mysqlPassword="xx" mysqlPassword="xx"
# conf/config/install_config.conf配置 # conf/config/install_config.conf config
# 注意：安装路径,不要当前路径(pwd)一样 # Note: the installation path is not the same as the current path (pwd)
installPath="/data1_1T/escheduler" installPath="/data1_1T/escheduler"
# 部署用户 # deployment user
# 注意：部署用户需要有sudo权限及操作hdfs的权限，如果开启hdfs，根目录需要自行创建 # Note: the deployment user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled, the root directory needs to be created by itself
deployUser="escheduler" deployUser="escheduler"
# zk集群 # zk cluster
zkQuorum="192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181" zkQuorum="192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181"
# 安装hosts # install hosts
# 注意：安装调度的机器hostname列表，如果是伪分布式，则只需写一个伪分布式hostname即可 # Note: install the scheduled hostname list. If it is pseudo-distributed, just write a pseudo-distributed hostname
ips="ark0,ark1,ark2,ark3,ark4" ips="ark0,ark1,ark2,ark3,ark4"
# conf/config/run_config.conf配置 # conf/config/run_config.conf config
# 运行Master的机器 # run master machine
# 注意：部署master的机器hostname列表 # Note: list of hosts hostname for deploying master
masters="ark0,ark1" masters="ark0,ark1"
# 运行Worker的机器 # run worker machine
# 注意：部署worker的机器hostname列表 # note: list of machine hostnames for deploying workers
workers="ark2,ark3,ark4" workers="ark2,ark3,ark4"
# 运行Alert的机器 # run alert machine
# 注意：部署alert server的机器hostname列表 # note: list of machine hostnames for deploying alert server
alertServer="ark3" alertServer="ark3"
# 运行Api的机器 # run api machine
# 注意：部署api server的机器hostname列表 # note: list of machine hostnames for deploying api server
apiServers="ark1" apiServers="ark1"
# alert配置 # alert config
# 邮件协议 # mail protocol
mailProtocol="SMTP" mailProtocol="SMTP"
# 邮件服务host # mail server host
mailServerHost="smtp.exmail.qq.com" mailServerHost="smtp.exmail.qq.com"
# 邮件服务端口 # mail server port
mailServerPort="25" mailServerPort="25"
# 发送人 # sender
mailSender="xxxxxxxxxx" mailSender="xxxxxxxxxx"
# 发送人密码 # sender password
mailPassword="xxxxxxxxxx" mailPassword="xxxxxxxxxx"
# TLS邮件协议支持 # TLS mail protocol support
starttlsEnable="false" starttlsEnable="false"
# SSL邮件协议支持 # SSL mail protocol support
# 注意：默认开启的是SSL协议，TLS和SSL只能有一个处于true状态 # note: The SSL protocol is enabled by default.
# only one of TLS and SSL can be in the true state.
sslEnable="true" sslEnable="true"
# 下载Excel路径 # download excel path
xlsFilePath="/tmp/xls" xlsFilePath="/tmp/xls"
# 企业微信企业ID配置 # Enterprise WeChat Enterprise ID Configuration
enterpriseWechatCorpId="xxxxxxxxxx" enterpriseWechatCorpId="xxxxxxxxxx"
# 企业微信应用Secret配置 # Enterprise WeChat application Secret configuration
enterpriseWechatSecret="xxxxxxxxxx" enterpriseWechatSecret="xxxxxxxxxx"
# 企业微信应用AgentId配置 # Enterprise WeChat Application AgentId Configuration
enterpriseWechatAgentId="xxxxxxxxxx" enterpriseWechatAgentId="xxxxxxxxxx"
# 企业微信用户配置,多个用户以,分割 # Enterprise WeChat user configuration, multiple users to , split
enterpriseWechatUsers="xxxxx,xxxxx" enterpriseWechatUsers="xxxxx,xxxxx"
#是否启动监控自启动脚本 # whether to start monitoring self-starting scripts
monitorServerState="false" monitorServerState="false"
# 资源中心上传选择存储方式：HDFS,S3,NONE # resource Center upload and select storage method：HDFS,S3,NONE
resUploadStartupType="NONE" resUploadStartupType="NONE"
# 如果resUploadStartupType为HDFS，defaultFS写namenode地址，支持HA,需要将core-site.xml和hdfs-site.xml放到conf目录下 # if resUploadStartupType is HDFS，defaultFS write namenode address，HA you need to put core-site.xml and hdfs-site.xml in the conf directory.
# 如果是S3，则写S3地址，比如说：s3a://escheduler，注意，一定要创建根目录/escheduler # if S3，write S3 address，HA，for example ：s3a://escheduler，
# Note，s3 be sure to create the root directory /escheduler
defaultFS="hdfs://mycluster:8020" defaultFS="hdfs://mycluster:8020"
# 如果配置了S3，则需要有以下配置 # if S3 is configured, the following configuration is required.
s3Endpoint="http://192.168.xx.xx:9010" s3Endpoint="http://192.168.xx.xx:9010"
s3AccessKey="xxxxxxxxxx" s3AccessKey="xxxxxxxxxx"
s3SecretKey="xxxxxxxxxx" s3SecretKey="xxxxxxxxxx"
# resourcemanager HA配置，如果是单resourcemanager,这里为yarnHaIps="" # resourcemanager HA configuration, if it is a single resourcemanager, here is yarnHaIps=""
yarnHaIps="192.168.xx.xx,192.168.xx.xx" yarnHaIps="192.168.xx.xx,192.168.xx.xx"
# 如果是单 resourcemanager,只需要配置一个主机名称,如果是resourcemanager HA,则默认配置就好 # if it is a single resourcemanager, you only need to configure one host name. If it is resourcemanager HA, the default configuration is fine.
singleYarnIp="ark1" singleYarnIp="ark1"
# hdfs根路径，根路径的owner必须是部署用户。1.1.0之前版本不会自动创建hdfs根目录，需要自行创建 # hdfs root path, the owner of the root path must be the deployment user.
# versions prior to 1.1.0 do not automatically create the hdfs root directory, you need to create it yourself.
hdfsPath="/escheduler" hdfsPath="/escheduler"
# 拥有在hdfs根路径/下创建目录权限的用户 # have users who create directory permissions under hdfs root path /
# 注意：如果开启了kerberos，则直接hdfsRootUser=""，就可以 # Note: if kerberos is enabled, hdfsRootUser="" can be used directly.
hdfsRootUser="hdfs" hdfsRootUser="hdfs"
# common 配置 # common config
# 程序路径 # Program root path
programPath="/tmp/escheduler" programPath="/tmp/escheduler"
#下载路径 # download path
downloadPath="/tmp/escheduler/download" downloadPath="/tmp/escheduler/download"
# 任务执行路径 # task execute path
execPath="/tmp/escheduler/exec" execPath="/tmp/escheduler/exec"
# SHELL环境变量路径 # SHELL environmental variable path
shellEnvPath="$installPath/conf/env/.escheduler_env.sh" shellEnvPath="$installPath/conf/env/.escheduler_env.sh"
# 资源文件的后缀 # suffix of the resource file
resSuffixs="txt,log,sh,conf,cfg,py,java,sql,hql,xml" resSuffixs="txt,log,sh,conf,cfg,py,java,sql,hql,xml"
# 开发状态,如果是true,对于SHELL脚本可以在execPath目录下查看封装后的SHELL脚本,如果是false则执行完成直接删除 # development status, if true, for the SHELL script, you can view the encapsulated SHELL script in the execPath directory.
# If it is false, execute the direct delete
devState="true" devState="true"
# kerberos 配置 # kerberos config
# kerberos 是否启动 # kerberos whether to start
kerberosStartUp="false" kerberosStartUp="false"
# kdc krb5 配置文件路径 # kdc krb5 config file path
krb5ConfPath="$installPath/conf/krb5.conf" krb5ConfPath="$installPath/conf/krb5.conf"
# keytab 用户名 # keytab username
keytabUserName="hdfs-mycluster@ESZ.COM" keytabUserName="hdfs-mycluster@ESZ.COM"
# 用户 keytab路径 # username keytab path
keytabPath="$installPath/conf/hdfs.headless.keytab" keytabPath="$installPath/conf/hdfs.headless.keytab"
# zk 配置 # zk config
# zk根目录 # zk root directory
zkRoot="/escheduler" zkRoot="/escheduler"
# 用来记录挂掉机器的zk目录 # used to record the zk directory of the hanging machine
zkDeadServers="/escheduler/dead-servers" zkDeadServers="/escheduler/dead-servers"
# masters目录 # masters directory
zkMasters="/escheduler/masters" zkMasters="$zkRoot/masters"
# workers目录 # workers directory
zkWorkers="/escheduler/workers" zkWorkers="$zkRoot/workers"
# zk master分布式锁 # zk master distributed lock
mastersLock="/escheduler/lock/masters" mastersLock="$zkRoot/lock/masters"
# zk worker分布式锁 # zk worker distributed lock
workersLock="/escheduler/lock/workers" workersLock="$zkRoot/lock/workers"
# zk master容错分布式锁 # zk master fault-tolerant distributed lock
mastersFailover="/escheduler/lock/failover/masters" mastersFailover="$zkRoot/lock/failover/masters"
# zk worker容错分布式锁 # zk worker fault-tolerant distributed lock
workersFailover="/escheduler/lock/failover/workers" workersFailover="$zkRoot/lock/failover/workers"
# zk master启动容错分布式锁 # zk master start fault tolerant distributed lock
mastersStartupFailover="/escheduler/lock/failover/startup-masters" mastersStartupFailover="$zkRoot/lock/failover/startup-masters"
# zk session 超时 # zk session timeout
zkSessionTimeout="300" zkSessionTimeout="300"
# zk 连接超时 # zk connection timeout
zkConnectionTimeout="300" zkConnectionTimeout="300"
# zk 重试间隔 # zk retry interval
zkRetrySleep="100" zkRetrySleep="100"
# zk重试最大次数 # zk retry maximum number of times
zkRetryMaxtime="5" zkRetryMaxtime="5"
# master 配置 # master config
# master执行线程最大数,流程实例的最大并行度 # master execution thread maximum number, maximum parallelism of process instance
masterExecThreads="100" masterExecThreads="100"
# master任务执行线程最大数,每一个流程实例的最大并行度 # the maximum number of master task execution threads, the maximum degree of parallelism for each process instance
masterExecTaskNum="20" masterExecTaskNum="20"
# master心跳间隔 # master heartbeat interval
masterHeartbeatInterval="10" masterHeartbeatInterval="10"
# master任务提交重试次数 # master task submission retries
masterTaskCommitRetryTimes="5" masterTaskCommitRetryTimes="5"
# master任务提交重试时间间隔 # master task submission retry interval
masterTaskCommitInterval="100" masterTaskCommitInterval="100"
# master最大cpu平均负载,用来判断master是否还有执行能力 # master maximum cpu average load, used to determine whether the master has execution capability
masterMaxCpuLoadAvg="10" masterMaxCpuLoadAvg="10"
# master预留内存,用来判断master是否还有执行能力 # master reserve memory to determine if the master has execution capability
masterReservedMemory="1" masterReservedMemory="1"
# worker 配置 # worker config
# worker执行线程 # worker execution thread
workerExecThreads="100" workerExecThreads="100"
# worker心跳间隔 # worker heartbeat interval
workerHeartbeatInterval="10" workerHeartbeatInterval="10"
# worker一次抓取任务数 # worker number of fetch tasks
workerFetchTaskNum="3" workerFetchTaskNum="3"
# worker最大cpu平均负载,用来判断worker是否还有执行能力,保持系统默认，默认为cpu核数的2倍，当负载达到2倍时， # workerThe maximum cpu average load, used to determine whether the worker still has the ability to execute,
# keep the system default, the default is twice the number of cpu cores, when the load reaches 2 times
#workerMaxCupLoadAvg="10" #workerMaxCupLoadAvg="10"
# worker预留内存,用来判断master是否还有执行能力 # worker reserve memory to determine if the master has execution capability
workerReservedMemory="1" workerReservedMemory="1"
# api 配置 # api config
# api 服务端口 # api server port
apiServerPort="12345" apiServerPort="12345"
# api session 超时 # api session timeout
apiServerSessionTimeout="7200" apiServerSessionTimeout="7200"
# api 上下文路径 # api server context path
apiServerContextPath="/escheduler/" apiServerContextPath="/escheduler/"
# spring 最大文件大小 # spring max file size
springMaxFileSize="1024MB" springMaxFileSize="1024MB"
# spring 最大请求文件大小 # spring max request size
springMaxRequestSize="1024MB" springMaxRequestSize="1024MB"
# api 最大post请求大小 # api max http post size
apiMaxHttpPostSize="5000000" apiMaxHttpPostSize="5000000"
# 1,替换文件 # 1,replace file
echo "1,替换文件" echo "1,replace file"
sed -i ${txt} "s#spring.datasource.url.*#spring.datasource.url=jdbc:mysql://${mysqlHost}/${mysqlDb}?characterEncoding=UTF-8#g" conf/dao/data_source.properties sed -i ${txt} "s#spring.datasource.url.*#spring.datasource.url=jdbc:mysql://${mysqlHost}/${mysqlDb}?characterEncoding=UTF-8#g" conf/dao/data_source.properties
sed -i ${txt} "s#spring.datasource.username.*#spring.datasource.username=${mysqlUserName}#g" conf/dao/data_source.properties sed -i ${txt} "s#spring.datasource.username.*#spring.datasource.username=${mysqlUserName}#g" conf/dao/data_source.properties
sed -i ${txt} "s#spring.datasource.password.*#spring.datasource.password=${mysqlPassword}#g" conf/dao/data_source.properties sed -i ${txt} "s#spring.datasource.password.*#spring.datasource.password=${mysqlPassword}#g" conf/dao/data_source.properties
@ -375,8 +380,8 @@ sed -i ${txt} "s#alertServer.*#alertServer=${alertServer}#g" conf/config/run_con
sed -i ${txt} "s#apiServers.*#apiServers=${apiServers}#g" conf/config/run_config.conf sed -i ${txt} "s#apiServers.*#apiServers=${apiServers}#g" conf/config/run_config.conf
# 2,创建目录 # 2,create directory
echo "2,创建目录" echo "2,create directory"
if [ ! -d $installPath ];then if [ ! -d $installPath ];then
sudo mkdir -p $installPath sudo mkdir -p $installPath
@ -387,22 +392,22 @@ hostsArr=(${ips//,/ })
for host in ${hostsArr[@]} for host in ${hostsArr[@]}
do do
# 如果programPath不存在,则创建 # create if programPath does not exist
if ! ssh $host test -e $programPath; then if ! ssh $host test -e $programPath; then
ssh $host "sudo mkdir -p $programPath;sudo chown -R $deployUser:$deployUser $programPath" ssh $host "sudo mkdir -p $programPath;sudo chown -R $deployUser:$deployUser $programPath"
fi fi
# 如果downloadPath不存在,则创建 # create if downloadPath does not exist
if ! ssh $host test -e $downloadPath; then if ! ssh $host test -e $downloadPath; then
ssh $host "sudo mkdir -p $downloadPath;sudo chown -R $deployUser:$deployUser $downloadPath" ssh $host "sudo mkdir -p $downloadPath;sudo chown -R $deployUser:$deployUser $downloadPath"
fi fi
# 如果$execPath不存在,则创建 # create if execPath does not exist
if ! ssh $host test -e $execPath; then if ! ssh $host test -e $execPath; then
ssh $host "sudo mkdir -p $execPath; sudo chown -R $deployUser:$deployUser $execPath" ssh $host "sudo mkdir -p $execPath; sudo chown -R $deployUser:$deployUser $execPath"
fi fi
# 如果$xlsFilePath不存在,则创建 # create if xlsFilePath does not exist
if ! ssh $host test -e $xlsFilePath; then if ! ssh $host test -e $xlsFilePath; then
ssh $host "sudo mkdir -p $xlsFilePath; sudo chown -R $deployUser:$deployUser $xlsFilePath" ssh $host "sudo mkdir -p $xlsFilePath; sudo chown -R $deployUser:$deployUser $xlsFilePath"
fi fi
@ -410,31 +415,31 @@ fi
done done
# 3,停止服务 # 3,stop server
echo "3,停止服务" echo "3,stop server"
sh ${workDir}/script/stop_all.sh sh ${workDir}/script/stop-all.sh
# 4,删除zk节点 # 4,delete zk node
echo "4,删除zk节点" echo "4,delete zk node"
sleep 1 sleep 1
python ${workDir}/script/del_zk_node.py $zkQuorum $zkRoot python ${workDir}/script/del-zk-node.py $zkQuorum $zkRoot
# 5,scp资源 # 5,scp resources
echo "5,scp资源" echo "5,scp resources"
sh ${workDir}/script/scp_hosts.sh sh ${workDir}/script/scp-hosts.sh
if [ $? -eq 0 ] if [ $? -eq 0 ]
then then
echo 'scp拷贝完成' echo 'scp copy completed'
else else
echo 'sc 拷贝失败退出' echo 'sc copy failed to exit'
exit -1 exit -1
fi fi
# 6,启动 # 6,startup
echo "6,启动" echo "6,startup"
sh ${workDir}/script/start_all.sh sh ${workDir}/script/start-all.sh
# 7,启动监控自启动脚本 # 7,start monitoring self-starting script
monitor_pid=${workDir}/monitor_server.pid monitor_pid=${workDir}/monitor_server.pid
if [ "true" = $monitorServerState ];then if [ "true" = $monitorServerState ];then
if [ -f $monitor_pid ]; then if [ -f $monitor_pid ]; then
@ -453,9 +458,8 @@ if [ "true" = $monitorServerState ];then
echo "monitor server running as process ${TARGET_PID}.Stopped success" echo "monitor server running as process ${TARGET_PID}.Stopped success"
rm -f $monitor_pid rm -f $monitor_pid
fi fi
nohup python -u ${workDir}/script/monitor_server.py $installPath $zkQuorum $zkMasters $zkWorkers > ${workDir}/monitor_server.log 2>&1 & nohup python -u ${workDir}/script/monitor-server.py $installPath $zkQuorum $zkMasters $zkWorkers > ${workDir}/monitor-server.log 2>&1 &
echo $! > $monitor_pid echo $! > $monitor_pid
echo "start monitor server success as process `cat $monitor_pid`" echo "start monitor server success as process `cat $monitor_pid`"
fi fi

45
script/monitor_server.py
Unescape View File

@ -1,21 +1,26 @@
#!/usr/bin/env python #!/usr/bin/env python
# -*- coding:utf-8 -*- # -*- coding:utf-8 -*-
# Author:qiaozhanwei
''' '''
yum 安装pip 1, yum install pip
yum -y install python-pip yum -y install python-pip
pip install kazoo 安装 2, pip install kazoo
conda install -c conda-forge kazoo 安装 pip install kazoo
运行脚本及参数说明： or
3, conda install kazoo
conda install -c conda-forge kazoo
run script and parameter description：
nohup python -u monitor_server.py /data1_1T/escheduler 192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181 /escheduler/masters /escheduler/workers> monitor_server.log 2>&1 & nohup python -u monitor_server.py /data1_1T/escheduler 192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181 /escheduler/masters /escheduler/workers> monitor_server.log 2>&1 &
参数说明如下: the parameters are as follows:
/data1_1T/escheduler的值来自install.sh中的installPath /data1_1T/escheduler : the value comes from the installPath in install.sh
192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181的值来自install.sh中的zkQuorum 192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181 : the value comes from zkQuorum in install.sh
/escheduler/masters的值来自install.sh中的zkMasters the value comes from zkWorkers in install.sh
/escheduler/workers的值来自install.sh中的zkWorkers /escheduler/masters : the value comes from zkMasters in install.sh
/escheduler/workers : the value comes from zkWorkers in install.sh
''' '''
import sys import sys
import socket import socket
@ -29,11 +34,11 @@ schedule = sched.scheduler(time.time, time.sleep)
class ZkClient: class ZkClient:
def __init__(self): def __init__(self):
# hosts配置zk地址集群 # hosts configuration zk address cluster
self.zk = KazooClient(hosts=zookeepers) self.zk = KazooClient(hosts=zookeepers)
self.zk.start() self.zk.start()
# 读取配置文件，组装成字典 # read configuration files and assemble them into a dictionary
def read_file(self,path): def read_file(self,path):
with open(path, 'r') as f: with open(path, 'r') as f:
dict = {} dict = {}
@ -43,11 +48,11 @@ class ZkClient:
dict[arr[0]] = arr[1] dict[arr[0]] = arr[1]
return dict return dict
# 根据hostname获取ip地址 # get the ip address according to hostname
def get_ip_by_hostname(self,hostname): def get_ip_by_hostname(self,hostname):
return socket.gethostbyname(hostname) return socket.gethostbyname(hostname)
# 重启服务 # restart server
def restart_server(self,inc): def restart_server(self,inc):
config_dict = self.read_file(install_path + '/conf/config/run_config.conf') config_dict = self.read_file(install_path + '/conf/config/run_config.conf')
@ -67,7 +72,7 @@ class ZkClient:
restart_master_list = list(set(master_list) - set(zk_master_list)) restart_master_list = list(set(master_list) - set(zk_master_list))
if (len(restart_master_list) != 0): if (len(restart_master_list) != 0):
for master in restart_master_list: for master in restart_master_list:
print("master " + self.get_ip_by_hostname(master) + " 服务已经掉了") print("master " + self.get_ip_by_hostname(master) + " server has down")
os.system('ssh ' + self.get_ip_by_hostname(master) + ' sh ' + install_path + '/bin/escheduler-daemon.sh start master-server') os.system('ssh ' + self.get_ip_by_hostname(master) + ' sh ' + install_path + '/bin/escheduler-daemon.sh start master-server')
if (self.zk.exists(workers_zk_path)): if (self.zk.exists(workers_zk_path)):
@ -78,15 +83,15 @@ class ZkClient:
restart_worker_list = list(set(worker_list) - set(zk_worker_list)) restart_worker_list = list(set(worker_list) - set(zk_worker_list))
if (len(restart_worker_list) != 0): if (len(restart_worker_list) != 0):
for worker in restart_worker_list: for worker in restart_worker_list:
print("worker " + self.get_ip_by_hostname(worker) + " 服务已经掉了") print("worker " + self.get_ip_by_hostname(worker) + " server has down")
os.system('ssh ' + self.get_ip_by_hostname(worker) + ' sh ' + install_path + '/bin/escheduler-daemon.sh start worker-server') os.system('ssh ' + self.get_ip_by_hostname(worker) + ' sh ' + install_path + '/bin/escheduler-daemon.sh start worker-server')
print(datetime.now().strftime("%Y-%m-%d %H:%M:%S")) print(datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
schedule.enter(inc, 0, self.restart_server, (inc,)) schedule.enter(inc, 0, self.restart_server, (inc,))
# 默认参数60s # default parameter 60s
def main(self,inc=60): def main(self,inc=60):
# enter四个参数分别为：间隔事件、优先级（用于同时间到达的两个事件同时执行时定序）、被调用触发的函数， # the enter four parameters are: interval event, priority (sequence for simultaneous execution of two events arriving at the same time), function triggered by the call，
# 给该触发函数的参数（tuple形式） # the argument to the trigger function (tuple form)
schedule.enter(0, 0, self.restart_server, (inc,)) schedule.enter(0, 0, self.restart_server, (inc,))
schedule.run() schedule.run()
if __name__ == '__main__': if __name__ == '__main__':
@ -97,4 +102,4 @@ if __name__ == '__main__':
masters_zk_path = sys.argv[3] masters_zk_path = sys.argv[3]
workers_zk_path = sys.argv[4] workers_zk_path = sys.argv[4]
zkClient = ZkClient() zkClient = ZkClient()
zkClient.main(300) zkClient.main(300)
Write Preview
Loading…
Cancel
Save