diff --git a/escheduler-ui/install-escheduler-ui.sh b/escheduler-ui/install-escheduler-ui.sh index be9a3801ac..3c9578746a 100755 --- a/escheduler-ui/install-escheduler-ui.sh +++ b/escheduler-ui/install-escheduler-ui.sh @@ -1,21 +1,21 @@ #!/bin/bash -# 当前路径 +# current path esc_basepath=$(cd `dirname $0`; pwd) menu(){ cat <> /etc/nginx/conf.d/escheduler.conf + " >> /etc/nginx/conf.d/dolphinscheduler.conf } ubuntu(){ - #更新源 + # update source apt-get update - #安装nginx + # install nginx apt-get install -y nginx - # 配置nginx - eschedulerConf $1 $2 + # config nginx + dolphinschedulerConf $1 $2 - # 启动nginx + # startup nginx /etc/init.d/nginx start sleep 1 if [ $? -ne 0 ];then @@ -81,17 +81,17 @@ centos7(){ rpm -Uvh http://nginx.org/packages/centos/7/noarch/RPMS/nginx-release-centos-7-0.el7.ngx.noarch.rpm yum install -y nginx - # 配置nginx - eschedulerConf $1 $2 + # config nginx + dolphinschedulerConf $1 $2 - # 解决 0.0.0.0:8888 问题 + # solve 0.0.0.0:8888 problem yum -y install policycoreutils-python semanage port -a -t http_port_t -p tcp $esc_proxy - # 开放前端访问端口 + # open front access port firewall-cmd --zone=public --add-port=$esc_proxy/tcp --permanent - # 启动nginx + # startup nginx systemctl start nginx sleep 1 if [ $? -ne 0 ];then @@ -99,9 +99,9 @@ centos7(){ fi nginx -s reload - # 调整SELinux的参数 + # set SELinux parameters sed -i "s/SELINUX=enforcing/SELINUX=disabled/g" /etc/selinux/config - # 临时生效 + # temporary effect setenforce 0 } @@ -114,10 +114,10 @@ centos6(){ # install nginx yum install nginx -y - # 配置nginx - eschedulerConf $1 $2 + # config nginx + dolphinschedulerConf $1 $2 - # 启动nginx + # startup nginx /etc/init.d/nginx start sleep 1 if [ $? -ne 0 ];then @@ -125,17 +125,17 @@ centos6(){ fi nginx -s reload - # 调整SELinux的参数 + # set SELinux parameters sed -i "s/SELINUX=enforcing/SELINUX=disabled/g" /etc/selinux/config - # 临时生效 + # temporary effect setenforce 0 } function main(){ - echo "欢迎使用easy scheduler前端部署脚本,目前前端部署脚本仅支持CentOS,Ubuntu" - echo "请在 escheduler-ui 目录下执行" + echo "Welcome to thedolphinscheduler front-end deployment script, which is currently only supported by front-end deployment scripts : CentOS and Ubuntu" + echo "Please execute in the dolphinscheduler-ui directory" #To be compatible with MacOS and Linux if [[ "$OSTYPE" == "darwin"* ]]; then @@ -166,33 +166,33 @@ function main(){ fi - # 配置前端访问端口 - read -p "请输入nginx代理端口，不输入，则默认8888 :" esc_proxy_port + # config front-end access ports + read -p "Please enter the nginx proxy port, do not enter, the default is 8888 :" esc_proxy_port if [ -z "${esc_proxy_port}" ];then esc_proxy_port="8888" fi - read -p "请输入api server代理ip,必须输入，例如：192.168.xx.xx :" esc_api_server_ip + read -p "Please enter the api server proxy ip, you must enter, for example: 192.168.xx.xx :" esc_api_server_ip if [ -z "${esc_api_server_ip}" ];then - echo "api server代理ip不能为空." + echo "api server proxy ip can not be empty." exit 1 fi - read -p "请输入api server代理端口,不输入，则默认12345 :" esc_api_server_port + read -p "Please enter the api server proxy port, do not enter, the default is 12345:" esc_api_server_port if [ -z "${esc_api_server_port}" ];then esc_api_server_port="12345" fi - # api server后端地址 + # api server backend address esc_api_server="http://$esc_api_server_ip:$esc_api_server_port" - # 本机ip地址 + # local ip address esc_ipaddr=$(ip a | grep inet | grep -v inet6 | grep -v 127 | sed 's/^[ \t]*//g' | cut -d ' ' -f2 | head -n 1 | awk -F '/' '{print $1}') - # 提示信息 + # Prompt message menu - read -p "请输入安装编号(1|2|3|4)：" num + read -p "Please enter the installation number(1|2|3|4)：" num case $num in 1) @@ -212,7 +212,7 @@ function main(){ echo $"Usage :sh $0" exit 1 esac - echo "请浏览器访问：http://${esc_ipaddr}:${esc_proxy_port}" + echo "Please visit the browser：http://${esc_ipaddr}:${esc_proxy_port}" } diff --git a/install.sh b/install.sh index 894a4c2f18..dc09ce3c97 100644 --- a/install.sh +++ b/install.sh @@ -34,252 +34,257 @@ fi source ${workDir}/conf/config/run_config.conf source ${workDir}/conf/config/install_config.conf -# mysql配置 -# mysql 地址,端口 +# mysql config +# mysql address and port mysqlHost="192.168.xx.xx:3306" -# mysql 数据库名称 +# mysql database mysqlDb="escheduler" -# mysql 用户名 +# mysql username mysqlUserName="xx" -# mysql 密码 -# 注意：如果有特殊字符，请用 \ 转移符进行转移 +# mysql passwprd +# Note: if there are special characters, please use the \ transfer character to transfer mysqlPassword="xx" -# conf/config/install_config.conf配置 -# 注意：安装路径,不要当前路径(pwd)一样 +# conf/config/install_config.conf config +# Note: the installation path is not the same as the current path (pwd) installPath="/data1_1T/escheduler" -# 部署用户 -# 注意：部署用户需要有sudo权限及操作hdfs的权限，如果开启hdfs，根目录需要自行创建 +# deployment user +# Note: the deployment user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled, the root directory needs to be created by itself deployUser="escheduler" -# zk集群 +# zk cluster zkQuorum="192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181" -# 安装hosts -# 注意：安装调度的机器hostname列表，如果是伪分布式，则只需写一个伪分布式hostname即可 +# install hosts +# Note: install the scheduled hostname list. If it is pseudo-distributed, just write a pseudo-distributed hostname ips="ark0,ark1,ark2,ark3,ark4" -# conf/config/run_config.conf配置 -# 运行Master的机器 -# 注意：部署master的机器hostname列表 +# conf/config/run_config.conf config +# run master machine +# Note: list of hosts hostname for deploying master masters="ark0,ark1" -# 运行Worker的机器 -# 注意：部署worker的机器hostname列表 +# run worker machine +# note: list of machine hostnames for deploying workers workers="ark2,ark3,ark4" -# 运行Alert的机器 -# 注意：部署alert server的机器hostname列表 +# run alert machine +# note: list of machine hostnames for deploying alert server alertServer="ark3" -# 运行Api的机器 -# 注意：部署api server的机器hostname列表 +# run api machine +# note: list of machine hostnames for deploying api server apiServers="ark1" -# alert配置 -# 邮件协议 +# alert config +# mail protocol mailProtocol="SMTP" -# 邮件服务host +# mail server host mailServerHost="smtp.exmail.qq.com" -# 邮件服务端口 +# mail server port mailServerPort="25" -# 发送人 +# sender mailSender="xxxxxxxxxx" -# 发送人密码 +# sender password mailPassword="xxxxxxxxxx" -# TLS邮件协议支持 +# TLS mail protocol support starttlsEnable="false" -# SSL邮件协议支持 -# 注意：默认开启的是SSL协议，TLS和SSL只能有一个处于true状态 +# SSL mail protocol support +# note: The SSL protocol is enabled by default. +# only one of TLS and SSL can be in the true state. sslEnable="true" -# 下载Excel路径 +# download excel path xlsFilePath="/tmp/xls" -# 企业微信企业ID配置 +# Enterprise WeChat Enterprise ID Configuration enterpriseWechatCorpId="xxxxxxxxxx" -# 企业微信应用Secret配置 +# Enterprise WeChat application Secret configuration enterpriseWechatSecret="xxxxxxxxxx" -# 企业微信应用AgentId配置 +# Enterprise WeChat Application AgentId Configuration enterpriseWechatAgentId="xxxxxxxxxx" -# 企业微信用户配置,多个用户以,分割 +# Enterprise WeChat user configuration, multiple users to , split enterpriseWechatUsers="xxxxx,xxxxx" -#是否启动监控自启动脚本 +# whether to start monitoring self-starting scripts monitorServerState="false" -# 资源中心上传选择存储方式：HDFS,S3,NONE +# resource Center upload and select storage method：HDFS,S3,NONE resUploadStartupType="NONE" -# 如果resUploadStartupType为HDFS，defaultFS写namenode地址，支持HA,需要将core-site.xml和hdfs-site.xml放到conf目录下 -# 如果是S3，则写S3地址，比如说：s3a://escheduler，注意，一定要创建根目录/escheduler +# if resUploadStartupType is HDFS，defaultFS write namenode address，HA you need to put core-site.xml and hdfs-site.xml in the conf directory. +# if S3，write S3 address，HA，for example ：s3a://escheduler， +# Note，s3 be sure to create the root directory /escheduler defaultFS="hdfs://mycluster:8020" -# 如果配置了S3，则需要有以下配置 +# if S3 is configured, the following configuration is required. s3Endpoint="http://192.168.xx.xx:9010" s3AccessKey="xxxxxxxxxx" s3SecretKey="xxxxxxxxxx" -# resourcemanager HA配置，如果是单resourcemanager,这里为yarnHaIps="" +# resourcemanager HA configuration, if it is a single resourcemanager, here is yarnHaIps="" yarnHaIps="192.168.xx.xx,192.168.xx.xx" -# 如果是单 resourcemanager,只需要配置一个主机名称,如果是resourcemanager HA,则默认配置就好 +# if it is a single resourcemanager, you only need to configure one host name. If it is resourcemanager HA, the default configuration is fine. singleYarnIp="ark1" -# hdfs根路径，根路径的owner必须是部署用户。1.1.0之前版本不会自动创建hdfs根目录，需要自行创建 +# hdfs root path, the owner of the root path must be the deployment user. +# versions prior to 1.1.0 do not automatically create the hdfs root directory, you need to create it yourself. hdfsPath="/escheduler" -# 拥有在hdfs根路径/下创建目录权限的用户 -# 注意：如果开启了kerberos，则直接hdfsRootUser=""，就可以 +# have users who create directory permissions under hdfs root path / +# Note: if kerberos is enabled, hdfsRootUser="" can be used directly. hdfsRootUser="hdfs" -# common 配置 -# 程序路径 +# common config +# Program root path programPath="/tmp/escheduler" -#下载路径 +# download path downloadPath="/tmp/escheduler/download" -# 任务执行路径 +# task execute path execPath="/tmp/escheduler/exec" -# SHELL环境变量路径 +# SHELL environmental variable path shellEnvPath="$installPath/conf/env/.escheduler_env.sh" -# 资源文件的后缀 +# suffix of the resource file resSuffixs="txt,log,sh,conf,cfg,py,java,sql,hql,xml" -# 开发状态,如果是true,对于SHELL脚本可以在execPath目录下查看封装后的SHELL脚本,如果是false则执行完成直接删除 +# development status, if true, for the SHELL script, you can view the encapsulated SHELL script in the execPath directory. +# If it is false, execute the direct delete devState="true" -# kerberos 配置 -# kerberos 是否启动 +# kerberos config +# kerberos whether to start kerberosStartUp="false" -# kdc krb5 配置文件路径 +# kdc krb5 config file path krb5ConfPath="$installPath/conf/krb5.conf" -# keytab 用户名 +# keytab username keytabUserName="hdfs-mycluster@ESZ.COM" -# 用户 keytab路径 +# username keytab path keytabPath="$installPath/conf/hdfs.headless.keytab" -# zk 配置 -# zk根目录 +# zk config +# zk root directory zkRoot="/escheduler" -# 用来记录挂掉机器的zk目录 +# used to record the zk directory of the hanging machine zkDeadServers="/escheduler/dead-servers" -# masters目录 -zkMasters="/escheduler/masters" +# masters directory +zkMasters="$zkRoot/masters" -# workers目录 -zkWorkers="/escheduler/workers" +# workers directory +zkWorkers="$zkRoot/workers" -# zk master分布式锁 -mastersLock="/escheduler/lock/masters" +# zk master distributed lock +mastersLock="$zkRoot/lock/masters" -# zk worker分布式锁 -workersLock="/escheduler/lock/workers" +# zk worker distributed lock +workersLock="$zkRoot/lock/workers" -# zk master容错分布式锁 -mastersFailover="/escheduler/lock/failover/masters" +# zk master fault-tolerant distributed lock +mastersFailover="$zkRoot/lock/failover/masters" -# zk worker容错分布式锁 -workersFailover="/escheduler/lock/failover/workers" +# zk worker fault-tolerant distributed lock +workersFailover="$zkRoot/lock/failover/workers" -# zk master启动容错分布式锁 -mastersStartupFailover="/escheduler/lock/failover/startup-masters" +# zk master start fault tolerant distributed lock +mastersStartupFailover="$zkRoot/lock/failover/startup-masters" -# zk session 超时 +# zk session timeout zkSessionTimeout="300" -# zk 连接超时 +# zk connection timeout zkConnectionTimeout="300" -# zk 重试间隔 +# zk retry interval zkRetrySleep="100" -# zk重试最大次数 +# zk retry maximum number of times zkRetryMaxtime="5" -# master 配置 -# master执行线程最大数,流程实例的最大并行度 +# master config +# master execution thread maximum number, maximum parallelism of process instance masterExecThreads="100" -# master任务执行线程最大数,每一个流程实例的最大并行度 +# the maximum number of master task execution threads, the maximum degree of parallelism for each process instance masterExecTaskNum="20" -# master心跳间隔 +# master heartbeat interval masterHeartbeatInterval="10" -# master任务提交重试次数 +# master task submission retries masterTaskCommitRetryTimes="5" -# master任务提交重试时间间隔 +# master task submission retry interval masterTaskCommitInterval="100" -# master最大cpu平均负载,用来判断master是否还有执行能力 +# master maximum cpu average load, used to determine whether the master has execution capability masterMaxCpuLoadAvg="10" -# master预留内存,用来判断master是否还有执行能力 +# master reserve memory to determine if the master has execution capability masterReservedMemory="1" -# worker 配置 -# worker执行线程 +# worker config +# worker execution thread workerExecThreads="100" -# worker心跳间隔 +# worker heartbeat interval workerHeartbeatInterval="10" -# worker一次抓取任务数 +# worker number of fetch tasks workerFetchTaskNum="3" -# worker最大cpu平均负载,用来判断worker是否还有执行能力,保持系统默认，默认为cpu核数的2倍，当负载达到2倍时， +# workerThe maximum cpu average load, used to determine whether the worker still has the ability to execute, +# keep the system default, the default is twice the number of cpu cores, when the load reaches 2 times #workerMaxCupLoadAvg="10" -# worker预留内存,用来判断master是否还有执行能力 +# worker reserve memory to determine if the master has execution capability workerReservedMemory="1" -# api 配置 -# api 服务端口 +# api config +# api server port apiServerPort="12345" -# api session 超时 +# api session timeout apiServerSessionTimeout="7200" -# api 上下文路径 +# api server context path apiServerContextPath="/escheduler/" -# spring 最大文件大小 +# spring max file size springMaxFileSize="1024MB" -# spring 最大请求文件大小 +# spring max request size springMaxRequestSize="1024MB" -# api 最大post请求大小 +# api max http post size apiMaxHttpPostSize="5000000" -# 1,替换文件 -echo "1,替换文件" +# 1,replace file +echo "1,replace file" sed -i ${txt} "s#spring.datasource.url.*#spring.datasource.url=jdbc:mysql://${mysqlHost}/${mysqlDb}?characterEncoding=UTF-8#g" conf/dao/data_source.properties sed -i ${txt} "s#spring.datasource.username.*#spring.datasource.username=${mysqlUserName}#g" conf/dao/data_source.properties sed -i ${txt} "s#spring.datasource.password.*#spring.datasource.password=${mysqlPassword}#g" conf/dao/data_source.properties @@ -375,8 +380,8 @@ sed -i ${txt} "s#alertServer.*#alertServer=${alertServer}#g" conf/config/run_con sed -i ${txt} "s#apiServers.*#apiServers=${apiServers}#g" conf/config/run_config.conf -# 2,创建目录 -echo "2,创建目录" +# 2,create directory +echo "2,create directory" if [ ! -d $installPath ];then sudo mkdir -p $installPath @@ -387,22 +392,22 @@ hostsArr=(${ips//,/ }) for host in ${hostsArr[@]} do -# 如果programPath不存在,则创建 +# create if programPath does not exist if ! ssh $host test -e $programPath; then ssh $host "sudo mkdir -p $programPath;sudo chown -R $deployUser:$deployUser $programPath" fi -# 如果downloadPath不存在,则创建 +# create if downloadPath does not exist if ! ssh $host test -e $downloadPath; then ssh $host "sudo mkdir -p $downloadPath;sudo chown -R $deployUser:$deployUser $downloadPath" fi -# 如果$execPath不存在,则创建 +# create if execPath does not exist if ! ssh $host test -e $execPath; then ssh $host "sudo mkdir -p $execPath; sudo chown -R $deployUser:$deployUser $execPath" fi -# 如果$xlsFilePath不存在,则创建 +# create if xlsFilePath does not exist if ! ssh $host test -e $xlsFilePath; then ssh $host "sudo mkdir -p $xlsFilePath; sudo chown -R $deployUser:$deployUser $xlsFilePath" fi @@ -410,31 +415,31 @@ fi done -# 3,停止服务 -echo "3,停止服务" -sh ${workDir}/script/stop_all.sh +# 3,stop server +echo "3,stop server" +sh ${workDir}/script/stop-all.sh -# 4,删除zk节点 -echo "4,删除zk节点" +# 4,delete zk node +echo "4,delete zk node" sleep 1 -python ${workDir}/script/del_zk_node.py $zkQuorum $zkRoot +python ${workDir}/script/del-zk-node.py $zkQuorum $zkRoot -# 5,scp资源 -echo "5,scp资源" -sh ${workDir}/script/scp_hosts.sh +# 5,scp resources +echo "5,scp resources" +sh ${workDir}/script/scp-hosts.sh if [ $? -eq 0 ] then - echo 'scp拷贝完成' + echo 'scp copy completed' else - echo 'sc 拷贝失败退出' + echo 'sc copy failed to exit' exit -1 fi -# 6,启动 -echo "6,启动" -sh ${workDir}/script/start_all.sh +# 6,startup +echo "6,startup" +sh ${workDir}/script/start-all.sh -# 7,启动监控自启动脚本 +# 7,start monitoring self-starting script monitor_pid=${workDir}/monitor_server.pid if [ "true" = $monitorServerState ];then if [ -f $monitor_pid ]; then @@ -453,9 +458,8 @@ if [ "true" = $monitorServerState ];then echo "monitor server running as process ${TARGET_PID}.Stopped success" rm -f $monitor_pid fi - nohup python -u ${workDir}/script/monitor_server.py $installPath $zkQuorum $zkMasters $zkWorkers > ${workDir}/monitor_server.log 2>&1 & + nohup python -u ${workDir}/script/monitor-server.py $installPath $zkQuorum $zkMasters $zkWorkers > ${workDir}/monitor-server.log 2>&1 & echo $! > $monitor_pid echo "start monitor server success as process `cat $monitor_pid`" -fi - +fi \ No newline at end of file diff --git a/script/monitor_server.py b/script/monitor_server.py index 5f236cac7e..546104c8f6 100644 --- a/script/monitor_server.py +++ b/script/monitor_server.py @@ -1,21 +1,26 @@ #!/usr/bin/env python # -*- coding:utf-8 -*- -# Author:qiaozhanwei ''' -yum 安装pip +1, yum install pip yum -y install python-pip -pip install kazoo 安装 -conda install -c conda-forge kazoo 安装 +2, pip install kazoo +pip install kazoo -运行脚本及参数说明： +or + +3, conda install kazoo +conda install -c conda-forge kazoo + +run script and parameter description： nohup python -u monitor_server.py /data1_1T/escheduler 192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181 /escheduler/masters /escheduler/workers> monitor_server.log 2>&1 & -参数说明如下: -/data1_1T/escheduler的值来自install.sh中的installPath -192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181的值来自install.sh中的zkQuorum -/escheduler/masters的值来自install.sh中的zkMasters -/escheduler/workers的值来自install.sh中的zkWorkers +the parameters are as follows: +/data1_1T/escheduler : the value comes from the installPath in install.sh +192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181 : the value comes from zkQuorum in install.sh +the value comes from zkWorkers in install.sh +/escheduler/masters : the value comes from zkMasters in install.sh +/escheduler/workers : the value comes from zkWorkers in install.sh ''' import sys import socket @@ -29,11 +34,11 @@ schedule = sched.scheduler(time.time, time.sleep) class ZkClient: def __init__(self): - # hosts配置zk地址集群 + # hosts configuration zk address cluster self.zk = KazooClient(hosts=zookeepers) self.zk.start() - # 读取配置文件，组装成字典 + # read configuration files and assemble them into a dictionary def read_file(self,path): with open(path, 'r') as f: dict = {} @@ -43,11 +48,11 @@ class ZkClient: dict[arr[0]] = arr[1] return dict - # 根据hostname获取ip地址 + # get the ip address according to hostname def get_ip_by_hostname(self,hostname): return socket.gethostbyname(hostname) - # 重启服务 + # restart server def restart_server(self,inc): config_dict = self.read_file(install_path + '/conf/config/run_config.conf') @@ -67,7 +72,7 @@ class ZkClient: restart_master_list = list(set(master_list) - set(zk_master_list)) if (len(restart_master_list) != 0): for master in restart_master_list: - print("master " + self.get_ip_by_hostname(master) + " 服务已经掉了") + print("master " + self.get_ip_by_hostname(master) + " server has down") os.system('ssh ' + self.get_ip_by_hostname(master) + ' sh ' + install_path + '/bin/escheduler-daemon.sh start master-server') if (self.zk.exists(workers_zk_path)): @@ -78,15 +83,15 @@ class ZkClient: restart_worker_list = list(set(worker_list) - set(zk_worker_list)) if (len(restart_worker_list) != 0): for worker in restart_worker_list: - print("worker " + self.get_ip_by_hostname(worker) + " 服务已经掉了") + print("worker " + self.get_ip_by_hostname(worker) + " server has down") os.system('ssh ' + self.get_ip_by_hostname(worker) + ' sh ' + install_path + '/bin/escheduler-daemon.sh start worker-server') print(datetime.now().strftime("%Y-%m-%d %H:%M:%S")) schedule.enter(inc, 0, self.restart_server, (inc,)) - # 默认参数60s + # default parameter 60s def main(self,inc=60): - # enter四个参数分别为：间隔事件、优先级（用于同时间到达的两个事件同时执行时定序）、被调用触发的函数， - # 给该触发函数的参数（tuple形式） + # the enter four parameters are: interval event, priority (sequence for simultaneous execution of two events arriving at the same time), function triggered by the call， + # the argument to the trigger function (tuple form) schedule.enter(0, 0, self.restart_server, (inc,)) schedule.run() if __name__ == '__main__': @@ -97,4 +102,4 @@ if __name__ == '__main__': masters_zk_path = sys.argv[3] workers_zk_path = sys.argv[4] zkClient = ZkClient() - zkClient.main(300) + zkClient.main(300) \ No newline at end of file