diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index c57d13b54d..0000000000 --- a/Dockerfile +++ /dev/null @@ -1,41 +0,0 @@ -#Maintin by jimmy -#Email: zhengge2012@gmail.com -FROM anapsix/alpine-java:8_jdk -WORKDIR /tmp -RUN wget http://archive.apache.org/dist/maven/maven-3/3.6.1/binaries/apache-maven-3.6.1-bin.tar.gz -RUN tar -zxvf apache-maven-3.6.1-bin.tar.gz && rm apache-maven-3.6.1-bin.tar.gz -RUN mv apache-maven-3.6.1 /usr/lib/mvn -RUN chown -R root:root /usr/lib/mvn -RUN ln -s /usr/lib/mvn/bin/mvn /usr/bin/mvn -RUN wget https://archive.apache.org/dist/zookeeper/zookeeper-3.4.6/zookeeper-3.4.6.tar.gz -RUN tar -zxvf zookeeper-3.4.6.tar.gz -RUN mv zookeeper-3.4.6 /opt/zookeeper -RUN rm -rf zookeeper-3.4.6.tar.gz -RUN echo "export ZOOKEEPER_HOME=/opt/zookeeper" >>/etc/profile -RUN echo "export PATH=$PATH:$ZOOKEEPER_HOME/bin" >>/etc/profile -ADD conf/zoo.cfg /opt/zookeeper/conf/zoo.cfg -#RUN source /etc/profile -#RUN zkServer.sh start -RUN apk add --no-cache git npm nginx mariadb mariadb-client mariadb-server-utils pwgen -WORKDIR /opt -RUN git clone https://github.com/analysys/EasyScheduler.git -WORKDIR /opt/EasyScheduler -RUN mvn -U clean package assembly:assembly -Dmaven.test.skip=true -RUN mv /opt/EasyScheduler/target/escheduler-1.0.0-SNAPSHOT /opt/easyscheduler -WORKDIR /opt/EasyScheduler/escheduler-ui -RUN npm install -RUN npm audit fix -RUN npm run build -RUN mkdir -p /opt/escheduler/front/server -RUN cp -rfv dist/* /opt/escheduler/front/server -WORKDIR / -RUN rm -rf /opt/EasyScheduler -#configure mysql server https://github.com/yobasystems/alpine-mariadb/tree/master/alpine-mariadb-amd64 -ADD conf/run.sh /scripts/run.sh -RUN mkdir /docker-entrypoint-initdb.d && \ - mkdir /scripts/pre-exec.d && \ - mkdir /scripts/pre-init.d && \ - chmod -R 755 /scripts -RUN rm -rf /var/cache/apk/* -EXPOSE 8888 -ENTRYPOINT ["/scripts/run.sh"] diff --git a/README.md b/README.md index c6f48b2bc3..edb75d4230 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -Easy Scheduler +Easy Scheduler ============ [![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html) diff --git a/conf/escheduler.conf b/conf/escheduler.conf deleted file mode 100644 index fa331e62f5..0000000000 --- a/conf/escheduler.conf +++ /dev/null @@ -1,31 +0,0 @@ -server { - listen 8888;# 访问端口 - server_name localhost; - #charset koi8-r; - #access_log /var/log/nginx/host.access.log main; - location / { - root /opt/escheduler/front/server; # 静态文件目录 - index index.html index.html; - } - location /escheduler { - proxy_pass http://127.0.0.1:12345; # 接口地址 - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header x_real_ipP $remote_addr; - proxy_set_header remote_addr $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_http_version 1.1; - proxy_connect_timeout 4s; - proxy_read_timeout 30s; - proxy_send_timeout 12s; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - } - #error_page 404 /404.html; - # redirect server error pages to the static page /50x.html - # - error_page 500 502 503 504 /50x.html; - location = /50x.html { - root /usr/share/nginx/html; - } -} diff --git a/conf/install.sh b/conf/install.sh deleted file mode 100644 index 10419b8f84..0000000000 --- a/conf/install.sh +++ /dev/null @@ -1,310 +0,0 @@ -#!/bin/sh - -workDir=`/opt/easyscheduler` -workDir=`cd ${workDir};pwd` - -#To be compatible with MacOS and Linux -txt="" -if [[ "$OSTYPE" == "darwin"* ]]; then - # Mac OSX - txt="''" -elif [[ "$OSTYPE" == "linux-gnu" ]]; then - # linux - txt="" -elif [[ "$OSTYPE" == "cygwin" ]]; then - # POSIX compatibility layer and Linux environment emulation for Windows - echo "Easy Scheduler not support Windows operating system" - exit 1 -elif [[ "$OSTYPE" == "msys" ]]; then - # Lightweight shell and GNU utilities compiled for Windows (part of MinGW) - echo "Easy Scheduler not support Windows operating system" - exit 1 -elif [[ "$OSTYPE" == "win32" ]]; then - echo "Easy Scheduler not support Windows operating system" - exit 1 -elif [[ "$OSTYPE" == "freebsd"* ]]; then - # ... - txt="" -else - # Unknown. - echo "Operating system unknown, please tell us(submit issue) for better service" - exit 1 -fi - -source ${workDir}/conf/config/run_config.conf -source ${workDir}/conf/config/install_config.conf - -# mysql配置 -# mysql 地址,端口 -mysqlHost="127.0.0.1:3306" - -# mysql 数据库名称 -mysqlDb="easyscheduler" - -# mysql 用户名 -mysqlUserName="easyscheduler" - -# mysql 密码 -mysqlPassword="easyschedulereasyscheduler" - -# conf/config/install_config.conf配置 -# 安装路径,不要当前路径(pwd)一样 -installPath="/opt/easyscheduler" - -# 部署用户 -deployUser="escheduler" - -# zk集群 -zkQuorum="192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181" - -# 安装hosts -ips="ark0,ark1,ark2,ark3,ark4" - -# conf/config/run_config.conf配置 -# 运行Master的机器 -masters="ark0,ark1" - -# 运行Worker的机器 -workers="ark2,ark3,ark4" - -# 运行Alert的机器 -alertServer="ark3" - -# 运行Api的机器 -apiServers="ark1" - -# alert配置 -# 邮件协议 -mailProtocol="SMTP" - -# 邮件服务host -mailServerHost="smtp.exmail.qq.com" - -# 邮件服务端口 -mailServerPort="25" - -# 发送人 -mailSender="xxxxxxxxxx" - -# 发送人密码 -mailPassword="xxxxxxxxxx" - -# 下载Excel路径 -xlsFilePath="/tmp/xls" - - -# hadoop 配置 -# 是否启动hdfs,如果启动则为true,需要配置以下hadoop相关参数; -# 不启动设置为false,如果为false,以下配置不需要修改 -hdfsStartupSate="false" - -# namenode地址,支持HA,需要将core-site.xml和hdfs-site.xml放到conf目录下 -namenodeFs="hdfs://mycluster:8020" - -# resourcemanager HA配置，如果是单resourcemanager,这里为空即可 -yarnHaIps="192.168.xx.xx,192.168.xx.xx" - -# 如果是单 resourcemanager,只需要配置一个主机名称,如果是resourcemanager HA,则默认配置就好 -singleYarnIp="ark1" - -# hdfs根路径,根路径的owner必须是部署用户 -hdfsPath="/escheduler" - -# common 配置 -# 程序路径 -programPath="/tmp/escheduler" - -#下载路径 -downloadPath="/tmp/escheduler/download" - -# 任务执行路径 -execPath="/tmp/escheduler/exec" - -# SHELL环境变量路径 -shellEnvPath="$installPath/conf/env/.escheduler_env.sh" - -# Python换将变量路径 -pythonEnvPath="$installPath/conf/env/escheduler_env.py" - -# 资源文件的后缀 -resSuffixs="txt,log,sh,conf,cfg,py,java,sql,hql,xml" - -# 开发状态,如果是true,对于SHELL脚本可以在execPath目录下查看封装后的SHELL脚本,如果是false则执行完成直接删除 -devState="true" - -# zk 配置 -# zk根目录 -zkRoot="/escheduler" - -# 用来记录挂掉机器的zk目录 -zkDeadServers="/escheduler/dead-servers" - -# masters目录 -zkMasters="/escheduler/masters" - -# workers目录 -zkWorkers="/escheduler/workers" - -# zk master分布式锁 -mastersLock="/escheduler/lock/masters" - -# zk worker分布式锁 -workersLock="/escheduler/lock/workers" - -# zk master容错分布式锁 -mastersFailover="/escheduler/lock/failover/masters" - -# zk worker容错分布式锁 -workersFailover="/escheduler/lock/failover/masters" - -# zk session 超时 -zkSessionTimeout="300" - -# zk 连接超时 -zkConnectionTimeout="300" - -# zk 重试间隔 -zkRetrySleep="100" - -# zk重试最大次数 -zkRetryMaxtime="5" - - -# master 配置 -# master执行线程最大数,流程实例的最大并行度 -masterExecThreads="100" - -# master任务执行线程最大数,每一个流程实例的最大并行度 -masterExecTaskNum="20" - -# master心跳间隔 -masterHeartbeatInterval="10" - -# master任务提交重试次数 -masterTaskCommitRetryTimes="5" - -# master任务提交重试时间间隔 -masterTaskCommitInterval="100" - -# master最大cpu平均负载,用来判断master是否还有执行能力 -masterMaxCupLoadAvg="10" - -# master预留内存,用来判断master是否还有执行能力 -masterReservedMemory="1" - - -# worker 配置 -# worker执行线程 -workerExecThreads="100" - -# worker心跳间隔 -workerHeartbeatInterval="10" - -# worker一次抓取任务数 -workerFetchTaskNum="10" - -# worker最大cpu平均负载,用来判断master是否还有执行能力 -workerMaxCupLoadAvg="10" - -# worker预留内存,用来判断master是否还有执行能力 -workerReservedMemory="1" - -# api 配置 -# api 服务端口 -apiServerPort="12345" - -# api session 超时 -apiServerSessionTimeout="7200" - -# api 上下文路径 -apiServerContextPath="/escheduler/" - -# spring 最大文件大小 -springMaxFileSize="1024MB" - -# spring 最大请求文件大小 -springMaxRequestSize="1024MB" - -# api 最大post请求大小 -apiMaxHttpPostSize="5000000" - -# 1,替换文件 -echo "1,替换文件" -sed -i ${txt} "s#spring.datasource.url.*#spring.datasource.url=jdbc:mysql://${mysqlHost}/${mysqlDb}?characterEncoding=UTF-8#g" conf/dao/data_source.properties -sed -i ${txt} "s#spring.datasource.username.*#spring.datasource.username=${mysqlUserName}#g" conf/dao/data_source.properties -sed -i ${txt} "s#spring.datasource.password.*#spring.datasource.password=${mysqlPassword}#g" conf/dao/data_source.properties - -sed -i ${txt} "s#org.quartz.dataSource.myDs.URL.*#org.quartz.dataSource.myDs.URL=jdbc:mysql://${mysqlHost}/${mysqlDb}?characterEncoding=UTF-8#g" conf/quartz.properties -sed -i ${txt} "s#org.quartz.dataSource.myDs.user.*#org.quartz.dataSource.myDs.user=${mysqlUserName}#g" conf/quartz.properties -sed -i ${txt} "s#org.quartz.dataSource.myDs.password.*#org.quartz.dataSource.myDs.password=${mysqlPassword}#g" conf/quartz.properties - - -sed -i ${txt} "s#fs.defaultFS.*#fs.defaultFS=${namenodeFs}#g" conf/common/hadoop/hadoop.properties -sed -i ${txt} "s#yarn.resourcemanager.ha.rm.ids.*#yarn.resourcemanager.ha.rm.ids=${yarnHaIps}#g" conf/common/hadoop/hadoop.properties -sed -i ${txt} "s#yarn.application.status.address.*#yarn.application.status.address=http://${singleYarnIp}:8088/ws/v1/cluster/apps/%s#g" conf/common/hadoop/hadoop.properties - -sed -i ${txt} "s#data.basedir.path.*#data.basedir.path=${programPath}#g" conf/common/common.properties -sed -i ${txt} "s#data.download.basedir.path.*#data.download.basedir.path=${downloadPath}#g" conf/common/common.properties -sed -i ${txt} "s#process.exec.basepath.*#process.exec.basepath=${execPath}#g" conf/common/common.properties -sed -i ${txt} "s#data.store2hdfs.basepath.*#data.store2hdfs.basepath=${hdfsPath}#g" conf/common/common.properties -sed -i ${txt} "s#hdfs.startup.state.*#hdfs.startup.state=${hdfsStartupSate}#g" conf/common/common.properties -sed -i ${txt} "s#escheduler.env.path.*#escheduler.env.path=${shellEnvPath}#g" conf/common/common.properties -sed -i ${txt} "s#escheduler.env.py.*#escheduler.env.py=${pythonEnvPath}#g" conf/common/common.properties -sed -i ${txt} "s#resource.view.suffixs.*#resource.view.suffixs=${resSuffixs}#g" conf/common/common.properties -sed -i ${txt} "s#development.state.*#development.state=${devState}#g" conf/common/common.properties - -sed -i ${txt} "s#zookeeper.quorum.*#zookeeper.quorum=${zkQuorum}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.escheduler.root.*#zookeeper.escheduler.root=${zkRoot}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.escheduler.dead.servers.*#zookeeper.escheduler.dead.servers=${zkDeadServers}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.escheduler.masters.*#zookeeper.escheduler.masters=${zkMasters}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.escheduler.workers.*#zookeeper.escheduler.workers=${zkWorkers}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.escheduler.lock.masters.*#zookeeper.escheduler.lock.masters=${mastersLock}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.escheduler.lock.workers.*#zookeeper.escheduler.lock.workers=${workersLock}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.escheduler.lock.failover.masters.*#zookeeper.escheduler.lock.failover.masters=${mastersFailover}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.escheduler.lock.failover.workers.*#zookeeper.escheduler.lock.failover.workers=${workersFailover}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.session.timeout.*#zookeeper.session.timeout=${zkSessionTimeout}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.connection.timeout.*#zookeeper.connection.timeout=${zkConnectionTimeout}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.retry.sleep.*#zookeeper.retry.sleep=${zkRetrySleep}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.retry.maxtime.*#zookeeper.retry.maxtime=${zkRetryMaxtime}#g" conf/zookeeper.properties - -sed -i ${txt} "s#master.exec.threads.*#master.exec.threads=${masterExecThreads}#g" conf/master.properties -sed -i ${txt} "s#master.exec.task.number.*#master.exec.task.number=${masterExecTaskNum}#g" conf/master.properties -sed -i ${txt} "s#master.heartbeat.interval.*#master.heartbeat.interval=${masterHeartbeatInterval}#g" conf/master.properties -sed -i ${txt} "s#master.task.commit.retryTimes.*#master.task.commit.retryTimes=${masterTaskCommitRetryTimes}#g" conf/master.properties -sed -i ${txt} "s#master.task.commit.interval.*#master.task.commit.interval=${masterTaskCommitInterval}#g" conf/master.properties -sed -i ${txt} "s#master.max.cpuload.avg.*#master.max.cpuload.avg=${masterMaxCupLoadAvg}#g" conf/master.properties -sed -i ${txt} "s#master.reserved.memory.*#master.reserved.memory=${masterReservedMemory}#g" conf/master.properties - - -sed -i ${txt} "s#worker.exec.threads.*#worker.exec.threads=${workerExecThreads}#g" conf/worker.properties -sed -i ${txt} "s#worker.heartbeat.interval.*#worker.heartbeat.interval=${workerHeartbeatInterval}#g" conf/worker.properties -sed -i ${txt} "s#worker.fetch.task.num.*#worker.fetch.task.num=${workerFetchTaskNum}#g" conf/worker.properties -sed -i ${txt} "s#worker.max.cpuload.avg.*#worker.max.cpuload.avg=${workerMaxCupLoadAvg}#g" conf/worker.properties -sed -i ${txt} "s#worker.reserved.memory.*#worker.reserved.memory=${workerReservedMemory}#g" conf/worker.properties - - -sed -i ${txt} "s#server.port.*#server.port=${apiServerPort}#g" conf/application.properties -sed -i ${txt} "s#server.session.timeout.*#server.session.timeout=${apiServerSessionTimeout}#g" conf/application.properties -sed -i ${txt} "s#server.context-path.*#server.context-path=${apiServerContextPath}#g" conf/application.properties -sed -i ${txt} "s#spring.http.multipart.max-file-size.*#spring.http.multipart.max-file-size=${springMaxFileSize}#g" conf/application.properties -sed -i ${txt} "s#spring.http.multipart.max-request-size.*#spring.http.multipart.max-request-size=${springMaxRequestSize}#g" conf/application.properties -sed -i ${txt} "s#server.max-http-post-size.*#server.max-http-post-size=${apiMaxHttpPostSize}#g" conf/application.properties - - -sed -i ${txt} "s#mail.protocol.*#mail.protocol=${mailProtocol}#g" conf/alert.properties -sed -i ${txt} "s#mail.server.host.*#mail.server.host=${mailServerHost}#g" conf/alert.properties -sed -i ${txt} "s#mail.server.port.*#mail.server.port=${mailServerPort}#g" conf/alert.properties -sed -i ${txt} "s#mail.sender.*#mail.sender=${mailSender}#g" conf/alert.properties -sed -i ${txt} "s#mail.passwd.*#mail.passwd=${mailPassword}#g" conf/alert.properties -sed -i ${txt} "s#xls.file.path.*#xls.file.path=${xlsFilePath}#g" conf/alert.properties - - -sed -i ${txt} "s#installPath.*#installPath=${installPath}#g" conf/config/install_config.conf -sed -i ${txt} "s#deployUser.*#deployUser=${deployUser}#g" conf/config/install_config.conf -sed -i ${txt} "s#ips.*#ips=${ips}#g" conf/config/install_config.conf - - -sed -i ${txt} "s#masters.*#masters=${masters}#g" conf/config/run_config.conf -sed -i ${txt} "s#workers.*#workers=${workers}#g" conf/config/run_config.conf -sed -i ${txt} "s#alertServer.*#alertServer=${alertServer}#g" conf/config/run_config.conf -sed -i ${txt} "s#apiServers.*#apiServers=${apiServers}#g" conf/config/run_config.conf diff --git a/conf/run.sh b/conf/run.sh deleted file mode 100644 index 3257b0e194..0000000000 --- a/conf/run.sh +++ /dev/null @@ -1,105 +0,0 @@ -#!/bin/sh - -# execute any pre-init scripts -for i in /scripts/pre-init.d/*sh -do - if [ -e "${i}" ]; then - echo "[i] pre-init.d - processing $i" - . "${i}" - fi -done - -if [ -d "/run/mysqld" ]; then - echo "[i] mysqld already present, skipping creation" - chown -R mysql:mysql /run/mysqld -else - echo "[i] mysqld not found, creating...." - mkdir -p /run/mysqld - chown -R mysql:mysql /run/mysqld -fi - -if [ -d /var/lib/mysql/mysql ]; then - echo "[i] MySQL directory already present, skipping creation" - chown -R mysql:mysql /var/lib/mysql -else - echo "[i] MySQL data directory not found, creating initial DBs" - - chown -R mysql:mysql /var/lib/mysql - - mysql_install_db --user=mysql --ldata=/var/lib/mysql > /dev/null - - if [ "$MYSQL_ROOT_PASSWORD" = "" ]; then - MYSQL_ROOT_PASSWORD=`pwgen 16 1` - echo "[i] MySQL root Password: $MYSQL_ROOT_PASSWORD" - fi - - MYSQL_DATABASE="easyscheduler" - MYSQL_USER="easyscheduler" - MYSQL_PASSWORD="easyschedulereasyscheduler" - - tfile=`mktemp` - if [ ! -f "$tfile" ]; then - return 1 - fi - - cat << EOF > $tfile -USE mysql; -FLUSH PRIVILEGES ; -GRANT ALL ON *.* TO 'root'@'%' identified by '$MYSQL_ROOT_PASSWORD' WITH GRANT OPTION ; -GRANT ALL ON *.* TO 'root'@'localhost' identified by '$MYSQL_ROOT_PASSWORD' WITH GRANT OPTION ; -SET PASSWORD FOR 'root'@'localhost'=PASSWORD('${MYSQL_ROOT_PASSWORD}') ; -DROP DATABASE IF EXISTS test ; -FLUSH PRIVILEGES ; -EOF - - if [ "$MYSQL_DATABASE" != "" ]; then - echo "[i] Creating database: $MYSQL_DATABASE" - echo "CREATE DATABASE IF NOT EXISTS \`$MYSQL_DATABASE\` CHARACTER SET utf8 COLLATE utf8_general_ci;" >> $tfile - - if [ "$MYSQL_USER" != "" ]; then - echo "[i] Creating user: $MYSQL_USER with password $MYSQL_PASSWORD" - echo "GRANT ALL ON \`$MYSQL_DATABASE\`.* to '$MYSQL_USER'@'%' IDENTIFIED BY '$MYSQL_PASSWORD';" >> $tfile - fi - fi - - /usr/bin/mysqld --user=mysql --bootstrap --verbose=0 --skip-name-resolve --skip-networking=0 < $tfile - rm -f $tfile - - for f in /docker-entrypoint-initdb.d/*; do - case "$f" in - *.sql) echo "$0: running $f"; /usr/bin/mysqld --user=mysql --bootstrap --verbose=0 --skip-name-resolve --skip-networking=0 < "$f"; echo ;; - *.sql.gz) echo "$0: running $f"; gunzip -c "$f" | /usr/bin/mysqld --user=mysql --bootstrap --verbose=0 --skip-name-resolve --skip-networking=0 < "$f"; echo ;; - *) echo "$0: ignoring or entrypoint initdb empty $f" ;; - esac - echo - done - - echo - echo 'MySQL init process done. Ready for start up.' - echo - - echo "exec /usr/bin/mysqld --user=mysql --console --skip-name-resolve --skip-networking=0" "$@" -fi - -# execute any pre-exec scripts -for i in /scripts/pre-exec.d/*sh -do - if [ -e "${i}" ]; then - echo "[i] pre-exec.d - processing $i" - . ${i} - fi -done - -mysql -ueasyscheduler -peasyschedulereasyscheduler --one-database easyscheduler -h127.0.0.1 < /opt/easyscheduler/sql/escheduler.sql -mysql -ueasyscheduler -peasyschedulereasyscheduler --one-database easyscheduler -h127.0.0.1 < /opt/easyscheduler/sql/quartz.sql -source /etc/profile -zkServer.sh start -cd /opt/easyscheduler -rm -rf /etc/nginx/conf.d/default.conf -sh ./bin/escheduler-daemon.sh start master-server -sh ./bin/escheduler-daemon.sh start worker-server -sh ./bin/escheduler-daemon.sh start api-server -sh ./bin/escheduler-daemon.sh start logger-server -sh ./bin/escheduler-daemon.sh start alert-server -nginx -c /etc/nginx/nginx.conf -exec /usr/bin/mysqld --user=mysql --console --skip-name-resolve --skip-networking=0 $@ diff --git a/conf/zoo.cfg b/conf/zoo.cfg deleted file mode 100644 index 228b7224a0..0000000000 --- a/conf/zoo.cfg +++ /dev/null @@ -1,30 +0,0 @@ -# The number of milliseconds of each tick -tickTime=2000 -# The number of ticks that the initial -# synchronization phase can take -initLimit=10 -# The number of ticks that can pass between -# sending a request and getting an acknowledgement -syncLimit=5 -# the directory where the snapshot is stored. -# do not use /tmp for storage, /tmp here is just -# example sakes. -dataDir=/tmp/zookeeper -# the port at which the clients will connect -clientPort=2181 -# the maximum number of client connections. -# increase this if you need to handle more clients -#maxClientCnxns=60 -# -# Be sure to read the maintenance section of the -# administrator guide before turning on autopurge. -# -# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance -# -# The number of snapshots to retain in dataDir -#autopurge.snapRetainCount=3 -# Purge task interval in hours -# Set to "0" to disable auto purge feature -#autopurge.purgeInterval=1 -dataDir=/opt/zookeeper/data -dataLogDir=/opt/zookeeper/logs diff --git a/docs/zh_CN/后端部署文档.md b/docs/zh_CN/后端部署文档.md index 4851686b65..f2df8a6989 100644 --- a/docs/zh_CN/后端部署文档.md +++ b/docs/zh_CN/后端部署文档.md @@ -4,7 +4,7 @@ ## 1、准备工作 -目前最新安装包版本是1.0.2，下载地址： [码云下载](https://gitee.com/easyscheduler/EasyScheduler/attach_files/) ，下载escheduler-backend-1.0.2.tar.gz(后端简称escheduler-backend),escheduler-ui-1.0.2.tar.gz(前端简称escheduler-ui) +目前最新安装包版本是1.0.3，下载地址： [码云下载](https://gitee.com/easyscheduler/EasyScheduler/attach_files/) ，下载escheduler-backend-1.0.3.tar.gz(后端简称escheduler-backend),escheduler-ui-1.0.3.tar.gz(前端简称escheduler-ui) #### 准备一: 基础软件安装(必装项请自行安装) @@ -101,6 +101,12 @@ install.sh : 一键部署脚本 - 修改部署参数(根据自己服务器及业务情况): - 修改 **install.sh**中的各参数，替换成自身业务所需的值 + - monitorServerState 开关变量,在1.0.3版本中增加，控制是否启动自启动脚本(监控master,worker状态,如果掉线会自动启动) + 默认值为"false"表示不启动自启动脚本,如果需要启动改为"true" + + - hdfsStartupSate 开关变量，控制是否启动hdfs + 默认值为"false"表示不启动hdfs + 如果需要启动改为"true",启动hdfs需要自行创建hdfs根路径，也就是install.sh中的 hdfsPath - 如果使用hdfs相关功能，需要拷贝**hdfs-site.xml**和**core-site.xml**到conf目录下 @@ -143,7 +149,7 @@ install.sh : 一键部署脚本 ### 2.2 编译源码来部署 -将源码包release版本1.0.2下载后，解压进入根目录 +将源码包release版本1.0.3下载后，解压进入根目录 * 执行编译命令： diff --git a/escheduler-alert/src/main/resources/alert.properties b/escheduler-alert/src/main/resources/alert.properties index e2cba1160d..2fc2bb70f9 100644 --- a/escheduler-alert/src/main/resources/alert.properties +++ b/escheduler-alert/src/main/resources/alert.properties @@ -9,7 +9,7 @@ mail.sender=xxxxxxx mail.passwd=xxxxxxx #xls file path,need create if not exist -xls.file.path=/opt/xls +xls.file.path=/tmp/xls diff --git a/escheduler-alert/src/main/resources/mail_templates/alert_mail_template.ftl b/escheduler-alert/src/main/resources/mail_templates/alert_mail_template.ftl index 6a729c2097..0ff763fa28 100644 --- a/escheduler-alert/src/main/resources/mail_templates/alert_mail_template.ftl +++ b/escheduler-alert/src/main/resources/mail_templates/alert_mail_template.ftl @@ -1,38 +1 @@ - - - easyscheduler - - - - - - - - <#if title??> ${title} - - <#if content??> ${content} -
- - \ No newline at end of file + easyscheduler<#if title??> ${title}<#if content??> ${content}
\ No newline at end of file diff --git a/escheduler-api/pom.xml b/escheduler-api/pom.xml index bfa01dba1e..73a3927749 100644 --- a/escheduler-api/pom.xml +++ b/escheduler-api/pom.xml @@ -34,6 +34,10 @@ leveldbjni-all org.fusesource.leveldbjni + + protobuf-java + com.google.protobuf + diff --git a/escheduler-api/src/main/java/cn/escheduler/api/quartz/QuartzExecutors.java b/escheduler-api/src/main/java/cn/escheduler/api/quartz/QuartzExecutors.java index 92e351d5cd..5cb3b33853 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/quartz/QuartzExecutors.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/quartz/QuartzExecutors.java @@ -26,8 +26,8 @@ import org.quartz.impl.matchers.GroupMatcher; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Calendar; import java.util.*; +import java.util.Calendar; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -226,8 +226,12 @@ public class QuartzExecutors { public boolean deleteJob(String jobName, String jobGroupName) { lock.writeLock().lock(); try { - logger.info("try to delete job, job name: {}, job group name: {},", jobName, jobGroupName); - return scheduler.deleteJob(new JobKey(jobName, jobGroupName)); + JobKey jobKey = new JobKey(jobName,jobGroupName); + if(scheduler.checkExists(jobKey)){ + logger.info("try to delete job, job name: {}, job group name: {},", jobName, jobGroupName); + return scheduler.deleteJob(jobKey); + } + } catch (SchedulerException e) { logger.error(String.format("delete job : %s failed",jobName), e); } finally { diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/SchedulerService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/SchedulerService.java index f2cf8e1905..5ea5faf83d 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/SchedulerService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/SchedulerService.java @@ -514,6 +514,13 @@ public class SchedulerService extends BaseService { putMsg(result, Status.SCHEDULE_CRON_NOT_EXISTS, scheduleId); return result; } + + // Determine if the login user is the owner of the schedule + if (loginUser.getId() != schedule.getUserId()) { + putMsg(result, Status.USER_NO_OPERATION_PERM); + return result; + } + // check schedule is already online if(schedule.getReleaseState() == ReleaseState.ONLINE){ putMsg(result, Status.SCHEDULE_CRON_STATE_ONLINE,schedule.getId()); diff --git a/escheduler-common/pom.xml b/escheduler-common/pom.xml index 4f76b55cac..afe34dbd3b 100644 --- a/escheduler-common/pom.xml +++ b/escheduler-common/pom.xml @@ -148,10 +148,10 @@ javax.servlet.jsp jsp-api - - com.google.protobuf - protobuf-java - + + + + @@ -175,10 +175,10 @@ org.codehaus.jackson jackson-xc - - com.google.protobuf - protobuf-java - + + + + org.fusesource.leveldbjni leveldbjni-all diff --git a/escheduler-common/src/main/java/cn/escheduler/common/Constants.java b/escheduler-common/src/main/java/cn/escheduler/common/Constants.java index abf29fdc99..b12145973e 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/Constants.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/Constants.java @@ -162,6 +162,11 @@ public final class Constants { */ public static final String ZOOKEEPER_ESCHEDULER_LOCK_FAILOVER_WORKERS = "zookeeper.escheduler.lock.failover.workers"; + /** + * MasterServer startup failover runing and fault tolerance process + */ + public static final String ZOOKEEPER_ESCHEDULER_LOCK_FAILOVER_STARTUP_MASTERS = "zookeeper.escheduler.lock.failover.startup.masters"; + /** * need send warn times when master server or worker server failover */ diff --git a/escheduler-common/src/main/resources/zookeeper.properties b/escheduler-common/src/main/resources/zookeeper.properties index 7df397ca90..207bc8230c 100644 --- a/escheduler-common/src/main/resources/zookeeper.properties +++ b/escheduler-common/src/main/resources/zookeeper.properties @@ -16,6 +16,7 @@ zookeeper.escheduler.lock.workers=/escheduler/lock/workers #escheduler failover directory zookeeper.escheduler.lock.failover.masters=/escheduler/lock/failover/masters zookeeper.escheduler.lock.failover.workers=/escheduler/lock/failover/workers +zookeeper.escheduler.lock.failover.startup.masters=/escheduler/lock/failover/startup-masters #escheduler failover directory zookeeper.session.timeout=300 diff --git a/escheduler-dao/pom.xml b/escheduler-dao/pom.xml index ca489b2107..39536fb04e 100644 --- a/escheduler-dao/pom.xml +++ b/escheduler-dao/pom.xml @@ -125,6 +125,12 @@ cn.analysys escheduler-common + + + protobuf-java + com.google.protobuf + + org.springframework diff --git a/escheduler-server/pom.xml b/escheduler-server/pom.xml index 3047296e09..00164f5233 100644 --- a/escheduler-server/pom.xml +++ b/escheduler-server/pom.xml @@ -18,6 +18,10 @@ cn.analysys escheduler-common + + protobuf-java + com.google.protobuf + io.netty netty diff --git a/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java b/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java index fe3360484c..e5ef726f9c 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java @@ -31,6 +31,7 @@ import cn.escheduler.dao.model.TaskInstance; import cn.escheduler.server.ResInfo; import cn.escheduler.server.utils.ProcessUtils; import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.imps.CuratorFrameworkState; import org.apache.curator.framework.recipes.cache.PathChildrenCache; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; @@ -111,21 +112,46 @@ public class ZKMasterClient extends AbstractZKClient { // init dao this.initDao(); - // init system znode - this.initSystemZNode(); + InterProcessMutex mutex = null; + try { + // create distributed lock with the root node path of the lock space as /escheduler/lock/failover/master + String znodeLock = getMasterStartUpLockPath(); + + mutex = new InterProcessMutex(zkClient, znodeLock); + mutex.acquire(); + + // init system znode + this.initSystemZNode(); + + // monitor master + this.listenerMaster(); - // monitor master - this.listenerMaster(); + // monitor worker + this.listenerWorker(); - // monitor worker - this.listenerWorker(); + // register master + this.registMaster(); - // register master - this.registMaster(); + // check if fault tolerance is required，failure and tolerance + if (getActiveMasterNum() == 1) { + processDao.selfFaultTolerant(ExecutionStatus.RUNNING_EXEUTION.ordinal(),ExecutionStatus.NEED_FAULT_TOLERANCE.ordinal()); + } + + }catch (Exception e){ + logger.error("master start up exception : " + e.getMessage(),e); + }finally { + if (mutex != null){ + try { + mutex.release(); + } catch (Exception e) { + if(e.getMessage().equals("instance must be started before calling this method")){ + logger.warn("lock release"); + }else{ + logger.error("lock release failed : " + e.getMessage(),e); + } - // check if fault tolerance is required，failure and tolerance - if (getActiveMasterNum() == 1) { - processDao.selfFaultTolerant(ExecutionStatus.RUNNING_EXEUTION.ordinal(),ExecutionStatus.NEED_FAULT_TOLERANCE.ordinal()); + } + } } } @@ -417,6 +443,14 @@ public class ZKMasterClient extends AbstractZKClient { return conf.getString(Constants.ZOOKEEPER_ESCHEDULER_LOCK_MASTERS); } + /** + * get master start up lock path + * @return + */ + public String getMasterStartUpLockPath(){ + return conf.getString(Constants.ZOOKEEPER_ESCHEDULER_LOCK_FAILOVER_STARTUP_MASTERS); + } + /** * get master failover lock path * @return diff --git a/escheduler-ui/.env b/escheduler-ui/.env index fc1b619f69..d83fa8435d 100644 --- a/escheduler-ui/.env +++ b/escheduler-ui/.env @@ -1,6 +1,5 @@ - -# 后端接口地址11 +# 后端接口地址 API_BASE = http://192.168.xx.xx:12345 # 本地开发如需ip访问项目把"#"号去掉 -#DEV_HOST = 192.168.xx.xx +#DEV_HOST = 192.168.xx.xx \ No newline at end of file diff --git a/escheduler-ui/install-escheduler-ui.sh b/escheduler-ui/install-escheduler-ui.sh index c6266b98b2..be9a3801ac 100755 --- a/escheduler-ui/install-escheduler-ui.sh +++ b/escheduler-ui/install-escheduler-ui.sh @@ -1,66 +1,17 @@ #!/bin/bash - # 当前路径 esc_basepath=$(cd `dirname $0`; pwd) - -echo "欢迎使用easy scheduler前端部署脚本,目前前端部署脚本仅支持Centos" -echo "请在 escheduler-ui 目录下执行" - -# 配置前端访问端口 -esc_proxy="8888" - -# 配置代理后端接口 -esc_proxy_port="http://192.168.xx.xx:12345" - -# 本机ip -esc_ipaddr='127.0.0.1' - -esc_ipaddr=$(ip addr | awk '/^[0-9]+: / {}; /inet.*global/ {print gensub(/(.*)\/(.*)/, "\\1", "g", $2)}') - - -#To be compatible with MacOS and Linux -if [[ "$OSTYPE" == "darwin"* ]]; then - # Mac OSX - echo "Easy Scheduler ui install not support Mac OSX operating system" - exit 1 -elif [[ "$OSTYPE" == "linux-gnu" ]]; then - # linux - echo "linux" -elif [[ "$OSTYPE" == "cygwin" ]]; then - # POSIX compatibility layer and Linux environment emulation for Windows - echo "Easy Scheduler ui not support Windows operating system" - exit 1 -elif [[ "$OSTYPE" == "msys" ]]; then - # Lightweight shell and GNU utilities compiled for Windows (part of MinGW) - echo "Easy Scheduler ui not support Windows operating system" - exit 1 -elif [[ "$OSTYPE" == "win32" ]]; then - echo "Easy Scheduler ui not support Windows operating system" - exit 1 -elif [[ "$OSTYPE" == "freebsd"* ]]; then - # ... - echo "freebsd" -else - # Unknown. - echo "Operating system unknown, please tell us(submit issue) for better service" - exit 1 -fi - -# 区分版本 -version=`cat /etc/redhat-release|sed -r 's/.* ([0-9]+)\..*/\1/'` - - -echo "========================================================================配置信息=======================================================================" - -echo "前端访问端口：${esc_proxy}" -echo "后端代理接口地址：${esc_proxy_port}" -echo "静态文件地址：${esc_basepath}/dist" -echo "当前路径：${esc_basepath}" -echo "本机ip：${esc_ipaddr}" - -echo "========================================================================配置信息=======================================================================" -echo "" +menu(){ + cat <> /etc/yum.repos.d/nginx.repo + + rpm -ivh http://nginx.org/packages/centos/6/noarch/RPMS/nginx-release-centos-6-0.el6.ngx.noarch.rpm # install nginx yum install nginx -y # 配置nginx - eschedulerConf - - # 防火墙 - E_iptables=`lsof -i:$esc_proxy | wc -l` - if [ "$E_iptables" -gt "0" ];then - # 已开启端口防火墙重启 - service iptables restart - else - # 未开启防火墙添加端口再重启 - iptables -I INPUT 5 -i eth0 -p tcp --dport $esc_proxy -m state --state NEW,ESTABLISHED -j ACCEPT - service iptables save - service iptables restart - fi + eschedulerConf $1 $2 - # start + # 启动nginx /etc/init.d/nginx start + sleep 1 + if [ $? -ne 0 ];then + /etc/init.d/nginx start + fi + nginx -s reload # 调整SELinux的参数 sed -i "s/SELINUX=enforcing/SELINUX=disabled/g" /etc/selinux/config @@ -178,16 +133,87 @@ centos6(){ } -# centos 6 -if [[ $version -eq 6 ]]; then - centos6 -fi - -# centos 7 -if [[ $version -eq 7 ]]; then - centos7 -fi +function main(){ + echo "欢迎使用easy scheduler前端部署脚本,目前前端部署脚本仅支持CentOS,Ubuntu" + echo "请在 escheduler-ui 目录下执行" + + #To be compatible with MacOS and Linux + if [[ "$OSTYPE" == "darwin"* ]]; then + # Mac OSX + echo "Easy Scheduler ui install not support Mac OSX operating system" + exit 1 + elif [[ "$OSTYPE" == "linux-gnu" ]]; then + # linux + echo "linux" + elif [[ "$OSTYPE" == "cygwin" ]]; then + # POSIX compatibility layer and Linux environment emulation for Windows + echo "Easy Scheduler ui not support Windows operating system" + exit 1 + elif [[ "$OSTYPE" == "msys" ]]; then + # Lightweight shell and GNU utilities compiled for Windows (part of MinGW) + echo "Easy Scheduler ui not support Windows operating system" + exit 1 + elif [[ "$OSTYPE" == "win32" ]]; then + echo "Easy Scheduler ui not support Windows operating system" + exit 1 + elif [[ "$OSTYPE" == "freebsd"* ]]; then + # ... + echo "freebsd" + else + # Unknown. + echo "Operating system unknown, please tell us(submit issue) for better service" + exit 1 + fi + + + # 配置前端访问端口 + read -p "请输入nginx代理端口，不输入，则默认8888 :" esc_proxy_port + if [ -z "${esc_proxy_port}" ];then + esc_proxy_port="8888" + fi + + read -p "请输入api server代理ip,必须输入，例如：192.168.xx.xx :" esc_api_server_ip + if [ -z "${esc_api_server_ip}" ];then + echo "api server代理ip不能为空." + exit 1 + fi + + read -p "请输入api server代理端口,不输入，则默认12345 :" esc_api_server_port + if [ -z "${esc_api_server_port}" ];then + esc_api_server_port="12345" + fi + + # api server后端地址 + esc_api_server="http://$esc_api_server_ip:$esc_api_server_port" + + # 本机ip地址 + esc_ipaddr=$(ip a | grep inet | grep -v inet6 | grep -v 127 | sed 's/^[ \t]*//g' | cut -d ' ' -f2 | head -n 1 | awk -F '/' '{print $1}') + + # 提示信息 + menu + + read -p "请输入安装编号(1|2|3|4)：" num + + case $num in + 1) + centos6 ${esc_proxy_port} ${esc_api_server} + ;; + 2) + centos7 ${esc_proxy_port} ${esc_api_server} + ;; + 3) + ubuntu ${esc_proxy_port} ${esc_api_server} + ;; + 4) + echo $"Usage :sh $0" + exit 1 + ;; + *) + echo $"Usage :sh $0" + exit 1 + esac + echo "请浏览器访问：http://${esc_ipaddr}:${esc_proxy_port}" +} -echo "请浏览器访问：http://${esc_ipaddr}:${esc_proxy}" - +main diff --git a/escheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss b/escheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss index dcfa6b7d5d..95592e0754 100644 --- a/escheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss +++ b/escheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss @@ -138,11 +138,27 @@ .assist-btn { position: absolute; left: 10px; - top: 8px; + top: 7px; >.name { padding-left: 6px; vertical-align: middle; } + >.copy-name { + cursor: pointer; + padding-left: 4px; + position: relative; + top: -2px; + &:hover { + i { + color: #47c3ff; + } + } + i { + color: #333; + font-size: 18px; + vertical-align: middle; + } + } } .save-btn { position: absolute; diff --git a/escheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue b/escheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue index dd573d0d56..37c4119676 100644 --- a/escheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue +++ b/escheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue @@ -29,6 +29,8 @@ icon="fa fa-code"> {{name}} +   +
@@ -88,6 +90,7 @@ import mUdp from './udp/udp' import i18n from '@/module/i18n' import { jsPlumb } from 'jsplumb' + import Clipboard from 'clipboard' import { allNodesId } from './plugIn/util' import { toolOper, tasksType } from './config' import mFormModel from './formModel/formModel' @@ -139,6 +142,23 @@ Dag.create() } }, + /** + * copy name + */ + _copyName(){ + let clipboard = new Clipboard(`.copy-name`) + clipboard.on('success', e => { + this.$message.success(`${i18n.$t('Copy success')}`) + // Free memory + clipboard.destroy() + }) + clipboard.on('error', e => { + // Copy is not supported + this.$message.warning(`${i18n.$t('The browser does not support automatic copying')}`) + // Free memory + clipboard.destroy() + }) + }, /** * Get state interface * @param isReset Whether to manually refresh @@ -154,6 +174,10 @@ let $item = _.filter(taskList, v => v.name === item.name)[0] return `
${i18n.$t('Name')}：${$item.name}
${i18n.$t('State')}：${desc}
${i18n.$t('type')}：${$item.taskType}
${i18n.$t('host')}：${$item.host || '-'}
${i18n.$t('Retry Count')}：${$item.retryTimes}
${i18n.$t('Submit Time')}：${formatDate($item.submitTime)}
${i18n.$t('Start Time')}：${formatDate($item.startTime)}
${i18n.$t('End Time')}：${$item.endTime ? formatDate($item.endTime) : '-'}
` } + + // remove tip state dom + $('.w').find('.state-p').html('') + data.forEach(v1 => { idArr.forEach(v2 => { if (v2.name === v1.name) { @@ -161,7 +185,6 @@ let state = dom.find('.state-p') dom.attr('data-state-id', v1.stateId) dom.attr('data-dependent-result', v1.dependentResult || '') - state.html('') state.append(`${v1.icoUnicode}`) state.find('b').attr('title', titleTpl(v2, v1.desc)) } diff --git a/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sql.vue b/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sql.vue index 0ff75b91d8..6be4b5c2d6 100644 --- a/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sql.vue +++ b/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sql.vue @@ -28,6 +28,17 @@