diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index c57d13b54d..0000000000 --- a/Dockerfile +++ /dev/null @@ -1,41 +0,0 @@ -#Maintin by jimmy -#Email: zhengge2012@gmail.com -FROM anapsix/alpine-java:8_jdk -WORKDIR /tmp -RUN wget http://archive.apache.org/dist/maven/maven-3/3.6.1/binaries/apache-maven-3.6.1-bin.tar.gz -RUN tar -zxvf apache-maven-3.6.1-bin.tar.gz && rm apache-maven-3.6.1-bin.tar.gz -RUN mv apache-maven-3.6.1 /usr/lib/mvn -RUN chown -R root:root /usr/lib/mvn -RUN ln -s /usr/lib/mvn/bin/mvn /usr/bin/mvn -RUN wget https://archive.apache.org/dist/zookeeper/zookeeper-3.4.6/zookeeper-3.4.6.tar.gz -RUN tar -zxvf zookeeper-3.4.6.tar.gz -RUN mv zookeeper-3.4.6 /opt/zookeeper -RUN rm -rf zookeeper-3.4.6.tar.gz -RUN echo "export ZOOKEEPER_HOME=/opt/zookeeper" >>/etc/profile -RUN echo "export PATH=$PATH:$ZOOKEEPER_HOME/bin" >>/etc/profile -ADD conf/zoo.cfg /opt/zookeeper/conf/zoo.cfg -#RUN source /etc/profile -#RUN zkServer.sh start -RUN apk add --no-cache git npm nginx mariadb mariadb-client mariadb-server-utils pwgen -WORKDIR /opt -RUN git clone https://github.com/analysys/EasyScheduler.git -WORKDIR /opt/EasyScheduler -RUN mvn -U clean package assembly:assembly -Dmaven.test.skip=true -RUN mv /opt/EasyScheduler/target/escheduler-1.0.0-SNAPSHOT /opt/easyscheduler -WORKDIR /opt/EasyScheduler/escheduler-ui -RUN npm install -RUN npm audit fix -RUN npm run build -RUN mkdir -p /opt/escheduler/front/server -RUN cp -rfv dist/* /opt/escheduler/front/server -WORKDIR / -RUN rm -rf /opt/EasyScheduler -#configure mysql server https://github.com/yobasystems/alpine-mariadb/tree/master/alpine-mariadb-amd64 -ADD conf/run.sh /scripts/run.sh -RUN mkdir /docker-entrypoint-initdb.d && \ - mkdir /scripts/pre-exec.d && \ - mkdir /scripts/pre-init.d && \ - chmod -R 755 /scripts -RUN rm -rf /var/cache/apk/* -EXPOSE 8888 -ENTRYPOINT ["/scripts/run.sh"] diff --git a/conf/escheduler.conf b/conf/escheduler.conf deleted file mode 100644 index fa331e62f5..0000000000 --- a/conf/escheduler.conf +++ /dev/null @@ -1,31 +0,0 @@ -server { - listen 8888;# 访问端口 - server_name localhost; - #charset koi8-r; - #access_log /var/log/nginx/host.access.log main; - location / { - root /opt/escheduler/front/server; # 静态文件目录 - index index.html index.html; - } - location /escheduler { - proxy_pass http://127.0.0.1:12345; # 接口地址 - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header x_real_ipP $remote_addr; - proxy_set_header remote_addr $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_http_version 1.1; - proxy_connect_timeout 4s; - proxy_read_timeout 30s; - proxy_send_timeout 12s; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - } - #error_page 404 /404.html; - # redirect server error pages to the static page /50x.html - # - error_page 500 502 503 504 /50x.html; - location = /50x.html { - root /usr/share/nginx/html; - } -} diff --git a/conf/install.sh b/conf/install.sh deleted file mode 100644 index 10419b8f84..0000000000 --- a/conf/install.sh +++ /dev/null @@ -1,310 +0,0 @@ -#!/bin/sh - -workDir=`/opt/easyscheduler` -workDir=`cd ${workDir};pwd` - -#To be compatible with MacOS and Linux -txt="" -if [[ "$OSTYPE" == "darwin"* ]]; then - # Mac OSX - txt="''" -elif [[ "$OSTYPE" == "linux-gnu" ]]; then - # linux - txt="" -elif [[ "$OSTYPE" == "cygwin" ]]; then - # POSIX compatibility layer and Linux environment emulation for Windows - echo "Easy Scheduler not support Windows operating system" - exit 1 -elif [[ "$OSTYPE" == "msys" ]]; then - # Lightweight shell and GNU utilities compiled for Windows (part of MinGW) - echo "Easy Scheduler not support Windows operating system" - exit 1 -elif [[ "$OSTYPE" == "win32" ]]; then - echo "Easy Scheduler not support Windows operating system" - exit 1 -elif [[ "$OSTYPE" == "freebsd"* ]]; then - # ... - txt="" -else - # Unknown. - echo "Operating system unknown, please tell us(submit issue) for better service" - exit 1 -fi - -source ${workDir}/conf/config/run_config.conf -source ${workDir}/conf/config/install_config.conf - -# mysql配置 -# mysql 地址,端口 -mysqlHost="127.0.0.1:3306" - -# mysql 数据库名称 -mysqlDb="easyscheduler" - -# mysql 用户名 -mysqlUserName="easyscheduler" - -# mysql 密码 -mysqlPassword="easyschedulereasyscheduler" - -# conf/config/install_config.conf配置 -# 安装路径,不要当前路径(pwd)一样 -installPath="/opt/easyscheduler" - -# 部署用户 -deployUser="escheduler" - -# zk集群 -zkQuorum="192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181" - -# 安装hosts -ips="ark0,ark1,ark2,ark3,ark4" - -# conf/config/run_config.conf配置 -# 运行Master的机器 -masters="ark0,ark1" - -# 运行Worker的机器 -workers="ark2,ark3,ark4" - -# 运行Alert的机器 -alertServer="ark3" - -# 运行Api的机器 -apiServers="ark1" - -# alert配置 -# 邮件协议 -mailProtocol="SMTP" - -# 邮件服务host -mailServerHost="smtp.exmail.qq.com" - -# 邮件服务端口 -mailServerPort="25" - -# 发送人 -mailSender="xxxxxxxxxx" - -# 发送人密码 -mailPassword="xxxxxxxxxx" - -# 下载Excel路径 -xlsFilePath="/tmp/xls" - - -# hadoop 配置 -# 是否启动hdfs,如果启动则为true,需要配置以下hadoop相关参数; -# 不启动设置为false,如果为false,以下配置不需要修改 -hdfsStartupSate="false" - -# namenode地址,支持HA,需要将core-site.xml和hdfs-site.xml放到conf目录下 -namenodeFs="hdfs://mycluster:8020" - -# resourcemanager HA配置，如果是单resourcemanager,这里为空即可 -yarnHaIps="192.168.xx.xx,192.168.xx.xx" - -# 如果是单 resourcemanager,只需要配置一个主机名称,如果是resourcemanager HA,则默认配置就好 -singleYarnIp="ark1" - -# hdfs根路径,根路径的owner必须是部署用户 -hdfsPath="/escheduler" - -# common 配置 -# 程序路径 -programPath="/tmp/escheduler" - -#下载路径 -downloadPath="/tmp/escheduler/download" - -# 任务执行路径 -execPath="/tmp/escheduler/exec" - -# SHELL环境变量路径 -shellEnvPath="$installPath/conf/env/.escheduler_env.sh" - -# Python换将变量路径 -pythonEnvPath="$installPath/conf/env/escheduler_env.py" - -# 资源文件的后缀 -resSuffixs="txt,log,sh,conf,cfg,py,java,sql,hql,xml" - -# 开发状态,如果是true,对于SHELL脚本可以在execPath目录下查看封装后的SHELL脚本,如果是false则执行完成直接删除 -devState="true" - -# zk 配置 -# zk根目录 -zkRoot="/escheduler" - -# 用来记录挂掉机器的zk目录 -zkDeadServers="/escheduler/dead-servers" - -# masters目录 -zkMasters="/escheduler/masters" - -# workers目录 -zkWorkers="/escheduler/workers" - -# zk master分布式锁 -mastersLock="/escheduler/lock/masters" - -# zk worker分布式锁 -workersLock="/escheduler/lock/workers" - -# zk master容错分布式锁 -mastersFailover="/escheduler/lock/failover/masters" - -# zk worker容错分布式锁 -workersFailover="/escheduler/lock/failover/masters" - -# zk session 超时 -zkSessionTimeout="300" - -# zk 连接超时 -zkConnectionTimeout="300" - -# zk 重试间隔 -zkRetrySleep="100" - -# zk重试最大次数 -zkRetryMaxtime="5" - - -# master 配置 -# master执行线程最大数,流程实例的最大并行度 -masterExecThreads="100" - -# master任务执行线程最大数,每一个流程实例的最大并行度 -masterExecTaskNum="20" - -# master心跳间隔 -masterHeartbeatInterval="10" - -# master任务提交重试次数 -masterTaskCommitRetryTimes="5" - -# master任务提交重试时间间隔 -masterTaskCommitInterval="100" - -# master最大cpu平均负载,用来判断master是否还有执行能力 -masterMaxCupLoadAvg="10" - -# master预留内存,用来判断master是否还有执行能力 -masterReservedMemory="1" - - -# worker 配置 -# worker执行线程 -workerExecThreads="100" - -# worker心跳间隔 -workerHeartbeatInterval="10" - -# worker一次抓取任务数 -workerFetchTaskNum="10" - -# worker最大cpu平均负载,用来判断master是否还有执行能力 -workerMaxCupLoadAvg="10" - -# worker预留内存,用来判断master是否还有执行能力 -workerReservedMemory="1" - -# api 配置 -# api 服务端口 -apiServerPort="12345" - -# api session 超时 -apiServerSessionTimeout="7200" - -# api 上下文路径 -apiServerContextPath="/escheduler/" - -# spring 最大文件大小 -springMaxFileSize="1024MB" - -# spring 最大请求文件大小 -springMaxRequestSize="1024MB" - -# api 最大post请求大小 -apiMaxHttpPostSize="5000000" - -# 1,替换文件 -echo "1,替换文件" -sed -i ${txt} "s#spring.datasource.url.*#spring.datasource.url=jdbc:mysql://${mysqlHost}/${mysqlDb}?characterEncoding=UTF-8#g" conf/dao/data_source.properties -sed -i ${txt} "s#spring.datasource.username.*#spring.datasource.username=${mysqlUserName}#g" conf/dao/data_source.properties -sed -i ${txt} "s#spring.datasource.password.*#spring.datasource.password=${mysqlPassword}#g" conf/dao/data_source.properties - -sed -i ${txt} "s#org.quartz.dataSource.myDs.URL.*#org.quartz.dataSource.myDs.URL=jdbc:mysql://${mysqlHost}/${mysqlDb}?characterEncoding=UTF-8#g" conf/quartz.properties -sed -i ${txt} "s#org.quartz.dataSource.myDs.user.*#org.quartz.dataSource.myDs.user=${mysqlUserName}#g" conf/quartz.properties -sed -i ${txt} "s#org.quartz.dataSource.myDs.password.*#org.quartz.dataSource.myDs.password=${mysqlPassword}#g" conf/quartz.properties - - -sed -i ${txt} "s#fs.defaultFS.*#fs.defaultFS=${namenodeFs}#g" conf/common/hadoop/hadoop.properties -sed -i ${txt} "s#yarn.resourcemanager.ha.rm.ids.*#yarn.resourcemanager.ha.rm.ids=${yarnHaIps}#g" conf/common/hadoop/hadoop.properties -sed -i ${txt} "s#yarn.application.status.address.*#yarn.application.status.address=http://${singleYarnIp}:8088/ws/v1/cluster/apps/%s#g" conf/common/hadoop/hadoop.properties - -sed -i ${txt} "s#data.basedir.path.*#data.basedir.path=${programPath}#g" conf/common/common.properties -sed -i ${txt} "s#data.download.basedir.path.*#data.download.basedir.path=${downloadPath}#g" conf/common/common.properties -sed -i ${txt} "s#process.exec.basepath.*#process.exec.basepath=${execPath}#g" conf/common/common.properties -sed -i ${txt} "s#data.store2hdfs.basepath.*#data.store2hdfs.basepath=${hdfsPath}#g" conf/common/common.properties -sed -i ${txt} "s#hdfs.startup.state.*#hdfs.startup.state=${hdfsStartupSate}#g" conf/common/common.properties -sed -i ${txt} "s#escheduler.env.path.*#escheduler.env.path=${shellEnvPath}#g" conf/common/common.properties -sed -i ${txt} "s#escheduler.env.py.*#escheduler.env.py=${pythonEnvPath}#g" conf/common/common.properties -sed -i ${txt} "s#resource.view.suffixs.*#resource.view.suffixs=${resSuffixs}#g" conf/common/common.properties -sed -i ${txt} "s#development.state.*#development.state=${devState}#g" conf/common/common.properties - -sed -i ${txt} "s#zookeeper.quorum.*#zookeeper.quorum=${zkQuorum}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.escheduler.root.*#zookeeper.escheduler.root=${zkRoot}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.escheduler.dead.servers.*#zookeeper.escheduler.dead.servers=${zkDeadServers}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.escheduler.masters.*#zookeeper.escheduler.masters=${zkMasters}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.escheduler.workers.*#zookeeper.escheduler.workers=${zkWorkers}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.escheduler.lock.masters.*#zookeeper.escheduler.lock.masters=${mastersLock}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.escheduler.lock.workers.*#zookeeper.escheduler.lock.workers=${workersLock}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.escheduler.lock.failover.masters.*#zookeeper.escheduler.lock.failover.masters=${mastersFailover}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.escheduler.lock.failover.workers.*#zookeeper.escheduler.lock.failover.workers=${workersFailover}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.session.timeout.*#zookeeper.session.timeout=${zkSessionTimeout}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.connection.timeout.*#zookeeper.connection.timeout=${zkConnectionTimeout}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.retry.sleep.*#zookeeper.retry.sleep=${zkRetrySleep}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.retry.maxtime.*#zookeeper.retry.maxtime=${zkRetryMaxtime}#g" conf/zookeeper.properties - -sed -i ${txt} "s#master.exec.threads.*#master.exec.threads=${masterExecThreads}#g" conf/master.properties -sed -i ${txt} "s#master.exec.task.number.*#master.exec.task.number=${masterExecTaskNum}#g" conf/master.properties -sed -i ${txt} "s#master.heartbeat.interval.*#master.heartbeat.interval=${masterHeartbeatInterval}#g" conf/master.properties -sed -i ${txt} "s#master.task.commit.retryTimes.*#master.task.commit.retryTimes=${masterTaskCommitRetryTimes}#g" conf/master.properties -sed -i ${txt} "s#master.task.commit.interval.*#master.task.commit.interval=${masterTaskCommitInterval}#g" conf/master.properties -sed -i ${txt} "s#master.max.cpuload.avg.*#master.max.cpuload.avg=${masterMaxCupLoadAvg}#g" conf/master.properties -sed -i ${txt} "s#master.reserved.memory.*#master.reserved.memory=${masterReservedMemory}#g" conf/master.properties - - -sed -i ${txt} "s#worker.exec.threads.*#worker.exec.threads=${workerExecThreads}#g" conf/worker.properties -sed -i ${txt} "s#worker.heartbeat.interval.*#worker.heartbeat.interval=${workerHeartbeatInterval}#g" conf/worker.properties -sed -i ${txt} "s#worker.fetch.task.num.*#worker.fetch.task.num=${workerFetchTaskNum}#g" conf/worker.properties -sed -i ${txt} "s#worker.max.cpuload.avg.*#worker.max.cpuload.avg=${workerMaxCupLoadAvg}#g" conf/worker.properties -sed -i ${txt} "s#worker.reserved.memory.*#worker.reserved.memory=${workerReservedMemory}#g" conf/worker.properties - - -sed -i ${txt} "s#server.port.*#server.port=${apiServerPort}#g" conf/application.properties -sed -i ${txt} "s#server.session.timeout.*#server.session.timeout=${apiServerSessionTimeout}#g" conf/application.properties -sed -i ${txt} "s#server.context-path.*#server.context-path=${apiServerContextPath}#g" conf/application.properties -sed -i ${txt} "s#spring.http.multipart.max-file-size.*#spring.http.multipart.max-file-size=${springMaxFileSize}#g" conf/application.properties -sed -i ${txt} "s#spring.http.multipart.max-request-size.*#spring.http.multipart.max-request-size=${springMaxRequestSize}#g" conf/application.properties -sed -i ${txt} "s#server.max-http-post-size.*#server.max-http-post-size=${apiMaxHttpPostSize}#g" conf/application.properties - - -sed -i ${txt} "s#mail.protocol.*#mail.protocol=${mailProtocol}#g" conf/alert.properties -sed -i ${txt} "s#mail.server.host.*#mail.server.host=${mailServerHost}#g" conf/alert.properties -sed -i ${txt} "s#mail.server.port.*#mail.server.port=${mailServerPort}#g" conf/alert.properties -sed -i ${txt} "s#mail.sender.*#mail.sender=${mailSender}#g" conf/alert.properties -sed -i ${txt} "s#mail.passwd.*#mail.passwd=${mailPassword}#g" conf/alert.properties -sed -i ${txt} "s#xls.file.path.*#xls.file.path=${xlsFilePath}#g" conf/alert.properties - - -sed -i ${txt} "s#installPath.*#installPath=${installPath}#g" conf/config/install_config.conf -sed -i ${txt} "s#deployUser.*#deployUser=${deployUser}#g" conf/config/install_config.conf -sed -i ${txt} "s#ips.*#ips=${ips}#g" conf/config/install_config.conf - - -sed -i ${txt} "s#masters.*#masters=${masters}#g" conf/config/run_config.conf -sed -i ${txt} "s#workers.*#workers=${workers}#g" conf/config/run_config.conf -sed -i ${txt} "s#alertServer.*#alertServer=${alertServer}#g" conf/config/run_config.conf -sed -i ${txt} "s#apiServers.*#apiServers=${apiServers}#g" conf/config/run_config.conf diff --git a/conf/run.sh b/conf/run.sh deleted file mode 100644 index 3257b0e194..0000000000 --- a/conf/run.sh +++ /dev/null @@ -1,105 +0,0 @@ -#!/bin/sh - -# execute any pre-init scripts -for i in /scripts/pre-init.d/*sh -do - if [ -e "${i}" ]; then - echo "[i] pre-init.d - processing $i" - . "${i}" - fi -done - -if [ -d "/run/mysqld" ]; then - echo "[i] mysqld already present, skipping creation" - chown -R mysql:mysql /run/mysqld -else - echo "[i] mysqld not found, creating...." - mkdir -p /run/mysqld - chown -R mysql:mysql /run/mysqld -fi - -if [ -d /var/lib/mysql/mysql ]; then - echo "[i] MySQL directory already present, skipping creation" - chown -R mysql:mysql /var/lib/mysql -else - echo "[i] MySQL data directory not found, creating initial DBs" - - chown -R mysql:mysql /var/lib/mysql - - mysql_install_db --user=mysql --ldata=/var/lib/mysql > /dev/null - - if [ "$MYSQL_ROOT_PASSWORD" = "" ]; then - MYSQL_ROOT_PASSWORD=`pwgen 16 1` - echo "[i] MySQL root Password: $MYSQL_ROOT_PASSWORD" - fi - - MYSQL_DATABASE="easyscheduler" - MYSQL_USER="easyscheduler" - MYSQL_PASSWORD="easyschedulereasyscheduler" - - tfile=`mktemp` - if [ ! -f "$tfile" ]; then - return 1 - fi - - cat << EOF > $tfile -USE mysql; -FLUSH PRIVILEGES ; -GRANT ALL ON *.* TO 'root'@'%' identified by '$MYSQL_ROOT_PASSWORD' WITH GRANT OPTION ; -GRANT ALL ON *.* TO 'root'@'localhost' identified by '$MYSQL_ROOT_PASSWORD' WITH GRANT OPTION ; -SET PASSWORD FOR 'root'@'localhost'=PASSWORD('${MYSQL_ROOT_PASSWORD}') ; -DROP DATABASE IF EXISTS test ; -FLUSH PRIVILEGES ; -EOF - - if [ "$MYSQL_DATABASE" != "" ]; then - echo "[i] Creating database: $MYSQL_DATABASE" - echo "CREATE DATABASE IF NOT EXISTS \`$MYSQL_DATABASE\` CHARACTER SET utf8 COLLATE utf8_general_ci;" >> $tfile - - if [ "$MYSQL_USER" != "" ]; then - echo "[i] Creating user: $MYSQL_USER with password $MYSQL_PASSWORD" - echo "GRANT ALL ON \`$MYSQL_DATABASE\`.* to '$MYSQL_USER'@'%' IDENTIFIED BY '$MYSQL_PASSWORD';" >> $tfile - fi - fi - - /usr/bin/mysqld --user=mysql --bootstrap --verbose=0 --skip-name-resolve --skip-networking=0 < $tfile - rm -f $tfile - - for f in /docker-entrypoint-initdb.d/*; do - case "$f" in - *.sql) echo "$0: running $f"; /usr/bin/mysqld --user=mysql --bootstrap --verbose=0 --skip-name-resolve --skip-networking=0 < "$f"; echo ;; - *.sql.gz) echo "$0: running $f"; gunzip -c "$f" | /usr/bin/mysqld --user=mysql --bootstrap --verbose=0 --skip-name-resolve --skip-networking=0 < "$f"; echo ;; - *) echo "$0: ignoring or entrypoint initdb empty $f" ;; - esac - echo - done - - echo - echo 'MySQL init process done. Ready for start up.' - echo - - echo "exec /usr/bin/mysqld --user=mysql --console --skip-name-resolve --skip-networking=0" "$@" -fi - -# execute any pre-exec scripts -for i in /scripts/pre-exec.d/*sh -do - if [ -e "${i}" ]; then - echo "[i] pre-exec.d - processing $i" - . ${i} - fi -done - -mysql -ueasyscheduler -peasyschedulereasyscheduler --one-database easyscheduler -h127.0.0.1 < /opt/easyscheduler/sql/escheduler.sql -mysql -ueasyscheduler -peasyschedulereasyscheduler --one-database easyscheduler -h127.0.0.1 < /opt/easyscheduler/sql/quartz.sql -source /etc/profile -zkServer.sh start -cd /opt/easyscheduler -rm -rf /etc/nginx/conf.d/default.conf -sh ./bin/escheduler-daemon.sh start master-server -sh ./bin/escheduler-daemon.sh start worker-server -sh ./bin/escheduler-daemon.sh start api-server -sh ./bin/escheduler-daemon.sh start logger-server -sh ./bin/escheduler-daemon.sh start alert-server -nginx -c /etc/nginx/nginx.conf -exec /usr/bin/mysqld --user=mysql --console --skip-name-resolve --skip-networking=0 $@ diff --git a/conf/zoo.cfg b/conf/zoo.cfg deleted file mode 100644 index 228b7224a0..0000000000 --- a/conf/zoo.cfg +++ /dev/null @@ -1,30 +0,0 @@ -# The number of milliseconds of each tick -tickTime=2000 -# The number of ticks that the initial -# synchronization phase can take -initLimit=10 -# The number of ticks that can pass between -# sending a request and getting an acknowledgement -syncLimit=5 -# the directory where the snapshot is stored. -# do not use /tmp for storage, /tmp here is just -# example sakes. -dataDir=/tmp/zookeeper -# the port at which the clients will connect -clientPort=2181 -# the maximum number of client connections. -# increase this if you need to handle more clients -#maxClientCnxns=60 -# -# Be sure to read the maintenance section of the -# administrator guide before turning on autopurge. -# -# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance -# -# The number of snapshots to retain in dataDir -#autopurge.snapRetainCount=3 -# Purge task interval in hours -# Set to "0" to disable auto purge feature -#autopurge.purgeInterval=1 -dataDir=/opt/zookeeper/data -dataLogDir=/opt/zookeeper/logs diff --git a/escheduler-alert/src/main/resources/alert.properties b/escheduler-alert/src/main/resources/alert.properties index e2cba1160d..2fc2bb70f9 100644 --- a/escheduler-alert/src/main/resources/alert.properties +++ b/escheduler-alert/src/main/resources/alert.properties @@ -9,7 +9,7 @@ mail.sender=xxxxxxx mail.passwd=xxxxxxx #xls file path,need create if not exist -xls.file.path=/opt/xls +xls.file.path=/tmp/xls diff --git a/escheduler-api/pom.xml b/escheduler-api/pom.xml index bfa01dba1e..73a3927749 100644 --- a/escheduler-api/pom.xml +++ b/escheduler-api/pom.xml @@ -34,6 +34,10 @@ leveldbjni-all org.fusesource.leveldbjni + + protobuf-java + com.google.protobuf + diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/ExecutorService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/ExecutorService.java index 640ffeb5f8..740fbc961c 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/ExecutorService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/ExecutorService.java @@ -178,9 +178,11 @@ public class ExecutorService extends BaseService{ } ProcessDefinition processDefinition = processDao.findProcessDefineById(processInstance.getProcessDefinitionId()); - result = checkProcessDefinitionValid(processDefinition, processInstance.getProcessDefinitionId()); - if (result.get(Constants.STATUS) != Status.SUCCESS) { - return result; + if(executeType != ExecuteType.STOP && executeType != ExecuteType.PAUSE){ + result = checkProcessDefinitionValid(processDefinition, processInstance.getProcessDefinitionId()); + if (result.get(Constants.STATUS) != Status.SUCCESS) { + return result; + } } checkResult = checkExecuteType(processInstance, executeType); diff --git a/escheduler-common/pom.xml b/escheduler-common/pom.xml index 4f76b55cac..afe34dbd3b 100644 --- a/escheduler-common/pom.xml +++ b/escheduler-common/pom.xml @@ -148,10 +148,10 @@ javax.servlet.jsp jsp-api - - com.google.protobuf - protobuf-java - + + + + @@ -175,10 +175,10 @@ org.codehaus.jackson jackson-xc - - com.google.protobuf - protobuf-java - + + + + org.fusesource.leveldbjni leveldbjni-all diff --git a/escheduler-common/src/main/java/cn/escheduler/common/Constants.java b/escheduler-common/src/main/java/cn/escheduler/common/Constants.java index abf29fdc99..b12145973e 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/Constants.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/Constants.java @@ -162,6 +162,11 @@ public final class Constants { */ public static final String ZOOKEEPER_ESCHEDULER_LOCK_FAILOVER_WORKERS = "zookeeper.escheduler.lock.failover.workers"; + /** + * MasterServer startup failover runing and fault tolerance process + */ + public static final String ZOOKEEPER_ESCHEDULER_LOCK_FAILOVER_STARTUP_MASTERS = "zookeeper.escheduler.lock.failover.startup.masters"; + /** * need send warn times when master server or worker server failover */ diff --git a/escheduler-common/src/main/resources/zookeeper.properties b/escheduler-common/src/main/resources/zookeeper.properties index 7df397ca90..207bc8230c 100644 --- a/escheduler-common/src/main/resources/zookeeper.properties +++ b/escheduler-common/src/main/resources/zookeeper.properties @@ -16,6 +16,7 @@ zookeeper.escheduler.lock.workers=/escheduler/lock/workers #escheduler failover directory zookeeper.escheduler.lock.failover.masters=/escheduler/lock/failover/masters zookeeper.escheduler.lock.failover.workers=/escheduler/lock/failover/workers +zookeeper.escheduler.lock.failover.startup.masters=/escheduler/lock/failover/startup-masters #escheduler failover directory zookeeper.session.timeout=300 diff --git a/escheduler-dao/pom.xml b/escheduler-dao/pom.xml index ca489b2107..39536fb04e 100644 --- a/escheduler-dao/pom.xml +++ b/escheduler-dao/pom.xml @@ -125,6 +125,12 @@ cn.analysys escheduler-common + + + protobuf-java + com.google.protobuf + + org.springframework diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java b/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java index 0eb65dbe14..f8182d692c 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java @@ -588,10 +588,12 @@ public class ProcessDao extends AbstractBaseDao { case START_FAILURE_TASK_PROCESS: // find failed tasks and init these tasks List failedList = this.findTaskIdByInstanceState(processInstance.getId(), ExecutionStatus.FAILURE); + List toleranceList = this.findTaskIdByInstanceState(processInstance.getId(), ExecutionStatus.NEED_FAULT_TOLERANCE); List killedList = this.findTaskIdByInstanceState(processInstance.getId(), ExecutionStatus.KILL); cmdParam.remove(Constants.CMDPARAM_RECOVERY_START_NODE_STRING); failedList.addAll(killedList); + failedList.addAll(toleranceList); for(Integer taskId : failedList){ initTaskInstance(this.findTaskInstanceById(taskId)); } diff --git a/escheduler-server/pom.xml b/escheduler-server/pom.xml index 3047296e09..00164f5233 100644 --- a/escheduler-server/pom.xml +++ b/escheduler-server/pom.xml @@ -18,6 +18,10 @@ cn.analysys escheduler-common + + protobuf-java + com.google.protobuf + io.netty netty diff --git a/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java b/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java index fe3360484c..e5ef726f9c 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java @@ -31,6 +31,7 @@ import cn.escheduler.dao.model.TaskInstance; import cn.escheduler.server.ResInfo; import cn.escheduler.server.utils.ProcessUtils; import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.imps.CuratorFrameworkState; import org.apache.curator.framework.recipes.cache.PathChildrenCache; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; @@ -111,21 +112,46 @@ public class ZKMasterClient extends AbstractZKClient { // init dao this.initDao(); - // init system znode - this.initSystemZNode(); + InterProcessMutex mutex = null; + try { + // create distributed lock with the root node path of the lock space as /escheduler/lock/failover/master + String znodeLock = getMasterStartUpLockPath(); + + mutex = new InterProcessMutex(zkClient, znodeLock); + mutex.acquire(); + + // init system znode + this.initSystemZNode(); + + // monitor master + this.listenerMaster(); - // monitor master - this.listenerMaster(); + // monitor worker + this.listenerWorker(); - // monitor worker - this.listenerWorker(); + // register master + this.registMaster(); - // register master - this.registMaster(); + // check if fault tolerance is required，failure and tolerance + if (getActiveMasterNum() == 1) { + processDao.selfFaultTolerant(ExecutionStatus.RUNNING_EXEUTION.ordinal(),ExecutionStatus.NEED_FAULT_TOLERANCE.ordinal()); + } + + }catch (Exception e){ + logger.error("master start up exception : " + e.getMessage(),e); + }finally { + if (mutex != null){ + try { + mutex.release(); + } catch (Exception e) { + if(e.getMessage().equals("instance must be started before calling this method")){ + logger.warn("lock release"); + }else{ + logger.error("lock release failed : " + e.getMessage(),e); + } - // check if fault tolerance is required，failure and tolerance - if (getActiveMasterNum() == 1) { - processDao.selfFaultTolerant(ExecutionStatus.RUNNING_EXEUTION.ordinal(),ExecutionStatus.NEED_FAULT_TOLERANCE.ordinal()); + } + } } } @@ -417,6 +443,14 @@ public class ZKMasterClient extends AbstractZKClient { return conf.getString(Constants.ZOOKEEPER_ESCHEDULER_LOCK_MASTERS); } + /** + * get master start up lock path + * @return + */ + public String getMasterStartUpLockPath(){ + return conf.getString(Constants.ZOOKEEPER_ESCHEDULER_LOCK_FAILOVER_STARTUP_MASTERS); + } + /** * get master failover lock path * @return diff --git a/escheduler-ui/.env b/escheduler-ui/.env index 2905723bd3..d83fa8435d 100644 --- a/escheduler-ui/.env +++ b/escheduler-ui/.env @@ -1,9 +1,5 @@ - -# 后端接口地址 -#API_BASE = http://192.168.221.188:12345 - # 后端接口地址 -API_BASE = http://192.168.220.154:12345 +API_BASE = http://192.168.xx.xx:12345 # 本地开发如需ip访问项目把"#"号去掉 -#DEV_HOST = 192.168.xx.xx +#DEV_HOST = 192.168.xx.xx \ No newline at end of file diff --git a/escheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss b/escheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss index bcff822a2e..95592e0754 100644 --- a/escheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss +++ b/escheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss @@ -138,7 +138,7 @@ .assist-btn { position: absolute; left: 10px; - top: 4px; + top: 7px; >.name { padding-left: 6px; vertical-align: middle; @@ -146,6 +146,8 @@ >.copy-name { cursor: pointer; padding-left: 4px; + position: relative; + top: -2px; &:hover { i { color: #47c3ff; @@ -154,6 +156,7 @@ i { color: #333; font-size: 18px; + vertical-align: middle; } } } diff --git a/escheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue b/escheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue index c60f5d3b49..37c4119676 100644 --- a/escheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue +++ b/escheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue @@ -29,7 +29,8 @@ icon="fa fa-code"> {{name}} - +   +
diff --git a/escheduler-ui/src/js/conf/home/pages/projects/pages/index/index.vue b/escheduler-ui/src/js/conf/home/pages/projects/pages/index/index.vue index 4d5d923ac2..28f61f9e65 100644 --- a/escheduler-ui/src/js/conf/home/pages/projects/pages/index/index.vue +++ b/escheduler-ui/src/js/conf/home/pages/projects/pages/index/index.vue @@ -1,5 +1,5 @@