Browse Source

Merge remote-tracking branch 'upstream/dev-20190415' into dev-20190415

pull/2/head
gongzijian 6 years ago
parent
commit
b7fd46f3e4
  1. 111
      .gitignore
  2. BIN
      docs/zh_CN/Easyscheduler文档.pdf
  3. 14
      docs/zh_CN/SUMMARY.md
  4. 70
      docs/zh_CN/前端开发文档.md
  5. 128
      docs/zh_CN/前端部署文档.md
  6. 48
      docs/zh_CN/后端开发文档.md
  7. 156
      docs/zh_CN/后端部署文档.md
  8. 5
      escheduler-api/src/main/java/cn/escheduler/api/controller/ExecutorController.java
  9. 129
      escheduler-api/src/main/java/cn/escheduler/api/controller/MonitorController.java
  10. 8
      escheduler-api/src/main/java/cn/escheduler/api/enums/Status.java
  11. 19
      escheduler-api/src/main/java/cn/escheduler/api/service/ExecutorService.java
  12. 72
      escheduler-api/src/main/java/cn/escheduler/api/service/MonitorService.java
  13. 6
      escheduler-api/src/main/java/cn/escheduler/api/service/ServerService.java
  14. 211
      escheduler-api/src/main/java/cn/escheduler/api/utils/ZooKeeperState.java
  15. 72
      escheduler-api/src/main/java/cn/escheduler/api/utils/ZookeeperMonitorUtils.java
  16. 20
      escheduler-api/src/test/java/cn/escheduler/api/controller/ExecutorControllerTest.java
  17. 105
      escheduler-api/src/test/java/cn/escheduler/api/controller/MonitorControllerTest.java
  18. 2
      escheduler-common/src/main/java/cn/escheduler/common/Constants.java
  19. 27
      escheduler-common/src/main/java/cn/escheduler/common/task/sql/SqlParameters.java
  20. 29
      escheduler-common/src/main/java/cn/escheduler/common/zk/AbstractZKClient.java
  21. 54
      escheduler-dao/readme.txt
  22. 145
      escheduler-dao/src/main/java/cn/escheduler/dao/MonitorDBDao.java
  23. 2
      escheduler-dao/src/main/java/cn/escheduler/dao/TaskRecordDao.java
  24. 88
      escheduler-dao/src/main/java/cn/escheduler/dao/mapper/MonitorMapper.java
  25. 115
      escheduler-dao/src/main/java/cn/escheduler/dao/model/MonitorRecord.java
  26. 217
      escheduler-dao/src/main/java/cn/escheduler/dao/model/ZookeeperRecord.java
  27. 22
      escheduler-server/src/main/java/cn/escheduler/server/ResInfo.java
  28. 7
      escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java
  29. 47
      escheduler-ui/install(线上环境).sh
  30. 164
      sql/upgrade/1.0.2_schema/mysql/escheduler_ddl.sql

111
.gitignore vendored

@ -33,3 +33,114 @@ yarn.lock
package-lock.json
config.gypi
test/coverage
/docs/zh_CN/介绍
/docs/zh_CN/贡献代码.md
/escheduler-common/src/main/resources/zookeeper.properties
escheduler-alert/logs/
escheduler-alert/src/main/resources/alert.properties_bak
escheduler-alert/src/main/resources/logback.xml
escheduler-server/src/main/resources/logback.xml
escheduler-ui/dist/css/common.16ac5d9.css
escheduler-ui/dist/css/home/index.b444b91.css
escheduler-ui/dist/css/login/index.5866c64.css
escheduler-ui/dist/js/0.ac94e5d.js
escheduler-ui/dist/js/0.ac94e5d.js.map
escheduler-ui/dist/js/1.0b043a3.js
escheduler-ui/dist/js/1.0b043a3.js.map
escheduler-ui/dist/js/10.1bce3dc.js
escheduler-ui/dist/js/10.1bce3dc.js.map
escheduler-ui/dist/js/11.79f04d8.js
escheduler-ui/dist/js/11.79f04d8.js.map
escheduler-ui/dist/js/12.420daa5.js
escheduler-ui/dist/js/12.420daa5.js.map
escheduler-ui/dist/js/13.e5bae1c.js
escheduler-ui/dist/js/13.e5bae1c.js.map
escheduler-ui/dist/js/14.f2a0dca.js
escheduler-ui/dist/js/14.f2a0dca.js.map
escheduler-ui/dist/js/15.45373e8.js
escheduler-ui/dist/js/15.45373e8.js.map
escheduler-ui/dist/js/16.fecb0fc.js
escheduler-ui/dist/js/16.fecb0fc.js.map
escheduler-ui/dist/js/17.84be279.js
escheduler-ui/dist/js/17.84be279.js.map
escheduler-ui/dist/js/18.307ea70.js
escheduler-ui/dist/js/18.307ea70.js.map
escheduler-ui/dist/js/19.144db9c.js
escheduler-ui/dist/js/19.144db9c.js.map
escheduler-ui/dist/js/2.8b4ef29.js
escheduler-ui/dist/js/2.8b4ef29.js.map
escheduler-ui/dist/js/20.4c527e9.js
escheduler-ui/dist/js/20.4c527e9.js.map
escheduler-ui/dist/js/21.831b2a2.js
escheduler-ui/dist/js/21.831b2a2.js.map
escheduler-ui/dist/js/22.2b4bb2a.js
escheduler-ui/dist/js/22.2b4bb2a.js.map
escheduler-ui/dist/js/23.81467ef.js
escheduler-ui/dist/js/23.81467ef.js.map
escheduler-ui/dist/js/24.54a00e4.js
escheduler-ui/dist/js/24.54a00e4.js.map
escheduler-ui/dist/js/25.8d7bd36.js
escheduler-ui/dist/js/25.8d7bd36.js.map
escheduler-ui/dist/js/26.2ec5e78.js
escheduler-ui/dist/js/26.2ec5e78.js.map
escheduler-ui/dist/js/27.3ab48c2.js
escheduler-ui/dist/js/27.3ab48c2.js.map
escheduler-ui/dist/js/28.363088a.js
escheduler-ui/dist/js/28.363088a.js.map
escheduler-ui/dist/js/29.6c5853a.js
escheduler-ui/dist/js/29.6c5853a.js.map
escheduler-ui/dist/js/3.a0edb5b.js
escheduler-ui/dist/js/3.a0edb5b.js.map
escheduler-ui/dist/js/30.940fdd3.js
escheduler-ui/dist/js/30.940fdd3.js.map
escheduler-ui/dist/js/31.168a460.js
escheduler-ui/dist/js/31.168a460.js.map
escheduler-ui/dist/js/32.8df6594.js
escheduler-ui/dist/js/32.8df6594.js.map
escheduler-ui/dist/js/33.4480bbe.js
escheduler-ui/dist/js/33.4480bbe.js.map
escheduler-ui/dist/js/34.b407fe1.js
escheduler-ui/dist/js/34.b407fe1.js.map
escheduler-ui/dist/js/35.f340b0a.js
escheduler-ui/dist/js/35.f340b0a.js.map
escheduler-ui/dist/js/36.8880c2d.js
escheduler-ui/dist/js/36.8880c2d.js.map
escheduler-ui/dist/js/37.ea2a25d.js
escheduler-ui/dist/js/37.ea2a25d.js.map
escheduler-ui/dist/js/38.98a59ee.js
escheduler-ui/dist/js/38.98a59ee.js.map
escheduler-ui/dist/js/39.a5e958a.js
escheduler-ui/dist/js/39.a5e958a.js.map
escheduler-ui/dist/js/4.4ca44db.js
escheduler-ui/dist/js/4.4ca44db.js.map
escheduler-ui/dist/js/40.e187b1e.js
escheduler-ui/dist/js/40.e187b1e.js.map
escheduler-ui/dist/js/41.0e89182.js
escheduler-ui/dist/js/41.0e89182.js.map
escheduler-ui/dist/js/42.341047c.js
escheduler-ui/dist/js/42.341047c.js.map
escheduler-ui/dist/js/43.27b8228.js
escheduler-ui/dist/js/43.27b8228.js.map
escheduler-ui/dist/js/44.e8869bc.js
escheduler-ui/dist/js/44.e8869bc.js.map
escheduler-ui/dist/js/45.8d54901.js
escheduler-ui/dist/js/45.8d54901.js.map
escheduler-ui/dist/js/5.e1ed7f3.js
escheduler-ui/dist/js/5.e1ed7f3.js.map
escheduler-ui/dist/js/6.241ba07.js
escheduler-ui/dist/js/6.241ba07.js.map
escheduler-ui/dist/js/7.ab2e297.js
escheduler-ui/dist/js/7.ab2e297.js.map
escheduler-ui/dist/js/8.83ff814.js
escheduler-ui/dist/js/8.83ff814.js.map
escheduler-ui/dist/js/9.39cb29f.js
escheduler-ui/dist/js/9.39cb29f.js.map
escheduler-ui/dist/js/common.733e342.js
escheduler-ui/dist/js/common.733e342.js.map
escheduler-ui/dist/js/home/index.78a5d12.js
escheduler-ui/dist/js/home/index.78a5d12.js.map
escheduler-ui/dist/js/login/index.291b8e3.js
escheduler-ui/dist/js/login/index.291b8e3.js.map
escheduler-ui/dist/lib/external/
escheduler-ui/src/js/conf/home/pages/projects/pages/taskInstance/index.vue
/escheduler-dao/src/main/resources/dao/data_source.properties

BIN
docs/zh_CN/Easyscheduler文档.pdf

Binary file not shown.

14
docs/zh_CN/SUMMARY.md

@ -2,17 +2,16 @@
* [Easyscheduler简介](README.md)
* 前端部署文档
* [环境搭建](前端部署文档.md#前端项目环境构建及编译)
* [安装及配置](前端部署文档.md#安装及配置)
* [项目生产环境Nginx配置](前端部署文档.md#项目生产环境配置)
* [前端项目发布](前端部署文档.md#前端项目发布)
* [问题](前端部署文档.md#问题)
* [准备工作](前端部署文档.md#1、准备工作)
* [部署](前端部署文档.md#2、部署)
* [常见问题](前端部署文档.md#前端常见问题)
* 后端部署文档
* [后端部署文档](后端部署文档.md#部署文档)
* [准备工作](后端部署文档.md#1、准备工作)
* [部署](后端部署文档.md#2、部署)
* [系统使用手册](系统使用手册.md#使用手册)
* [系统架构设计](系统架构设计.md#系统架构设计)
* 前端开发文档
* [开发环境搭建](前端部署文档.md)
* [开发环境搭建](前端开发文档.md#开发环境搭建)
* [项目目录结构](前端开发文档.md#项目目录结构)
* [系统功能模块](前端开发文档.md#系统功能模块)
* [路由和状态管理](前端开发文档.md#路由和状态管理)
@ -20,6 +19,7 @@
* [接口](前端开发文档.md#接口)
* [扩展开发](前端开发文档.md#扩展开发)
* 后端开发文档
* [开发环境搭建](后端开发文档.md#项目编译)
* [自定义任务插件文档](任务插件开发.md#任务插件开发)

70
docs/zh_CN/前端开发文档.md

@ -15,6 +15,75 @@ Jsplumb 连线插件库
Lodash 高性能的 JavaScript 实用工具库
```
### 开发环境搭建
- #### Node安装
Node包下载 (注意版本 8.9.4) `https://nodejs.org/download/release/v8.9.4/`
- #### 前端项目构建
用命令行模式 `cd` 进入 `escheduler-ui`项目目录并执行 `npm install` 拉取项目依赖包
> 如果 `npm install` 速度非常慢
> 可以转淘宝镜像命令行输入 `npm install -g cnpm --registry=https://registry.npm.taobao.org`
> 运行 `cnpm install`
> ##### !!!这里特别注意 项目如果在拉取依赖包的过程中报 " node-sass error " 错误,请在执行完后再次执行以下命令
```
npm install node-sass --unsafe-perm //单独安装node-sass依赖
```
- #### 开发环境运行
- `npm start` 项目开发环境 (启动后访问地址 http://localhost:8888/#/)
#### 前端项目发布
- `npm run build` 项目打包 (打包后根目录会创建一个名为dist文件夹,用于发布线上Nginx)
运行 `npm run build` 命令,生成打包文件(dist)包
再拷贝到服务器对应的目录下(前端服务静态页面存放目录)
访问地址 `http://localhost:8888/#/`
#### Liunx下使用node启动并且守护进程
安装pm2 `npm install -g pm2`
在项目`escheduler-ui`根目录执行 `pm2 start npm -- run dev` 启动项目
#### 命令
- 启用 `pm2 start npm -- run dev`
- 停止 `pm2 stop npm`
- 删除 `pm2 delete npm`
- 状态 `pm2 list`
```
[root@localhost escheduler-ui]# pm2 start npm -- run dev
[PM2] Applying action restartProcessId on app [npm](ids: 0)
[PM2] [npm](0) ✓
[PM2] Process successfully started
┌──────────┬────┬─────────┬──────┬──────┬────────┬─────────┬────────┬─────┬──────────┬──────┬──────────┐
│ App name │ id │ version │ mode │ pid │ status │ restart │ uptime │ cpu │ mem │ user │ watching │
├──────────┼────┼─────────┼──────┼──────┼────────┼─────────┼────────┼─────┼──────────┼──────┼──────────┤
│ npm │ 0 │ N/A │ fork │ 6168 │ online │ 31 │ 0s │ 0% │ 5.6 MB │ root │ disabled │
└──────────┴────┴─────────┴──────┴──────┴────────┴─────────┴────────┴─────┴──────────┴──────┴──────────┘
Use `pm2 show <id|name>` to get more details about an app
```
### 项目目录结构
`build` 打包及开发环境项目的一些webpack配置
@ -64,6 +133,7 @@ Lodash 高性能的 JavaScript 实用工具库
`util` => 工具
### 系统功能模块
首页 => `http://localhost:8888/#/home`

128
docs/zh_CN/前端部署文档.md

@ -1,53 +1,31 @@
# 前端部署文档
- ##### 1. 开发环境搭建
前端有3种部署方式,分别为自动化部署,手动部署和编译源码部署
- ##### 2. 自动化部署
## 1、准备工作
#### 准备一:下载安装包
- ##### 3. 手动部署
目前最新安装包版本是1.0.1,下载地址: [码云下载](https://gitee.com/easyscheduler/EasyScheduler/attach_files/)
- ##### 4. Liunx下使用node启动并且守护进程
下载escheduler-ui-1.0.1.tar.gz后,解压后会产生dist目录,进入dist目录
> cd dist
#### 准备二:新建一个`.env`文件
### 1.开发环境搭建
- #### node安装
Node包下载 (注意版本 8.9.4) `https://nodejs.org/download/release/v8.9.4/`
- #### 前端项目构建
用命令行模式 `cd` 进入 `escheduler-ui`项目目录并执行 `npm install` 拉取项目依赖包
> 如果 `npm install` 速度非常慢
> 可以转淘宝镜像命令行输入 `npm install -g cnpm --registry=https://registry.npm.taobao.org`
> 运行 `cnpm install`
> ##### !!!这里特别注意 项目如果在拉取依赖包的过程中报 " node-sass error " 错误,请在执行完后再次执行以下命令
```
npm install node-sass --unsafe-perm //单独安装node-sass依赖
在dist目录下新建一个`.env`文件,在文件里添加后端服务的ip地址和端口,用于跟后端交互,`.env`文件内容如下:
```
在项目`escheduler-ui`根目录`.env`添加代理的端口地址
```
# 前端代理的接口地址(例)
API_BASE = http://192.168.220.204:12345
# 代理的接口地址(自行修改)
API_BASE = http://192.168.xx.xx:12345
# 如果您需要用ip访问项目可以把 "#" 号去掉(例)
#DEV_HOST = 192.168.6.132
#DEV_HOST = 192.168.xx.xx
```
运行
- `npm start` 项目开发环境 (启动后访问地址 http://localhost:8888/#/)
- `npm run build` 项目打包 (打包后根目录会创建一个名为dist文件夹,用于发布线上Nginx)
## 2、部署
以下两种方式任选其一部署即可,推荐自动化部署
### 2.1 自动化部署
### 2.自动化部署
在项目`escheduler-ui`根目录编辑安装文件`vi install(线上环境).sh`
在前端项目根目录dist下编辑安装文件`vi install(线上环境).sh`(执行时,最好修改install(线上环境).sh为install-ui.sh,跟后端部署区分)
更改前端访问端口和后端代理接口地址
@ -56,40 +34,26 @@ API_BASE = http://192.168.220.204:12345
esc_proxy="8888"
# 配置代理后端接口
esc_proxy_port="http://192.168.220.154:12345"
esc_proxy_port="http://192.168.xx.xx:12345"
```
前端自动部署基于`yum`操作,部署之前请先安装更新`yum
在项目`escheduler-ui`根目录执行`./install(线上环境).sh`
前端自动部署基于linux系统`yum`操作,部署之前请先安装更新`yum`
在前端项目根目录dist下执行`./install(线上环境).sh` 或者改名后的 `./install-ui.sh`
### 3.手动部署
### 2.2 手动部署
安装epel源 `yum install epel-release -y`
安装Nginx `yum install nginx -y`
#### 命令
- 启用 `systemctl enable nginx`
- 重启 `systemctl restart nginx`
- 状态 `systemctl status nginx`
> #### 创建静态页面存放目录
> #### nginx配置文件地址
```
mkdir /data2_4T/escheduler_front/escheduler/server
/etc/nginx/conf.d/default.conf
```
> #### 配置文件地址
```
/etc/nginx/conf.d/escheduler.conf
```
> #### 配置信息
> #### 配置信息(自行修改)
```
server {
listen 8888;# 访问端口
@ -97,11 +61,11 @@ server {
#charset koi8-r;
#access_log /var/log/nginx/host.access.log main;
location / {
root /data2_4T/escheduler_front/escheduler/server; # 静态文件目录
root /xx/dist; # 上面前端解压的dist目录地址(自行修改)
index index.html index.html;
}
location /escheduler {
proxy_pass http://192.168.220.181:12345; # 接口地址
proxy_pass http://192.168.xx.xx:12345; # 接口地址(自行修改)
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header x_real_ipP $remote_addr;
@ -128,50 +92,16 @@ server {
systemctl restart nginx
```
#### nginx命令
#### 前端项目发布
前端在开发环境(dev)中运行 `npm run build` 命令,生成打包文件(dist)包
再拷贝到服务器 `/data2_4T/escheduler_front/escheduler/server`(服务器静态页面存放目录)
访问地址 `http://localhost:8888/#/`
### 4.Liunx下使用node启动并且守护进程
安装pm2 `npm install -g pm2`
在项目`escheduler-ui`根目录执行 `pm2 start npm -- run dev` 启动项目
#### 命令
- 启用 `pm2 start npm -- run dev`
- 停止 `pm2 stop npm`
- 删除 `pm2 delete npm`
- 状态 `pm2 list`
```
- 启用 `systemctl enable nginx`
[root@localhost escheduler-ui]# pm2 start npm -- run dev
[PM2] Applying action restartProcessId on app [npm](ids: 0)
[PM2] [npm](0) ✓
[PM2] Process successfully started
┌──────────┬────┬─────────┬──────┬──────┬────────┬─────────┬────────┬─────┬──────────┬──────┬──────────┐
│ App name │ id │ version │ mode │ pid │ status │ restart │ uptime │ cpu │ mem │ user │ watching │
├──────────┼────┼─────────┼──────┼──────┼────────┼─────────┼────────┼─────┼──────────┼──────┼──────────┤
│ npm │ 0 │ N/A │ fork │ 6168 │ online │ 31 │ 0s │ 0% │ 5.6 MB │ root │ disabled │
└──────────┴────┴─────────┴──────┴──────┴────────┴─────────┴────────┴─────┴──────────┴──────┴──────────┘
Use `pm2 show <id|name>` to get more details about an app
- 重启 `systemctl restart nginx`
```
- 状态 `systemctl status nginx`
## 问题
## 前端常见问题
#### 1. 上传文件大小限制
编辑配置文件 `vi /etc/nginx/nginx.conf`
```

48
docs/zh_CN/后端开发文档.md

@ -0,0 +1,48 @@
# 后端开发文档
## 环境要求
* [Mysql](http://geek.analysys.cn/topic/124) (5.5+) : 必装
* [JDK](https://www.oracle.com/technetwork/java/javase/downloads/index.html) (1.8+) : 必装
* [ZooKeeper](https://mirrors.tuna.tsinghua.edu.cn/apache/zookeeper)(3.4.6+) :必装
* [Maven](http://maven.apache.org/download.cgi)(3.3+) :必装
因EasyScheduler中escheduler-rpc模块使用到Grpc,需要用到Maven编译生成所需要的类
对Maven不熟的伙伴请参考: [maven in five minutes](http://maven.apache.org/guides/getting-started/maven-in-five-minutes.html)(3.3+)
http://maven.apache.org/install.html
## 项目编译
将EasyScheduler源码下载导入Idea等开发工具后,首先转为Maven项目(右键点击后选择"Add Framework Support")
* 执行编译命令:
```
mvn -U clean package assembly:assembly -Dmaven.test.skip=true
```
* 查看目录
正常编译完后,会在当前目录生成 target/escheduler-{version}/
```
bin
conf
lib
script
sql
install.sh
```
- 说明
```
bin : 基础服务启动脚本
conf : 项目配置文件
lib : 项目依赖jar包,包括各个模块jar和第三方jar
script : 集群启动、停止和服务监控启停脚本
sql : 项目依赖sql文件
install.sh : 一键部署脚本
```

156
docs/zh_CN/后端部署文档.md

@ -1,11 +1,16 @@
# 后端部署文档
后端有2种部署方式,分别为自动化部署和编译源码部署
## 基础软件安装
## 1、准备工作
目前最新安装包版本是1.0.1,下载地址: [码云下载](https://gitee.com/easyscheduler/EasyScheduler/attach_files/) ,下载escheduler-backend-1.0.1.tar.gz后,解压后会产生escheduler-backend-1.0.1目录(后面简称escheduler-backend)
#### 准备一: 基础软件安装(必装项请自行安装)
* [Mysql](http://geek.analysys.cn/topic/124) (5.5+) : 必装
* [JDK](https://www.oracle.com/technetwork/java/javase/downloads/index.html) (1.8+) : 必装
* [ZooKeeper](https://www.jianshu.com/p/de90172ea680)(3.4.6) :必装
* [ZooKeeper](https://www.jianshu.com/p/de90172ea680)(3.4.6+) :必装
* [Hadoop](https://blog.csdn.net/Evankaka/article/details/51612437)(2.6+) :选装, 如果需要使用到资源上传功能,MapReduce任务提交则需要配置Hadoop(上传的资源文件目前保存在Hdfs上)
* [Hive](https://staroon.pro/2017/12/09/HiveInstall/)(1.2.1) : 选装,hive任务提交需要安装
* Spark(1.x,2.x) : 选装,Spark任务提交需要安装
@ -15,103 +20,82 @@
注意:EasyScheduler本身不依赖Hadoop、Hive、Spark、PostgreSQL,仅是会调用他们的Client,用于对应任务的运行。
```
## 项目编译
* 执行编译命令:
```
mvn -U clean package assembly:assembly -Dmaven.test.skip=true
```
* 查看目录
正常编译完后,会在当前目录生成 target/escheduler-{version}/
- 说明
```
bin : 基础服务启动脚本
conf : 项目配置文件
lib : 项目依赖jar包,包括各个模块jar和第三方jar
script : 集群启动、停止和服务监控启停脚本
sql : 项目依赖sql文件
install.sh : 一键部署脚本
```
## 数据库初始化
* 创建database和账号
```
mysql -h {host} -u {user} -p{password}
mysql> CREATE DATABASE escheduler DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
mysql> GRANT ALL PRIVILEGES ON escheduler.* TO '{user}'@'%' IDENTIFIED BY '{password}';
mysql> GRANT ALL PRIVILEGES ON escheduler.* TO '{user}'@'localhost' IDENTIFIED BY '{password}';
mysql> flush privileges;
```
* 创建表和导入基础数据
```
说明:在 target/escheduler-{version}/sql/escheduler.sql和quartz.sql
mysql -h {host} -u {user} -p{password} -D {db} < escheduler.sql
mysql -h {host} -u {user} -p{password} -D {db} < quartz.sql
```
## 创建部署用户
#### 准备二: 创建部署用户
- 在所有需要部署调度的机器上创建部署用户,因为worker服务是以 sudo -u {linux-user} 方式来执行作业,所以部署用户需要有 sudo 权限,而且是免密的。
```部署账号
vi /etc/sudoers
# 部署用户是 escheduler 账号
# 例如部署用户是escheduler账号
escheduler ALL=(ALL) NOPASSWD: NOPASSWD: ALL
# 并且需要注释掉 Default requiretty 一行
#Default requiretty
```
## ssh免密配置
#### 准备三: ssh免密配置
在部署机器和其他安装机器上配置ssh免密登录,如果要在部署机上安装调度,需要配置本机免密登录自己
- [将 **主机器** 和各个其它机器SSH打通](http://geek.analysys.cn/topic/113)
## 部署
### 1. 修改安装目录权限
#### 准备四:数据库初始化
- 安装目录如下:
* 创建database和账号
```
bin
conf
install.sh
lib
script
sql
通过以下mysql命令进入mysql命令行服务:
> mysql -h {host} -u {user} -p{password}
然后执行以下命令创建database和账号
```
- 修改权限(deployUser修改为对应部署用户)
```sql
CREATE DATABASE escheduler DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL PRIVILEGES ON escheduler.* TO '{user}'@'%' IDENTIFIED BY '{password}';
GRANT ALL PRIVILEGES ON escheduler.* TO '{user}'@'localhost' IDENTIFIED BY '{password}';
flush privileges;
```
`sudo chown -R deployUser:deployUser *`
* 创建表和导入基础数据
说明:在escheduler-backend/sql/escheduler.sql和quartz.sql
### 2. 修改环境变量文件
```sql
mysql -h {host} -u {user} -p{password} -D {db} < escheduler.sql
mysql -h {host} -u {user} -p{password} -D {db} < quartz.sql
```
- 根据业务需求,修改conf/env/目录下的**escheduler_env.py**,**.escheduler_env.sh**两个文件中的环境变量
#### 准备五: 修改部署目录权限及运行参数
### 3. 修改部署参数
我们先来大体了解下解压后escheduler-backend目录下的文件(夹)的作用
- 修改 **install.sh**中的参数,替换成自身业务所需的值
```
bin : 基础服务启动脚本
conf : 项目配置文件
lib : 项目依赖jar包,包括各个模块jar和第三方jar
script : 集群启动、停止和服务监控启停脚本
sql : 项目依赖sql文件
install.sh : 一键部署脚本
```
- 修改权限(请将deployUser自行修改为对应部署用户),使得部署用户对escheduler-backend目录有操作权限
`sudo chown -R deployUser:deployUser escheduler-backend`
- 修改conf/env/目录下的`escheduler_env.py`, `.escheduler_env.sh` 两个文件中的环境变量
- 修改部署参数(根据自己服务器及业务情况):
- 修改 **install.sh**中的各参数,替换成自身业务所需的值
- 如果使用hdfs相关功能,需要拷贝**hdfs-site.xml**和**core-site.xml**到conf目录下
### 4. 一键部署
## 2、部署
推荐自动化部署,有经验的小伙伴也可以使用源码部署
### 2.1 自动部署
- 安装zookeeper工具
@ -121,7 +105,7 @@ escheduler ALL=(ALL) NOPASSWD: NOPASSWD: ALL
`sh install.sh`
- jps查看服务是否启动
- 使用jps命令查看服务是否启动(jps为java JDK自带)
```aidl
MasterServer ----- master服务
@ -130,9 +114,10 @@ escheduler ALL=(ALL) NOPASSWD: NOPASSWD: ALL
ApiApplicationServer ----- api服务
AlertServer ----- alert服务
```
如果有以上5个服务,说明自动部署成功
## 日志查看
日志统一存放于指定文件夹内
部署成功后,可以进行日志查看,日志统一存放于指定文件夹内
```日志路径
logs/
@ -142,8 +127,24 @@ escheduler ALL=(ALL) NOPASSWD: NOPASSWD: ALL
|—— escheduler-api-server.log
|—— escheduler-logger-server.log
```
## 启停服务
### 2.2 编译源码来部署
将源码包release版本1.0.1下载后,解压进入根目录
* 执行编译命令:
```
mvn -U clean package assembly:assembly -Dmaven.test.skip=true
```
* 查看目录
正常编译完后,会在当前目录生成 target/escheduler-{version}/ , 然后参考
### 2.3 系统常用启停服务(服务用途请具体参见《系统架构设计》小节)
* 一键停止集群所有服务
@ -184,5 +185,4 @@ sh ./bin/escheduler-daemon.sh stop logger-server
```
sh ./bin/escheduler-daemon.sh start alert-server
sh ./bin/escheduler-daemon.sh stop alert-server
```
```

5
escheduler-api/src/main/java/cn/escheduler/api/controller/ExecutorController.java

@ -149,10 +149,11 @@ public class ExecutorController extends BaseController {
@GetMapping(value = "/get-receiver-cc")
@ResponseStatus(HttpStatus.OK)
public Result getReceiverCc(@RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "processDefinitionId") int processDefinitionId){
@RequestParam(value = "processDefinitionId",required = false) Integer processDefinitionId,
@RequestParam(value = "processInstanceId",required = false) Integer processInstanceId) {
logger.info("login user {}, get process definition receiver and cc", loginUser.getUserName());
try {
Map<String, Object> result = execService.getReceiverCc(processDefinitionId);
Map<String, Object> result = execService.getReceiverCc(processDefinitionId,processInstanceId);
return returnDataList(result);
} catch (Exception e) {
logger.error(QUERY_RECIPIENTS_AND_COPYERS_BY_PROCESS_DEFINITION_ERROR.getMsg(),e);

129
escheduler-api/src/main/java/cn/escheduler/api/controller/MonitorController.java

@ -0,0 +1,129 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package cn.escheduler.api.controller;
import cn.escheduler.api.service.MonitorService;
import cn.escheduler.api.service.ServerService;
import cn.escheduler.api.utils.Constants;
import cn.escheduler.api.utils.Result;
import cn.escheduler.dao.model.User;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.*;
import java.util.Map;
import static cn.escheduler.api.enums.Status.*;
/**
* monitor controller
*/
@RestController
@RequestMapping("/monitor")
public class MonitorController extends BaseController{
private static final Logger logger = LoggerFactory.getLogger(MonitorController.class);
@Autowired
private ServerService serverService;
@Autowired
private MonitorService monitorService;
/**
* master list
* @param loginUser
* @return
*/
@GetMapping(value = "/master/list")
@ResponseStatus(HttpStatus.OK)
public Result listMaster(@RequestAttribute(value = Constants.SESSION_USER) User loginUser) {
logger.info("login user: {}, query all master", loginUser.getUserName());
try{
logger.info("list master, user:{}", loginUser.getUserName());
Map<String, Object> result = serverService.queryMaster(loginUser);
return returnDataList(result);
}catch (Exception e){
logger.error(LIST_MASTERS_ERROR.getMsg(),e);
return error(LIST_MASTERS_ERROR.getCode(),
LIST_MASTERS_ERROR.getMsg());
}
}
/**
* worker list
* @param loginUser
* @return
*/
@GetMapping(value = "/worker/list")
@ResponseStatus(HttpStatus.OK)
public Result listWorker(@RequestAttribute(value = Constants.SESSION_USER) User loginUser) {
logger.info("login user: {}, query all workers", loginUser.getUserName());
try{
Map<String, Object> result = serverService.queryWorker(loginUser);
return returnDataList(result);
}catch (Exception e){
logger.error(LIST_WORKERS_ERROR.getMsg(),e);
return error(LIST_WORKERS_ERROR.getCode(),
LIST_WORKERS_ERROR.getMsg());
}
}
/**
* query database state
* @param loginUser
* @return
*/
@GetMapping(value = "/database")
@ResponseStatus(HttpStatus.OK)
public Result queryDatabaseState(@RequestAttribute(value = Constants.SESSION_USER) User loginUser) {
logger.info("login user: {}, query database state", loginUser.getUserName());
try{
Map<String, Object> result = monitorService.queryDatabaseState(loginUser);
return returnDataList(result);
}catch (Exception e){
logger.error(QUERY_DATABASE_STATE_ERROR.getMsg(),e);
return error(QUERY_DATABASE_STATE_ERROR.getCode(),
QUERY_DATABASE_STATE_ERROR.getMsg());
}
}
/**
* query zookeeper state
* @param loginUser
* @return
*/
@GetMapping(value = "/zookeeper/list")
@ResponseStatus(HttpStatus.OK)
public Result queryZookeeperState(@RequestAttribute(value = Constants.SESSION_USER) User loginUser) {
logger.info("login user: {}, query zookeeper state", loginUser.getUserName());
try{
Map<String, Object> result = monitorService.queryZookeeperState(loginUser);
return returnDataList(result);
}catch (Exception e){
logger.error(QUERY_ZOOKEEPER_STATE_ERROR.getMsg(),e);
return error(QUERY_ZOOKEEPER_STATE_ERROR.getCode(),
QUERY_ZOOKEEPER_STATE_ERROR.getMsg());
}
}
}

8
escheduler-api/src/main/java/cn/escheduler/api/enums/Status.java

@ -201,9 +201,15 @@ public enum Status {
PROCESS_NODE_HAS_CYCLE(50019,"process node has cycle"),
PROCESS_NODE_S_PARAMETER_INVALID(50020,"process node %s parameter invalid"),
HDFS_NOT_STARTUP(60001,"hdfs not startup"),
/**
* for monitor
*/
QUERY_DATABASE_STATE_ERROR(70001,"query database state error"),
QUERY_ZOOKEEPER_STATE_ERROR(70002,"query zookeeper state error"),
CREATE_ACCESS_TOKEN_ERROR(70001,"create access token error"),
GENERATE_TOKEN_ERROR(70002,"generate token error"),

19
escheduler-api/src/main/java/cn/escheduler/api/service/ExecutorService.java

@ -361,18 +361,29 @@ public class ExecutorService extends BaseService{
}
/**
* query recipients and copyers by process definition id
* query recipients and copyers by process definition id or processInstanceId
*
* @param processDefineId
* @return
*/
public Map<String, Object> getReceiverCc(int processDefineId) {
public Map<String, Object> getReceiverCc(Integer processDefineId,Integer processInstanceId) {
Map<String, Object> result = new HashMap<>();
logger.info("processInstanceId {}",processInstanceId);
if(processDefineId == null && processInstanceId == null){
throw new RuntimeException("You must set values for parameters processDefineId or processInstanceId");
}
if(processDefineId == null && processInstanceId != null) {
ProcessInstance processInstance = processInstanceMapper.queryById(processInstanceId);
if (processInstance == null) {
throw new RuntimeException("processInstanceId is not exists");
}
processDefineId = processInstance.getProcessDefinitionId();
}
ProcessDefinition processDefinition = processDefinitionMapper.queryByDefineId(processDefineId);
if (processDefinition == null){
throw new RuntimeException("processDefineId is not exists");
throw new RuntimeException(String.format("processDefineId %d is not exists",processDefineId));
}
String receivers = processDefinition.getReceivers();
String receiversCc = processDefinition.getReceiversCc();
Map<String,String> dataMap = new HashMap<>();

72
escheduler-api/src/main/java/cn/escheduler/api/service/MonitorService.java

@ -0,0 +1,72 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package cn.escheduler.api.service;
import cn.escheduler.api.enums.Status;
import cn.escheduler.api.utils.Constants;
import cn.escheduler.api.utils.ZookeeperMonitorUtils;
import cn.escheduler.dao.MonitorDBDao;
import cn.escheduler.dao.model.MonitorRecord;
import cn.escheduler.dao.model.User;
import cn.escheduler.dao.model.ZookeeperRecord;
import org.springframework.stereotype.Service;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* monitor service
*/
@Service
public class MonitorService extends BaseService{
/**
* query database state
*
* @return
*/
public Map<String,Object> queryDatabaseState(User loginUser) {
Map<String, Object> result = new HashMap<>(5);
List<MonitorRecord> monitorRecordList = MonitorDBDao.queryDatabaseState();
result.put(Constants.DATA_LIST, monitorRecordList);
putMsg(result, Status.SUCCESS);
return result;
}
/**
* query zookeeper state
*
* @return
*/
public Map<String,Object> queryZookeeperState(User loginUser) {
Map<String, Object> result = new HashMap<>(5);
List<ZookeeperRecord> zookeeperRecordList = ZookeeperMonitorUtils.zookeeperInfoList();
result.put(Constants.DATA_LIST, zookeeperRecordList);
putMsg(result, Status.SUCCESS);
return result;
}
}

6
escheduler-api/src/main/java/cn/escheduler/api/service/ServerService.java

@ -52,9 +52,6 @@ public class ServerService extends BaseService{
public Map<String,Object> queryMaster(User loginUser) {
Map<String, Object> result = new HashMap<>(5);
if (checkAdmin(loginUser, result)){
return result;
}
List<MasterServer> masterList = masterServerMapper.queryAllMaster();
result.put(Constants.DATA_LIST, masterList);
@ -71,9 +68,6 @@ public class ServerService extends BaseService{
*/
public Map<String,Object> queryWorker(User loginUser) {
Map<String, Object> result = new HashMap<>();
if (checkAdmin(loginUser, result)){
return result;
}
List<WorkerServer> workerList = workerServerMapper.queryAllWorker();
result.put(Constants.DATA_LIST, workerList);

211
escheduler-api/src/main/java/cn/escheduler/api/utils/ZooKeeperState.java

@ -0,0 +1,211 @@
package cn.escheduler.api.utils;
import org.apache.commons.lang3.StringUtils;
import org.apache.zookeeper.client.FourLetterWordMain;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.Scanner;
/**
* zookeeper状态监控:4字口诀
*
*/
public class ZooKeeperState {
private static final Logger logger = LoggerFactory.getLogger(ZooKeeperState.class);
private final String host;
private final int port;
private int minLatency = -1, avgLatency = -1, maxLatency = -1;
private long received = -1;
private long sent = -1;
private int outStanding = -1;
private long zxid = -1;
private String mode = null;
private int nodeCount = -1;
private int watches = -1;
private int connections = -1;
public ZooKeeperState(String connectionString) {
String host = connectionString.substring(0,
connectionString.indexOf(':'));
int port = Integer.parseInt(connectionString.substring(connectionString
.indexOf(':') + 1));
this.host = host;
this.port = port;
}
public void getZookeeperInfo() {
String content = cmd("srvr");
if (StringUtils.isNotBlank(content)) {
Scanner scannerForStat = new Scanner(content);
while (scannerForStat.hasNext()) {
String line = scannerForStat.nextLine();
if (line.startsWith("Latency min/avg/max:")) {
String[] latencys = getStringValueFromLine(line).split("/");
minLatency = Integer.parseInt(latencys[0]);
avgLatency = Integer.parseInt(latencys[1]);
maxLatency = Integer.parseInt(latencys[2]);
} else if (line.startsWith("Received:")) {
received = Long.parseLong(getStringValueFromLine(line));
} else if (line.startsWith("Sent:")) {
sent = Long.parseLong(getStringValueFromLine(line));
} else if (line.startsWith("Outstanding:")) {
outStanding = Integer.parseInt(getStringValueFromLine(line));
} else if (line.startsWith("Zxid:")) {
zxid = Long.parseLong(getStringValueFromLine(line).substring(2), 16);
} else if (line.startsWith("Mode:")) {
mode = getStringValueFromLine(line);
} else if (line.startsWith("Node count:")) {
nodeCount = Integer.parseInt(getStringValueFromLine(line));
}
}
scannerForStat.close();
}
String wchsText = cmd("wchs");
if (StringUtils.isNotBlank(wchsText)) {
Scanner scannerForWchs = new Scanner(wchsText);
while (scannerForWchs.hasNext()) {
String line = scannerForWchs.nextLine();
if (line.startsWith("Total watches:")) {
watches = Integer.parseInt(getStringValueFromLine(line));
}
}
scannerForWchs.close();
}
String consText = cmd("cons");
if (StringUtils.isNotBlank(consText)) {
Scanner scannerForCons = new Scanner(consText);
if (StringUtils.isNotBlank(consText)) {
connections = 0;
}
while (scannerForCons.hasNext()) {
@SuppressWarnings("unused")
String line = scannerForCons.nextLine();
++connections;
}
scannerForCons.close();
}
}
public boolean ruok() {
return "imok\n".equals(cmd("ruok"));
}
private String getStringValueFromLine(String line) {
return line.substring(line.indexOf(":") + 1, line.length()).replaceAll(
" ", "").trim();
}
private class SendThread extends Thread {
private String cmd;
public String ret = "";
public SendThread(String cmd) {
this.cmd = cmd;
}
@Override
public void run() {
try {
ret = FourLetterWordMain.send4LetterWord(host, port, cmd);
} catch (IOException e) {
logger.error(e.getMessage(),e);
return;
}
}
}
private String cmd(String cmd) {
final int waitTimeout = 5;
SendThread sendThread = new SendThread(cmd);
sendThread.setName("FourLetterCmd:" + cmd);
sendThread.start();
try {
sendThread.join(waitTimeout * 1000);
return sendThread.ret;
} catch (InterruptedException e) {
logger.error("send " + cmd + " to server " + host + ":" + port + " failed!", e);
}
return "";
}
public Logger getLogger() {
return logger;
}
public String getHost() {
return host;
}
public int getPort() {
return port;
}
public int getMinLatency() {
return minLatency;
}
public int getAvgLatency() {
return avgLatency;
}
public int getMaxLatency() {
return maxLatency;
}
public long getReceived() {
return received;
}
public long getSent() {
return sent;
}
public int getOutStanding() {
return outStanding;
}
public long getZxid() {
return zxid;
}
public String getMode() {
return mode;
}
public int getNodeCount() {
return nodeCount;
}
public int getWatches() {
return watches;
}
public int getConnections() {
return connections;
}
@Override
public String toString() {
return "ZooKeeperState [host=" + host + ", port=" + port
+ ", minLatency=" + minLatency + ", avgLatency=" + avgLatency
+ ", maxLatency=" + maxLatency + ", received=" + received
+ ", sent=" + sent + ", outStanding=" + outStanding + ", zxid="
+ zxid + ", mode=" + mode + ", nodeCount=" + nodeCount
+ ", watches=" + watches + ", connections="
+ connections + "]";
}
}

72
escheduler-api/src/main/java/cn/escheduler/api/utils/ZookeeperMonitorUtils.java

@ -0,0 +1,72 @@
package cn.escheduler.api.utils;
import cn.escheduler.common.zk.AbstractZKClient;
import cn.escheduler.dao.model.ZookeeperRecord;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
/**
* monitor zookeeper info
*/
public class ZookeeperMonitorUtils {
private static final Logger LOG = LoggerFactory.getLogger(ZookeeperMonitorUtils.class);
private static final String zookeeperList = AbstractZKClient.getZookeeperQuorum();
/**
*
* @return zookeeper info list
*/
public static List<ZookeeperRecord> zookeeperInfoList(){
String zookeeperServers = zookeeperList.replaceAll("[\\t\\n\\x0B\\f\\r]", "");
try{
return zookeeperInfoList(zookeeperServers);
}catch(Exception e){
LOG.error(e.getMessage(),e);
}
return null;
}
private static List<ZookeeperRecord> zookeeperInfoList(String zookeeperServers) {
List<ZookeeperRecord> list = new ArrayList<>(5);
if(StringUtils.isNotBlank(zookeeperServers)){
String[] zookeeperServersArray = zookeeperServers.split(",");
for (String zookeeperServer : zookeeperServersArray) {
ZooKeeperState state = new ZooKeeperState(zookeeperServer);
boolean ok = state.ruok();
if(ok){
state.getZookeeperInfo();
}
String hostName = zookeeperServer;
int connections = state.getConnections();
int watches = state.getWatches();
long sent = state.getSent();
long received = state.getReceived();
String mode = state.getMode();
int minLatency = state.getMinLatency();
int avgLatency = state.getAvgLatency();
int maxLatency = state.getMaxLatency();
int nodeCount = state.getNodeCount();
int status = ok ? 1 : 0;
Date date = new Date();
ZookeeperRecord zookeeperRecord = new ZookeeperRecord(hostName,connections,watches,sent,received,mode,minLatency,avgLatency,maxLatency,nodeCount,status,date);
list.add(zookeeperRecord);
}
}
return list;
}
}

20
escheduler-api/src/test/java/cn/escheduler/api/controller/ExecutorControllerTest.java

@ -32,8 +32,11 @@ import org.springframework.test.context.junit4.SpringRunner;
import org.springframework.test.web.servlet.MockMvc;
import org.springframework.test.web.servlet.MvcResult;
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import org.springframework.web.context.WebApplicationContext;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
@ -66,4 +69,21 @@ public class ExecutorControllerTest {
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
}
@Test
public void getReceiverCc() throws Exception {
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
//paramsMap.add("processDefinitionId","4");
paramsMap.add("processInstanceId","13");
//paramsMap.add("processInstanceId","13");
MvcResult mvcResult = mockMvc.perform(get("/projects/{projectName}/executors/get-receiver-cc","li_sql_test")
.header("sessionId", "e79b3353-e227-4680-88c0-544194e64025")
.params(paramsMap))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8))
.andReturn();
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
}
}

105
escheduler-api/src/test/java/cn/escheduler/api/controller/MonitorControllerTest.java

@ -0,0 +1,105 @@
package cn.escheduler.api.controller;
import cn.escheduler.api.enums.Status;
import cn.escheduler.api.utils.Result;
import cn.escheduler.common.enums.ResourceType;
import cn.escheduler.common.utils.JSONUtils;
import com.alibaba.fastjson.JSONObject;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.http.MediaType;
import org.springframework.test.context.junit4.SpringRunner;
import org.springframework.test.web.servlet.MockMvc;
import org.springframework.test.web.servlet.MvcResult;
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
import org.springframework.web.context.WebApplicationContext;
import static org.junit.Assert.*;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
@RunWith(SpringRunner.class)
@SpringBootTest
public class MonitorControllerTest {
private static final Logger logger = LoggerFactory.getLogger(MonitorControllerTest.class);
public static final String SESSION_ID = "sessionId";
public static String SESSION_ID_VALUE;
private MockMvc mockMvc;
@Autowired
private WebApplicationContext webApplicationContext;
@Before
public void setUp() {
mockMvc = MockMvcBuilders.webAppContextSetup(webApplicationContext).build();
SESSION_ID_VALUE = "bad76fc4-2eb4-4aae-b32b-d650e4beb6af";
}
@Test
public void listMaster() throws Exception {
MvcResult mvcResult = mockMvc.perform(get("/monitor/master/list")
.header(SESSION_ID, SESSION_ID_VALUE)
/* .param("type", ResourceType.FILE.name())*/ )
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8))
.andReturn();
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
result.getCode().equals(Status.SUCCESS.getCode());
JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString());
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
}
@Test
public void queryDatabaseState() throws Exception {
MvcResult mvcResult = mockMvc.perform(get("/monitor/database")
.header(SESSION_ID, SESSION_ID_VALUE)
/* .param("type", ResourceType.FILE.name())*/ )
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8))
.andReturn();
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
result.getCode().equals(Status.SUCCESS.getCode());
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
}
@Test
public void queryZookeeperState() throws Exception {
MvcResult mvcResult = mockMvc.perform(get("/monitor/zookeeper/list")
.header(SESSION_ID, SESSION_ID_VALUE)
/* .param("type", ResourceType.FILE.name())*/ )
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8))
.andReturn();
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
result.getCode().equals(Status.SUCCESS.getCode());
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
}
}

2
escheduler-common/src/main/java/cn/escheduler/common/Constants.java

@ -437,7 +437,7 @@ public final class Constants {
/**
* task record configuration path
*/
public static final String TASK_RECORD_PROPERTIES_PATH = "dao/data_source.properties";
public static final String DATA_SOURCE_PROPERTIES = "dao/data_source.properties";
public static final String TASK_RECORD_URL = "task.record.datasource.url";

27
escheduler-common/src/main/java/cn/escheduler/common/task/sql/SqlParameters.java

@ -65,6 +65,16 @@ public class SqlParameters extends AbstractParameters {
*/
private String connParams;
/**
* receivers
*/
private String receivers;
/**
* receivers cc
*/
private String receiversCc;
public String getType() {
return type;
}
@ -121,6 +131,21 @@ public class SqlParameters extends AbstractParameters {
this.connParams = connParams;
}
public String getReceivers() {
return receivers;
}
public void setReceivers(String receivers) {
this.receivers = receivers;
}
public String getReceiversCc() {
return receiversCc;
}
public void setReceiversCc(String receiversCc) {
this.receiversCc = receiversCc;
}
@Override
public boolean checkParameters() {
@ -142,6 +167,8 @@ public class SqlParameters extends AbstractParameters {
", udfs='" + udfs + '\'' +
", showType='" + showType + '\'' +
", connParams='" + connParams + '\'' +
", receivers='" + receivers + '\'' +
", receiversCc='" + receiversCc + '\'' +
'}';
}
}

29
escheduler-common/src/main/java/cn/escheduler/common/zk/AbstractZKClient.java

@ -87,15 +87,6 @@ public abstract class AbstractZKClient {
public AbstractZKClient() {
StringBuilder sb = new StringBuilder();
String[] zookeeperParamslist = conf.getStringArray(Constants.ZOOKEEPER_QUORUM);
for (String param : zookeeperParamslist) {
sb.append(param).append(Constants.COMMA);
}
if(sb.length() > 0){
sb.deleteCharAt(sb.length() - 1);
}
// retry strategy
RetryPolicy retryPolicy = new ExponentialBackoffRetry(
@ -105,7 +96,7 @@ public abstract class AbstractZKClient {
try{
// crate zookeeper client
zkClient = CuratorFrameworkFactory.builder()
.connectString(sb.toString())
.connectString(getZookeeperQuorum())
.retryPolicy(retryPolicy)
.sessionTimeoutMs(1000 * Integer.parseInt(conf.getString(Constants.ZOOKEEPER_SESSION_TIMEOUT)))
.connectionTimeoutMs(1000 * Integer.parseInt(conf.getString(Constants.ZOOKEEPER_CONNECTION_TIMEOUT)))
@ -327,6 +318,24 @@ public abstract class AbstractZKClient {
return childrenList.size();
}
/**
*
* @return zookeeper quorum
*/
public static String getZookeeperQuorum(){
StringBuilder sb = new StringBuilder();
String[] zookeeperParamslist = conf.getStringArray(Constants.ZOOKEEPER_QUORUM);
for (String param : zookeeperParamslist) {
sb.append(param).append(Constants.COMMA);
}
if(sb.length() > 0){
sb.deleteCharAt(sb.length() - 1);
}
return sb.toString();
}
@Override
public String toString() {
return "AbstractZKClient{" +

54
escheduler-dao/readme.txt

@ -1,54 +0,0 @@
-- 用户指定队列
alter table t_escheduler_user add queue varchar(64);
-- 访问token
CREATE TABLE `t_escheduler_access_token` (
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键',
`user_id` int(11) DEFAULT NULL COMMENT '用户id',
`token` varchar(64) DEFAULT NULL COMMENT 'token令牌',
`expire_time` datetime DEFAULT NULL COMMENT 'token有效结束时间',
`create_time` datetime DEFAULT NULL COMMENT '创建时间',
`update_time` datetime DEFAULT NULL COMMENT '更新时间',
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=6 DEFAULT CHARSET=utf8;
CREATE TABLE `t_escheduler_error_command` (
`id` int(11) NOT NULL COMMENT '主键',
`command_type` tinyint(4) NULL DEFAULT NULL COMMENT '命令类型:0 启动工作流,1 从当前节点开始执行,2 恢复被容错的工作流,3 恢复暂停流程,4 从失败节点开始执行,5 补数,6 调度,7 重跑,8 暂停,9 停止,10 恢复等待线程',
`executor_id` int(11) NULL DEFAULT NULL COMMENT '命令执行者',
`process_definition_id` int(11) NULL DEFAULT NULL COMMENT '流程定义id',
`command_param` text CHARACTER SET utf8 COLLATE utf8_general_ci NULL COMMENT '命令的参数(json格式)',
`task_depend_type` tinyint(4) NULL DEFAULT NULL COMMENT '节点依赖类型',
`failure_strategy` tinyint(4) NULL DEFAULT 0 COMMENT '失败策略:0结束,1继续',
`warning_type` tinyint(4) NULL DEFAULT 0 COMMENT '告警类型',
`warning_group_id` int(11) NULL DEFAULT NULL COMMENT '告警组',
`schedule_time` datetime(0) NULL DEFAULT NULL COMMENT '预期运行时间',
`start_time` datetime(0) NULL DEFAULT NULL COMMENT '开始时间',
`update_time` datetime(0) NULL DEFAULT NULL COMMENT '更新时间',
`dependence` text CHARACTER SET utf8 COLLATE utf8_general_ci NULL COMMENT '依赖字段',
`process_instance_priority` int(11) NULL DEFAULT NULL COMMENT '流程实例优先级:0 Highest,1 High,2 Medium,3 Low,4 Lowest',
`message` text CHARACTER SET utf8 COLLATE utf8_general_ci NULL COMMENT '执行信息',
PRIMARY KEY (`id`) USING BTREE
) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci ROW_FORMAT = Dynamic;
CREATE TABLE `t_escheduler_worker_group` (
`id` bigint(11) NOT NULL AUTO_INCREMENT COMMENT 'id',
`name` varchar(256) CHARACTER SET latin1 COLLATE latin1_swedish_ci NULL DEFAULT NULL COMMENT '组名称',
`ip_list` varchar(256) CHARACTER SET latin1 COLLATE latin1_swedish_ci NULL DEFAULT NULL COMMENT 'worker地址列表',
`create_time` datetime(0) NULL DEFAULT NULL COMMENT '创建时间',
`update_time` datetime(0) NULL DEFAULT NULL COMMENT '更新时间',
PRIMARY KEY (`id`) USING BTREE
) ENGINE = InnoDB AUTO_INCREMENT = 1 CHARACTER SET = utf8 COLLATE = utf8_general_ci ROW_FORMAT = Dynamic;
ALTER TABLE `t_escheduler_task_instance`
ADD COLUMN `worker_group_id` int(11) NULL DEFAULT -1 COMMENT '任务指定运行的worker分组' AFTER `task_instance_priority`;
ALTER TABLE `t_escheduler_command`
ADD COLUMN `worker_group_id` int(11) NULL DEFAULT -1 COMMENT '任务指定运行的worker分组' NULL AFTER `process_instance_priority`;
ALTER TABLE `t_escheduler_error_command`
ADD COLUMN `worker_group_id` int(11) NULL DEFAULT -1 COMMENT '任务指定运行的worker分组' NULL AFTER `process_instance_priority`;
ALTER TABLE `t_escheduler_schedules`
ADD COLUMN `worker_group_id` int(11) NULL DEFAULT -1 COMMENT '任务指定运行的worker分组' NULL AFTER `process_instance_priority`;

145
escheduler-dao/src/main/java/cn/escheduler/dao/MonitorDBDao.java

@ -0,0 +1,145 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package cn.escheduler.dao;
import cn.escheduler.common.Constants;
import cn.escheduler.dao.model.MonitorRecord;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.sql.*;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
/**
* database state dao
*/
public class MonitorDBDao {
private static Logger logger = LoggerFactory.getLogger(MonitorDBDao.class);
public static final String VARIABLE_NAME = "variable_name";
/**
* 加载配置文件
*/
private static Configuration conf;
static {
try {
conf = new PropertiesConfiguration(Constants.DATA_SOURCE_PROPERTIES);
}catch (ConfigurationException e){
logger.error("load configuration excetpion",e);
System.exit(1);
}
}
/**
* create connection
* @return
*/
private static Connection getConn() {
String url = conf.getString(Constants.SPRING_DATASOURCE_URL);
String username = conf.getString(Constants.SPRING_DATASOURCE_USERNAME);
String password = conf.getString(Constants.SPRING_DATASOURCE_PASSWORD);
Connection conn = null;
try {
//classloader,load driver
Class.forName(Constants.JDBC_MYSQL_CLASS_NAME);
conn = DriverManager.getConnection(url, username, password);
} catch (ClassNotFoundException e) {
logger.error("ClassNotFoundException ", e);
} catch (SQLException e) {
logger.error("SQLException ", e);
}
return conn;
}
/**
* query database state
* @return
*/
public static List<MonitorRecord> queryDatabaseState() {
List<MonitorRecord> list = new ArrayList<>(1);
Connection conn = null;
long maxConnections = 0;
long maxUsedConnections = 0;
long threadsConnections = 0;
long threadsRunningConnections = 0;
//mysql running state
int state = 1;
MonitorRecord monitorRecord = new MonitorRecord();
try {
conn = getConn();
if(conn == null){
return list;
}
Statement pstmt = conn.createStatement();
ResultSet rs1 = pstmt.executeQuery("show global variables");
while(rs1.next()){
if(rs1.getString(VARIABLE_NAME).toUpperCase().equals("MAX_CONNECTIONS")){
maxConnections= Long.parseLong(rs1.getString("value"));
}
}
ResultSet rs2 = pstmt.executeQuery("show global status");
while(rs2.next()){
if(rs2.getString(VARIABLE_NAME).toUpperCase().equals("MAX_USED_CONNECTIONS")){
maxUsedConnections = Long.parseLong(rs2.getString("value"));
}else if(rs2.getString(VARIABLE_NAME).toUpperCase().equals("THREADS_CONNECTED")){
threadsConnections = Long.parseLong(rs2.getString("value"));
}else if(rs2.getString(VARIABLE_NAME).toUpperCase().equals("THREADS_RUNNING")){
threadsRunningConnections= Long.parseLong(rs2.getString("value"));
}
}
} catch (SQLException e) {
logger.error("SQLException ", e);
state = 0;
}finally {
try {
if(conn != null){
conn.close();
}
} catch (SQLException e) {
logger.error("SQLException ", e);
}
}
monitorRecord.setDate(new Date());
monitorRecord.setMaxConnections(maxConnections);
monitorRecord.setMaxUsedConnections(maxUsedConnections);
monitorRecord.setThreadsConnections(threadsConnections);
monitorRecord.setThreadsRunningConnections(threadsRunningConnections);
monitorRecord.setState(state);
list.add(monitorRecord);
return list;
}
}

2
escheduler-dao/src/main/java/cn/escheduler/dao/TaskRecordDao.java

@ -49,7 +49,7 @@ public class TaskRecordDao {
static {
try {
conf = new PropertiesConfiguration(Constants.TASK_RECORD_PROPERTIES_PATH);
conf = new PropertiesConfiguration(Constants.DATA_SOURCE_PROPERTIES);
}catch (ConfigurationException e){
logger.error("load configuration excetpion",e);
System.exit(1);

88
escheduler-dao/src/main/java/cn/escheduler/dao/mapper/MonitorMapper.java

@ -0,0 +1,88 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package cn.escheduler.dao.mapper;
import cn.escheduler.dao.model.Queue;
import org.apache.ibatis.annotations.*;
import org.apache.ibatis.type.JdbcType;
import java.util.List;
/**
* queue mapper
*/
public interface MonitorMapper {
/**
* insert queue
* @param queue
* @return
*/
@InsertProvider(type = QueueMapperProvider.class, method = "insert")
@Options(useGeneratedKeys = true,keyProperty = "queue.id")
@SelectKey(statement = "SELECT LAST_INSERT_ID()", keyProperty = "queue.id", before = false, resultType = int.class)
int insert(@Param("queue") Queue queue);
/**
* delete queue
* @param queueId
* @return
*/
@DeleteProvider(type = QueueMapperProvider.class, method = "delete")
int delete(@Param("queueId") int queueId);
/**
* update queue
*
* @param queue
* @return
*/
@UpdateProvider(type = QueueMapperProvider.class, method = "update")
int update(@Param("queue") Queue queue);
/**
* query queue by id
* @param queueId
* @return
*/
@Results(value = {@Result(property = "id", column = "id", id = true, javaType = Integer.class, jdbcType = JdbcType.INTEGER),
@Result(property = "queueName", column = "queue_name", javaType = String.class, jdbcType = JdbcType.VARCHAR),
@Result(property = "queue", column = "queue", javaType = String.class, jdbcType = JdbcType.VARCHAR)
})
@SelectProvider(type = QueueMapperProvider.class, method = "queryById")
Queue queryById(@Param("queueId") int queueId);
/**
* query all queue list
* @return
*/
@Results(value = {@Result(property = "id", column = "id", id = true, javaType = Integer.class, jdbcType = JdbcType.INTEGER),
@Result(property = "queueName", column = "queue_name", javaType = String.class, jdbcType = JdbcType.VARCHAR),
@Result(property = "queue", column = "queue", javaType = String.class, jdbcType = JdbcType.VARCHAR)
})
@SelectProvider(type = QueueMapperProvider.class, method = "queryAllQueue")
List<Queue> queryAllQueue();
}

115
escheduler-dao/src/main/java/cn/escheduler/dao/model/MonitorRecord.java

@ -0,0 +1,115 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package cn.escheduler.dao.model;
import java.util.Date;
/**
* monitor record for database
*/
public class MonitorRecord {
/**
* is normal or not , 1: normal
*/
private int state;
/**
* max connections
*/
private long maxConnections;
/**
* max used connections
*/
private long maxUsedConnections;
/**
* threads connections
*/
private long threadsConnections;
/**
* threads running connections
*/
private long threadsRunningConnections;
/**
* start date
*/
private Date date;
public int getState() {
return state;
}
public void setState(int state) {
this.state = state;
}
public long getMaxConnections() {
return maxConnections;
}
public void setMaxConnections(long maxConnections) {
this.maxConnections = maxConnections;
}
public long getMaxUsedConnections() {
return maxUsedConnections;
}
public void setMaxUsedConnections(long maxUsedConnections) {
this.maxUsedConnections = maxUsedConnections;
}
public long getThreadsConnections() {
return threadsConnections;
}
public void setThreadsConnections(long threadsConnections) {
this.threadsConnections = threadsConnections;
}
public long getThreadsRunningConnections() {
return threadsRunningConnections;
}
public void setThreadsRunningConnections(long threadsRunningConnections) {
this.threadsRunningConnections = threadsRunningConnections;
}
public Date getDate() {
return date;
}
public void setDate(Date date) {
this.date = date;
}
@Override
public String toString() {
return "MonitorRecord{" +
"state=" + state +
", maxConnections=" + maxConnections +
", maxUsedConnections=" + maxUsedConnections +
", threadsConnections=" + threadsConnections +
", threadsRunningConnections=" + threadsRunningConnections +
", date=" + date +
'}';
}
}

217
escheduler-dao/src/main/java/cn/escheduler/dao/model/ZookeeperRecord.java

@ -0,0 +1,217 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package cn.escheduler.dao.model;
import java.util.Date;
/**
* monitor record for zookeeper
*/
public class ZookeeperRecord {
/**
* hostname
*/
private String hostname;
/**
* connections
*/
private int connections;
/**
* max connections
*/
private int watches;
/**
* sent
*/
private long sent;
/**
* received
*/
private long received;
/**
* mode: leader or follower
*/
private String mode;
/**
* min Latency
*/
private int minLatency;
/**
* avg Latency
*/
private int avgLatency;
/**
* max Latency
*/
private int maxLatency;
/**
* node count
*/
private int nodeCount;
/**
* date
*/
private Date date;
/**
* is normal or not, 1:normal
*/
private int state;
public ZookeeperRecord(String hostname,int connections, int watches, long sent, long received, String mode, int minLatency, int avgLatency, int maxLatency, int nodeCount, int state,Date date) {
this.hostname = hostname;
this.connections = connections;
this.watches = watches;
this.sent = sent;
this.received = received;
this.mode = mode;
this.minLatency = minLatency;
this.avgLatency = avgLatency;
this.maxLatency = maxLatency;
this.nodeCount = nodeCount;
this.state = state;
this.date = date;
}
public String getHostname() {
return hostname;
}
public void setHostname(String hostname) {
this.hostname = hostname;
}
public int getConnections() {
return connections;
}
public void setConnections(int connections) {
this.connections = connections;
}
public int getWatches() {
return watches;
}
public void setWatches(int watches) {
this.watches = watches;
}
public long getSent() {
return sent;
}
public void setSent(long sent) {
this.sent = sent;
}
public long getReceived() {
return received;
}
public void setReceived(long received) {
this.received = received;
}
public String getMode() {
return mode;
}
public void setMode(String mode) {
this.mode = mode;
}
public int getMinLatency() {
return minLatency;
}
public void setMinLatency(int minLatency) {
this.minLatency = minLatency;
}
public int getAvgLatency() {
return avgLatency;
}
public void setAvgLatency(int avgLatency) {
this.avgLatency = avgLatency;
}
public int getMaxLatency() {
return maxLatency;
}
public void setMaxLatency(int maxLatency) {
this.maxLatency = maxLatency;
}
public int getNodeCount() {
return nodeCount;
}
public void setNodeCount(int nodeCount) {
this.nodeCount = nodeCount;
}
public int getState() {
return state;
}
public void setState(int state) {
this.state = state;
}
public Date getDate() {
return date;
}
public void setDate(Date date) {
this.date = date;
}
@Override
public String toString() {
return "ZookeeperRecord{" +
"hostname='" + hostname + '\'' +
", connections=" + connections +
", watches=" + watches +
", sent=" + sent +
", received=" + received +
", mode='" + mode + '\'' +
", minLatency=" + minLatency +
", avgLatency=" + avgLatency +
", maxLatency=" + maxLatency +
", nodeCount=" + nodeCount +
", date=" + date +
", state=" + state +
'}';
}
}

22
escheduler-server/src/main/java/cn/escheduler/server/ResInfo.java

@ -35,6 +35,11 @@ public class ResInfo {
*/
private double memoryUsage;
/**
* loadAverage
*/
private double loadAverage;
public ResInfo(){}
public ResInfo(double cpuUsage , double memoryUsage){
@ -42,6 +47,12 @@ public class ResInfo {
this.memoryUsage = memoryUsage;
}
public ResInfo(double cpuUsage, double memoryUsage, double loadAverage) {
this.cpuUsage = cpuUsage;
this.memoryUsage = memoryUsage;
this.loadAverage = loadAverage;
}
public double getCpuUsage() {
return cpuUsage;
}
@ -58,12 +69,21 @@ public class ResInfo {
this.memoryUsage = memoryUsage;
}
public double getLoadAverage() {
return loadAverage;
}
public void setLoadAverage(double loadAverage) {
this.loadAverage = loadAverage;
}
/**
* get CPU and memory usage
* add cpu load average by lidong for service monitor
* @return
*/
public static String getResInfoJson(){
ResInfo resInfo = new ResInfo(OSUtils.cpuUsage(), OSUtils.memoryUsage());
ResInfo resInfo = new ResInfo(OSUtils.cpuUsage(), OSUtils.memoryUsage(),OSUtils.loadAverage());
return JSONUtils.toJson(resInfo);
}

7
escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java

@ -305,7 +305,7 @@ public class SqlTask extends AbstractTask {
receviersList.add(user.getEmail());
}
// custom receiver
String receivers = processDefine.getReceivers();
String receivers = sqlParameters.getReceivers();
if (StringUtils.isNotEmpty(receivers)){
String[] splits = receivers.split(Constants.COMMA);
for (String receiver : splits){
@ -315,11 +315,8 @@ public class SqlTask extends AbstractTask {
// copy list
List<String> receviersCcList = new ArrayList<String>();
// Custom Copier
String receiversCc = processDefine.getReceiversCc();
String receiversCc = sqlParameters.getReceiversCc();
if (StringUtils.isNotEmpty(receiversCc)){
String[] splits = receiversCc.split(Constants.COMMA);
for (String receiverCc : splits){

47
escheduler-ui/install(线上环境).sh

@ -1,21 +1,52 @@
#!/bin/bash
echo "escheduler-ui-install.sh"
echo "escheduler-ui目录下执行"
# 当前路径
esc_basepath=$(cd `dirname $0`; pwd)
echo "欢迎使用easy scheduler前端部署脚本,目前前端部署脚本仅支持Centos"
echo "请在 escheduler-ui 目录下执行"
# 配置前端访问端口
esc_proxy="8888"
# 配置代理后端接口
esc_proxy_port="http://192.168.220.154:12345"
# 当前路径
esc_basepath=$(cd `dirname $0`; pwd)
esc_proxy_port="http://192.168.xx.xx:12345"
# 本机ip
esc_ipaddr='172.0.0.1'
esc_ipaddr='127.0.0.1'
esc_ipaddr=$(ip addr | awk '/^[0-9]+: / {}; /inet.*global/ {print gensub(/(.*)\/(.*)/, "\\1", "g", $2)}')
#To be compatible with MacOS and Linux
if [[ "$OSTYPE" == "darwin"* ]]; then
# Mac OSX
echo "Easy Scheduler ui install not support Mac OSX operating system"
exit 1
elif [[ "$OSTYPE" == "linux-gnu" ]]; then
# linux
echo "linux"
elif [[ "$OSTYPE" == "cygwin" ]]; then
# POSIX compatibility layer and Linux environment emulation for Windows
echo "Easy Scheduler ui not support Windows operating system"
exit 1
elif [[ "$OSTYPE" == "msys" ]]; then
# Lightweight shell and GNU utilities compiled for Windows (part of MinGW)
echo "Easy Scheduler ui not support Windows operating system"
exit 1
elif [[ "$OSTYPE" == "win32" ]]; then
echo "Easy Scheduler ui not support Windows operating system"
exit 1
elif [[ "$OSTYPE" == "freebsd"* ]]; then
# ...
echo "freebsd"
else
# Unknown.
echo "Operating system unknown, please tell us(submit issue) for better service"
exit 1
fi
# 区分版本
version=`cat /etc/redhat-release|sed -r 's/.* ([0-9]+)\..*/\1/'`

164
sql/upgrade/1.0.2_schema/mysql/escheduler_ddl.sql

@ -18,4 +18,166 @@ d//
delimiter ;
CALL ac_escheduler_T_t_escheduler_version;
DROP PROCEDURE ac_escheduler_T_t_escheduler_version;
DROP PROCEDURE ac_escheduler_T_t_escheduler_version;
-- ac_escheduler_T_t_escheduler_user_C_queue
drop PROCEDURE if EXISTS ac_escheduler_T_t_escheduler_user_C_queue;
delimiter d//
CREATE PROCEDURE ac_escheduler_T_t_escheduler_user_C_queue()
BEGIN
IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS
WHERE TABLE_NAME='t_escheduler_user'
AND TABLE_SCHEMA=(SELECT DATABASE())
AND COLUMN_NAME='queue')
THEN
ALTER TABLE t_escheduler_user ADD COLUMN queue varchar(64) COMMENT '队列' AFTER update_time;
END IF;
END;
d//
delimiter ;
CALL ac_escheduler_T_t_escheduler_user_C_queue;
DROP PROCEDURE ac_escheduler_T_t_escheduler_user_C_queue;
-- ac_escheduler_T_t_escheduler_access_token
drop PROCEDURE if EXISTS ac_escheduler_T_t_escheduler_access_token;
delimiter d//
CREATE PROCEDURE ac_escheduler_T_t_escheduler_access_token()
BEGIN
drop table if exists t_escheduler_access_token;
CREATE TABLE IF NOT EXISTS `t_escheduler_access_token` (
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键',
`user_id` int(11) DEFAULT NULL COMMENT '用户id',
`token` varchar(64) DEFAULT NULL COMMENT 'token令牌',
`expire_time` datetime DEFAULT NULL COMMENT 'token有效结束时间',
`create_time` datetime DEFAULT NULL COMMENT '创建时间',
`update_time` datetime DEFAULT NULL COMMENT '更新时间',
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
END;
d//
delimiter ;
CALL ac_escheduler_T_t_escheduler_access_token;
DROP PROCEDURE ac_escheduler_T_t_escheduler_access_token;
-- ac_escheduler_T_t_escheduler_error_command
drop PROCEDURE if EXISTS ac_escheduler_T_t_escheduler_error_command;
delimiter d//
CREATE PROCEDURE ac_escheduler_T_t_escheduler_error_command()
BEGIN
drop table if exists t_escheduler_error_command;
CREATE TABLE IF NOT EXISTS `t_escheduler_error_command` (
`id` int(11) NOT NULL COMMENT '主键',
`command_type` tinyint(4) NULL DEFAULT NULL COMMENT '命令类型:0 启动工作流,1 从当前节点开始执行,2 恢复被容错的工作流,3 恢复暂停流程,4 从失败节点开始执行,5 补数,6 调度,7 重跑,8 暂停,9 停止,10 恢复等待线程',
`executor_id` int(11) NULL DEFAULT NULL COMMENT '命令执行者',
`process_definition_id` int(11) NULL DEFAULT NULL COMMENT '流程定义id',
`command_param` text CHARACTER SET utf8 COLLATE utf8_general_ci NULL COMMENT '命令的参数(json格式)',
`task_depend_type` tinyint(4) NULL DEFAULT NULL COMMENT '节点依赖类型',
`failure_strategy` tinyint(4) NULL DEFAULT 0 COMMENT '失败策略:0结束,1继续',
`warning_type` tinyint(4) NULL DEFAULT 0 COMMENT '告警类型',
`warning_group_id` int(11) NULL DEFAULT NULL COMMENT '告警组',
`schedule_time` datetime(0) NULL DEFAULT NULL COMMENT '预期运行时间',
`start_time` datetime(0) NULL DEFAULT NULL COMMENT '开始时间',
`update_time` datetime(0) NULL DEFAULT NULL COMMENT '更新时间',
`dependence` text CHARACTER SET utf8 COLLATE utf8_general_ci NULL COMMENT '依赖字段',
`process_instance_priority` int(11) NULL DEFAULT NULL COMMENT '流程实例优先级:0 Highest,1 High,2 Medium,3 Low,4 Lowest',
`worker_group_id` int(11) NULL DEFAULT -1 COMMENT '任务指定运行的worker分组',
`message` text CHARACTER SET utf8 COLLATE utf8_general_ci NULL COMMENT '执行信息',
PRIMARY KEY (`id`) USING BTREE
) ENGINE = InnoDB AUTO_INCREMENT=1 CHARACTER SET = utf8 COLLATE = utf8_general_ci ROW_FORMAT = Dynamic;
END;
d//
delimiter ;
CALL ac_escheduler_T_t_escheduler_error_command;
DROP PROCEDURE ac_escheduler_T_t_escheduler_error_command;
-- ac_escheduler_T_t_escheduler_worker_group
drop PROCEDURE if EXISTS ac_escheduler_T_t_escheduler_worker_group;
delimiter d//
CREATE PROCEDURE ac_escheduler_T_t_escheduler_worker_group()
BEGIN
drop table if exists t_escheduler_worker_group;
CREATE TABLE IF NOT EXISTS `t_escheduler_worker_group` (
`id` bigint(11) NOT NULL AUTO_INCREMENT COMMENT 'id',
`name` varchar(256) CHARACTER SET latin1 COLLATE latin1_swedish_ci NULL DEFAULT NULL COMMENT '组名称',
`ip_list` varchar(256) CHARACTER SET latin1 COLLATE latin1_swedish_ci NULL DEFAULT NULL COMMENT 'worker地址列表',
`create_time` datetime(0) NULL DEFAULT NULL COMMENT '创建时间',
`update_time` datetime(0) NULL DEFAULT NULL COMMENT '更新时间',
PRIMARY KEY (`id`) USING BTREE
) ENGINE = InnoDB AUTO_INCREMENT=1 CHARACTER SET = utf8 COLLATE = utf8_general_ci ROW_FORMAT = Dynamic;
END;
d//
delimiter ;
CALL ac_escheduler_T_t_escheduler_worker_group;
DROP PROCEDURE ac_escheduler_T_t_escheduler_worker_group;
-- ac_escheduler_T_t_escheduler_task_instance_C_worker_group_id
drop PROCEDURE if EXISTS ac_escheduler_T_t_escheduler_task_instance_C_worker_group_id;
delimiter d//
CREATE PROCEDURE ac_escheduler_T_t_escheduler_task_instance_C_worker_group_id()
BEGIN
IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS
WHERE TABLE_NAME='t_escheduler_task_instance'
AND TABLE_SCHEMA=(SELECT DATABASE())
AND COLUMN_NAME='worker_group_id')
THEN
ALTER TABLE t_escheduler_task_instance ADD COLUMN `worker_group_id` int(11) NULL DEFAULT -1 COMMENT '任务指定运行的worker分组' AFTER `task_instance_priority`;
END IF;
END;
d//
delimiter ;
CALL ac_escheduler_T_t_escheduler_task_instance_C_worker_group_id;
DROP PROCEDURE ac_escheduler_T_t_escheduler_task_instance_C_worker_group_id;
-- ac_escheduler_T_t_escheduler_command_C_worker_group_id
drop PROCEDURE if EXISTS ac_escheduler_T_t_escheduler_command_C_worker_group_id;
delimiter d//
CREATE PROCEDURE ac_escheduler_T_t_escheduler_command_C_worker_group_id()
BEGIN
IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS
WHERE TABLE_NAME='t_escheduler_command'
AND TABLE_SCHEMA=(SELECT DATABASE())
AND COLUMN_NAME='worker_group_id')
THEN
ALTER TABLE t_escheduler_command ADD COLUMN `worker_group_id` int(11) NULL DEFAULT -1 COMMENT '任务指定运行的worker分组' AFTER `process_instance_priority`;
END IF;
END;
d//
delimiter ;
CALL ac_escheduler_T_t_escheduler_command_C_worker_group_id;
DROP PROCEDURE ac_escheduler_T_t_escheduler_command_C_worker_group_id;
-- ac_escheduler_T_t_escheduler_schedules_C_worker_group_id
drop PROCEDURE if EXISTS ac_escheduler_T_t_escheduler_schedules_C_worker_group_id;
delimiter d//
CREATE PROCEDURE ac_escheduler_T_t_escheduler_schedules_C_worker_group_id()
BEGIN
IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS
WHERE TABLE_NAME='t_escheduler_schedules'
AND TABLE_SCHEMA=(SELECT DATABASE())
AND COLUMN_NAME='worker_group_id')
THEN
ALTER TABLE t_escheduler_schedules ADD COLUMN `worker_group_id` int(11) NULL DEFAULT -1 COMMENT '任务指定运行的worker分组' AFTER `process_instance_priority`;
END IF;
END;
d//
delimiter ;
CALL ac_escheduler_T_t_escheduler_schedules_C_worker_group_id;
DROP PROCEDURE ac_escheduler_T_t_escheduler_schedules_C_worker_group_id;
Loading…
Cancel
Save