<!DOCTYPE HTML>
< html lang = "zh-hans" >
< head >
< meta charset = "UTF-8" >
< meta content = "text/html; charset=utf-8" http-equiv = "Content-Type" >
< title > 后端部署 · 调度系统-EasyScheduler< / title >
< meta http-equiv = "X-UA-Compatible" content = "IE=edge" / >
< meta name = "description" content = "" >
< meta name = "generator" content = "GitBook 3.2.3" >
< meta name = "author" content = "YIGUAN" >
< link rel = "stylesheet" href = "../gitbook/style.css" >
< link rel = "stylesheet" href = "../gitbook/gitbook-plugin-expandable-chapters/expandable-chapters.css" >
< link rel = "stylesheet" href = "../gitbook/gitbook-plugin-insert-logo-link/plugin.css" >
< link rel = "stylesheet" href = "../gitbook/gitbook-plugin-highlight/website.css" >
< link rel = "stylesheet" href = "../gitbook/gitbook-plugin-search/search.css" >
< link rel = "stylesheet" href = "../gitbook/gitbook-plugin-fontsettings/website.css" >
< link rel = "stylesheet" href = "../styles/website.css" >
< meta name = "HandheldFriendly" content = "true" / >
< meta name = "viewport" content = "width=device-width, initial-scale=1, user-scalable=no" >
< meta name = "apple-mobile-web-app-capable" content = "yes" >
< meta name = "apple-mobile-web-app-status-bar-style" content = "black" >
< link rel = "apple-touch-icon-precomposed" sizes = "152x152" href = "../gitbook/images/apple-touch-icon-precomposed-152.png" >
< link rel = "shortcut icon" href = "../gitbook/images/favicon.ico" type = "image/x-icon" >
< link rel = "next" href = "deploy-background.html" / >
< link rel = "prev" href = "deploy-foreground.html" / >
< / head >
< body >
< div class = "book" >
< div class = "book-summary" >
< div id = "book-search-input" role = "search" >
< input type = "text" placeholder = "输入并搜索" / >
< / div >
< nav role = "navigation" >
< ul class = "summary" >
< li class = "chapter " data-level = "1.1" data-path = "../" >
< a href = "../" >
Easyscheduler简介
< / a >
< / li >
< li class = "chapter " data-level = "1.2" >
< span >
导读指南
< / span >
< ul class = "articles" >
< li class = "chapter " data-level = "1.2.1" data-path = "guide-architecture.html" >
< a href = "guide-architecture.html" >
系统架构分析
< / a >
< ul class = "articles" >
< li class = "chapter " data-level = "1.2.1.1" data-path = "guide-architecture.html" >
< a href = "guide-architecture.html#导语" >
导语
< / a >
< / li >
< li class = "chapter " data-level = "1.2.1.2" data-path = "guide-architecture.html" >
< a href = "guide-architecture.html#背景" >
背景
< / a >
< / li >
< li class = "chapter " data-level = "1.2.1.3" data-path = "guide-architecture.html" >
< a href = "guide-architecture.html#总架构设计" >
总架构设计
< / a >
< / li >
< li class = "chapter " data-level = "1.2.1.4" data-path = "guide-architecture.html" >
< a href = "guide-architecture.html#中心化思想" >
中心化思想
< / a >
< / li >
< li class = "chapter " data-level = "1.2.1.5" data-path = "guide-architecture.html" >
< a href = "guide-architecture.html#分布式锁实践" >
分布式锁实践
< / a >
< / li >
< li class = "chapter " data-level = "1.2.1.6" data-path = "guide-architecture.html" >
< a href = "guide-architecture.html#线程不足循环等待问题" >
线程不足循环等待问题
< / a >
< / li >
< li class = "chapter " data-level = "1.2.1.7" data-path = "guide-architecture.html" >
< a href = "guide-architecture.html#容错设计" >
容错设计
< / a >
< / li >
< li class = "chapter " data-level = "1.2.1.8" data-path = "guide-architecture.html" >
< a href = "guide-architecture.html#日志访问" >
Logback和gRPC实现日志访问
< / a >
< / li >
< li class = "chapter " data-level = "1.2.1.9" data-path = "guide-architecture.html" >
< a href = "guide-architecture.html#总结" >
总结
< / a >
< / li >
< / ul >
< / li >
< li class = "chapter " data-level = "1.2.2" data-path = "guide-manual.html" >
< a href = "guide-manual.html" >
使用手册
< / a >
< ul class = "articles" >
< li class = "chapter " data-level = "1.2.2.1" data-path = "guide-manual.html" >
< a href = "guide-manual.html#登录" >
登录
< / a >
< / li >
< li class = "chapter " data-level = "1.2.2.2" data-path = "guide-manual.html" >
< a href = "guide-manual.html#安全中心" >
安全中心
< / a >
< / li >
< li class = "chapter " data-level = "1.2.2.3" data-path = "guide-manual.html" >
< a href = "guide-manual.html#资源中心" >
资源中心
< / a >
< / li >
< li class = "chapter " data-level = "1.2.2.4" data-path = "guide-manual.html" >
< a href = "guide-manual.html#数据源中心" >
数据源中心
< / a >
< / li >
< li class = "chapter " data-level = "1.2.2.5" data-path = "guide-manual.html" >
< a href = "guide-manual.html#项目管理" >
项目管理
< / a >
< / li >
< li class = "chapter " data-level = "1.2.2.6" data-path = "guide-manual.html" >
< a href = "guide-manual.html#首页" >
站点首页
< / a >
< / li >
< li class = "chapter " data-level = "1.2.2.7" data-path = "guide-manual.html" >
< a href = "guide-manual.html#任务依赖" >
任务依赖
< / a >
< / li >
< li class = "chapter " data-level = "1.2.2.8" data-path = "guide-manual.html" >
< a href = "guide-manual.html#系统参数" >
系统参数
< / a >
< / li >
< / ul >
< / li >
< / ul >
< / li >
< li class = "chapter " data-level = "1.3" >
< span >
项目部署文档
< / span >
< ul class = "articles" >
< li class = "chapter " data-level = "1.3.1" data-path = "deploy-foreground.html" >
< a href = "deploy-foreground.html" >
前端部署
< / a >
< ul class = "articles" >
< li class = "chapter " data-level = "1.3.1.1" data-path = "deploy-foreground.html" >
< a href = "deploy-foreground.html#前端项目环境构建及编译" >
前端项目环境构建及编译
< / a >
< / li >
< li class = "chapter " data-level = "1.3.1.2" data-path = "deploy-foreground.html" >
< a href = "deploy-foreground.html#安装及配置" >
安装及配置
< / a >
< / li >
< li class = "chapter " data-level = "1.3.1.3" data-path = "deploy-foreground.html" >
< a href = "deploy-foreground.html#项目生产环境配置" >
项目生产环境Nginx配置
< / a >
< / li >
< li class = "chapter " data-level = "1.3.1.4" data-path = "deploy-foreground.html" >
< a href = "deploy-foreground.html#前端项目发布" >
前端项目发布
< / a >
< / li >
< li class = "chapter " data-level = "1.3.1.5" data-path = "deploy-foreground.html" >
< a href = "deploy-foreground.html#问题" >
问题
< / a >
< / li >
< / ul >
< / li >
< li class = "chapter active" data-level = "1.3.2" data-path = "deploy-background.html" >
< a href = "deploy-background.html" >
后端部署
< / a >
< ul class = "articles" >
< li class = "chapter " data-level = "1.3.2.1" data-path = "deploy-background.html" >
< a href = "deploy-background.html#基础软件安装" >
基础软件安装
< / a >
< / li >
< li class = "chapter " data-level = "1.3.2.2" data-path = "deploy-background.html" >
< a href = "deploy-background.html#项目编译" >
项目编译
< / a >
< / li >
< li class = "chapter " data-level = "1.3.2.3" data-path = "deploy-background.html" >
< a href = "deploy-background.html#数据库初始化" >
数据库初始化
< / a >
< / li >
< li class = "chapter " data-level = "1.3.2.4" data-path = "deploy-background.html" >
< a href = "deploy-background.html#创建部署用户" >
创建部署用户
< / a >
< / li >
< li class = "chapter " data-level = "1.3.2.5" data-path = "deploy-background.html" >
< a href = "deploy-background.html#配置文件" >
配置文件
< / a >
< ul class = "articles" >
< li class = "chapter " data-level = "1.3.2.5.1" data-path = "deploy-background.html" >
< a href = "deploy-background.html#escheduler-alert" >
escheduler-alert
< / a >
< / li >
< li class = "chapter " data-level = "1.3.2.5.2" data-path = "deploy-background.html" >
< a href = "deploy-background.html#escheduler-common" >
escheduler-common
< / a >
< / li >
< li class = "chapter " data-level = "1.3.2.5.3" data-path = "deploy-background.html" >
< a href = "deploy-background.html#escheduler-dao" >
escheduler-dao
< / a >
< / li >
< li class = "chapter " data-level = "1.3.2.5.4" data-path = "deploy-background.html" >
< a href = "deploy-background.html#escheduler-server" >
escheduler-server
< / a >
< / li >
< li class = "chapter " data-level = "1.3.2.5.5" data-path = "deploy-background.html" >
< a href = "deploy-background.html#escheduler-web" >
escheduler-web
< / a >
< / li >
< / ul >
< / li >
< li class = "chapter " data-level = "1.3.2.6" data-path = "deploy-background.html" >
< a href = "deploy-background.html#启动停止命令" >
启动停止命令
< / a >
< / li >
< li class = "chapter " data-level = "1.3.2.7" data-path = "deploy-background.html" >
< a href = "deploy-background.html#一键启停脚本" >
一键启停脚本
< / a >
< / li >
< li class = "chapter " data-level = "1.3.2.8" data-path = "deploy-background.html" >
< a href = "deploy-background.html#监控服务" >
监控服务
< / a >
< / li >
< li class = "chapter " data-level = "1.3.2.9" data-path = "deploy-background.html" >
< a href = "deploy-background.html#日志查看" >
日志查看
< / a >
< / li >
< / ul >
< / li >
< / ul >
< / li >
< li class = "chapter " data-level = "1.4" >
< span >
前端开发文档
< / span >
< ul class = "articles" >
< li class = "chapter " data-level = "1.4.1" data-path = "foreground-2.html" >
< a href = "foreground-2.html" >
安装
< / a >
< / li >
< / ul >
< / li >
< li class = "chapter " data-level = "1.5" >
< span >
后端开发文档
< / span >
< ul class = "articles" >
< li class = "chapter " data-level = "1.5.1" data-path = "background-2.html" >
< a href = "background-2.html" >
后端说明2
< / a >
< / li >
< / ul >
< / li >
< li class = "divider" > < / li >
< li >
< a href = "https://www.gitbook.com" target = "blank" class = "gitbook-link" >
本书使用 GitBook 发布
< / a >
< / li >
< / ul >
< / nav >
< / div >
< div class = "book-body" >
< div class = "body-inner" >
< div class = "book-header" role = "navigation" >
<!-- Title -->
< h1 >
< i class = "fa fa-circle-o-notch fa-spin" > < / i >
< a href = ".." > 后端部署< / a >
< / h1 >
< / div >
< div class = "page-wrapper" tabindex = "-1" role = "main" >
< div class = "page-inner" >
< div id = "book-search-results" >
< div class = "search-noresults" >
< section class = "normal markdown-section" >
< p > ​ < / p >
< h1 id = "部署文档" > 部 署 文 档 < / h1 >
< h2 id = "软件要求" > 软 件 要 求 < / h2 >
< ul >
< li > < a href = "https://blog.csdn.net/u011886447/article/details/79796802" target = "_blank" > Mysql< / a > (5.5+) : 必 装 < / li >
< li > < a href = "http://www.oracle.com/technetwork/java/javase/downloads/index.html" target = "_blank" > JDK< / a > (1.8+) ： 必 装 < / li >
< li > < a href = "https://www.jianshu.com/p/de90172ea680" target = "_blank" > Zookeeper< / a > (3.4.6) ： 必 装 < / li >
< li > < a href = "https://blog.csdn.net/Evankaka/article/details/51612437" target = "_blank" > Hadoop< / a > (2.7+) ： 选 装 ， 如 果 需 要 使 用 到 EasyScheduler的 资 源 上 传 ， MapReduce任 务 在 线 提 交 则 需 要 安 装 (上 传 的 资 源 文 件 目 前 保 存 在 Hdfs上 )< / li >
< li > < a href = "https://staroon.pro/2017/12/09/HiveInstall/" target = "_blank" > Hive< / a > (1.2.1+) : 选 装 ， 如 果 跑 Hive任 务 需 要 安 装 (< / li >
< li > < a href = "https://www.cnblogs.com/caoguo/p/4625662.html" target = "_blank" > Reids安 装 < / a > (2.7.0+) : 选 装 ， 任 务 队 列 选 择 Redis时 需 要 安 装 < / li >
< li > Spark(1.x,2.x) : 选 装 ， Spark任 务 提 交 需 要 安 装 < / li >
< li > < p > PostgreSQL(8.2.15+) : 选 装 ， PostgreSQL存 储 过 程 需 要 安 装 < / p >
< blockquote >
< p > 注 意 ： EasyScheduler本 身 不 依 赖 Hadoop、 Hive、 Spark、 PostgreSQL、 Redis,仅 是 用 到 了 他 们 的 Client jar， 用 于 对 应 任 务 的 运 行 。 < / p >
< / blockquote >
< / li >
< / ul >
< h2 id = "项目编译" > 项 目 编 译 < / h2 >
< ul >
< li > 执 行 编 译 命 令 ： < / li >
< / ul >
< blockquote >
< p > < code > mvn -U clean package assembly:assembly -Dmaven.test.skip=true< / code > < / p >
< / blockquote >
< ul >
< li > 查 看 目 录 < / li >
< / ul >
< p > 正 常 编 译 完 后 ， 会 在 当 前 目 录 生 成 target/escheduler-{version}-SNAPSHOT/< / p >
< pre > < code > bin
conf
lib
script
sql
< / code > < / pre > < ul >
< li > < p > 说 明 < / p >
< pre > < code > bin : 工 程 服 务 启 动 脚 本
conf : 工 程 配 置 文 件
lib : 工 程 依 赖 jar包 ， 包 括 各 个 模 块 jar和 第 三 方 jar
script : 工 程 自 动 化 部 署 、 启 动 脚 本
sql : 工 程 依 赖 sql文 件
< / code > < / pre > < / li >
< / ul >
< h2 id = "数据库初始化" > 数 据 库 初 始 化 < / h2 >
< ul >
< li > 创 建 database和 账 号 < / li >
< / ul >
< pre > < code > mysql -h {host} -u {user} -p{password}
mysql> CREATE DATABASE escheduler DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
mysql> GRANT ALL PRIVILEGES ON escheduler.* TO ' {user}' @' %' IDENTIFIED BY ' {password}' ;
mysql> GRANT ALL PRIVILEGES ON escheduler.* TO ' {user}' @' localhost' IDENTIFIED BY ' {password}' ;
mysql> flush privileges;
< / code > < / pre > < ul >
< li > 创 建 表 < / li >
< / ul >
< pre > < code > 说 明 ： 在 target/escheduler-{version}-SNAPSHOT/sql/有 两 个 sql创 建 表 文 件 ： escheduler.sql和 quartz.sql
执 行 ：
mysql -h {host} -u {user} -p{password} -D {db} < escheduler.sql
mysql -h {host} -u {user} -p{password} -D {db} < quartz.sql
< / code > < / pre > < h2 id = "创建部署用户" > 创 建 部 署 用 户 < / h2 >
< p > 因 为 easyscheduler worker都 是 以 sudo -u {linux-user} 方 式 来 执 行 作 业 ， 所 以 部 署 用 户 需 要 有 sudo 权 限 ， 而 且 是 免 密 的 。 < / p >
< pre > < code > 部 署 账 号
vi /etc/sudoers
# 部 署 用 户 是 escheduler 账 号
escheduler ALL=(ALL) NOPASSWD: NOPASSWD: ALL
# 并 且 需 要 注 释 掉 Default requiretty 一 行
#Default requiretty
< / code > < / pre > < h2 id = "配置文件" > 配 置 文 件 < / h2 >
< p > < code > 说 明 ： 配 置 文 件 位 于 target/escheduler-{version}-SNAPSHOT/conf 下 面 < / code > < / p >
< h3 id = "escheduler-alert" > escheduler-alert< / h3 >
< p > 配 置 邮 件 告 警 信 息 < / p >
< ul >
< li > alert.properties < / li >
< / ul >
< pre > < code > #以 qq邮 箱 为 例 ， 如 果 是 别 的 邮 箱 ， 请 更 改 对 应 配 置
#alert type is EMAIL/SMS
alert.type=EMAIL
# mail server configuration
mail.protocol=SMTP
mail.server.host=smtp.exmail.qq.com
mail.server.port=25
mail.sender=xxxxxx@qq.com
mail.passwd=xxxxxxx
# xls file path, need manually create it before use if not exist
xls.file.path=/opt/xls
< / code > < / pre > < p > 配 置 告 警 数 据 源 信 息 < / p >
< ul >
< li > alert/data_source.properties < / li >
< / ul >
< pre > < code > #注 意 ： 请 替 换 ${xxx}里 的 内 容
# common configuration
spring.datasource.type=com.alibaba.druid.pool.DruidDataSource
spring.datasource.driver-class-name=com.mysql.jdbc.Driver
spring.datasource.url=jdbc:mysql://${ip}:3306/escheduler?characterEncoding=UTF-8
spring.datasource.username=${username}
spring.datasource.password=${password}
# supplement configuration
spring.datasource.initialSize=5
# min connection number
spring.datasource.minIdle=5
# max connection number
spring.datasource.maxActive=20
# max wait time for get connection
spring.datasource.maxWait=60000
# idle connections closed， unit milliseconds
spring.datasource.timeBetweenEvictionRunsMillis=60000
# connection minimum survival time， unit milliseconds
spring.datasource.minEvictableIdleTimeMillis=300000
spring.datasource.validationQuery=SELECT 1
spring.datasource.validationQueryTimeout=3
spring.datasource.testWhileIdle=true
spring.datasource.testOnBorrow=true
spring.datasource.testOnReturn=false
spring.datasource.defaultAutoCommit=true
# open PSCache， set PSCache size
spring.datasource.poolPreparedStatements=false
spring.datasource.maxPoolPreparedStatementPerConnectionSize=20
< / code > < / pre > < p > 日 志 配 置 文 件 < / p >
< ul >
< li > alert_logback.xml< / li >
< / ul >
< pre > < code > < !-- Logback configuration. See http://logback.qos.ch/manual/index.html -->
< configuration scan=" true" scanPeriod=" 120 seconds" > < !--debug=" true" -->
< property name=" log.base" value=" logs" />
< appender name=" STDOUT" class=" ch.qos.logback.core.ConsoleAppender" >
< encoder>
< pattern>
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
< /pattern>
< charset> UTF-8< /charset>
< /encoder>
< /appender>
< appender name=" ALERTLOGFILE" class=" ch.qos.logback.core.rolling.RollingFileAppender" >
< file> ${log.base}/escheduler-alert.log< /file>
< rollingPolicy class=" ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy" >
< fileNamePattern> ${log.base}/escheduler-alert.%d{yyyy-MM-dd_HH}.%i.log< /fileNamePattern>
< maxHistory> 20< /maxHistory>
< maxFileSize> 64MB< /maxFileSize>
< /rollingPolicy>
< encoder>
< pattern>
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
< /pattern>
< charset> UTF-8< /charset>
< /encoder>
< /appender>
< root level=" INFO" >
< appender-ref ref=" ALERTLOGFILE" />
< /root>
< /configuration>
< / code > < / pre > < h3 id = "escheduler-common" > escheduler-common< / h3 >
< p > 通 用 配 置 文 件 配 置 ， 队 列 选 择 及 地 址 配 置 ， 通 用 文 件 目 录 配 置 。 < / p >
< ul >
< li > common/common.properties< / li >
< / ul >
< pre > < code > #task queue implementation, can choose " redis" or " zookeeper" , default " zookeeper"
escheduler.queue.impl=zookeeper
#if escheduler.queue.impl=redis, you need to configuration relevant information with redis. redis configuration start
spring.redis.host=${redis_ip}
spring.redis.port=6379
spring.redis.maxIdle=1000
spring.redis.maxTotal=10000
#redis configuration end
# user data directory path, self configuration, please make sure the directory exists and have read write permissions
data.basedir.path=/xxx/xxx
# directory path for user data download. self configuration, please make sure the directory exists and have read write permissions
data.download.basedir.path=/xxx/xxx
# process execute directory. self configuration, please make sure the directory exists and have read write permissions
process.exec.basepath=/xxx/xxx
# data base dir, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。 " /escheduler" is recommended
data.store2hdfs.basepath=/escheduler
# system env path. self configuration, please make sure the directory and file exists and have read write execute permissions
escheduler.env.path=/xxx/xxx/.escheduler_env.sh
escheduler.env.py=/xxx/xxx/escheduler_env.py
#resource.view.suffixs
resource.view.suffixs=txt,log,sh,conf,cfg,py,java,sql,hql,xml
# is development state? default " false"
development.state=false
< / code > < / pre > < p > SHELL任 务 环 境 变 量 配 置 < / p >
< p > .escheduler_env.sh < / p >
< pre > < code > #self configuration, please make sure the directory exists and have read write permissions
export HADOOP_HOME=/opt/soft/hadoop
export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop
export SPARK_HOME1=/opt/soft/spark1
export SPARK_HOME2=/opt/soft/spark2
export PYTHON_HOME=/opt/soft/python
export JAVA_HOME=/opt/soft/java
export HIVE_HOME=/opt/soft/hive
export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH
< / code > < / pre > < p > ​ < / p >
< p > Python任 务 环 境 变 量 配 置 < / p >
< p > escheduler_env.py< / p >
< pre > < code > #self configuration, please make sure the directory exists and have read write execute permissions
import os
HADOOP_HOME=" /opt/soft/hadoop"
PYTHON_HOME=" /opt/soft/python"
JAVA_HOME=" /opt/soft/java"
PATH=os.environ[' PATH' ]
PATH=" %s/bin:%s/bin:%s/bin:%s" %(HADOOP_HOME,JAVA_HOME,PYTHON_HOME,PATH)
os.putenv(' PATH' ,' %s' %PATH)
< / code > < / pre > < p > hadoop 配 置 文 件 < / p >
< ul >
< li > common/hadoop/hadoop.properties< / li >
< / ul >
< pre > < code > #please replace the content in ${xxx}
# ha or single namenode
fs.defaultFS=hdfs://${cluster_ipOrName}:8020
#resourcemanager ha note this need ips , eg. 192.168.220.188,192.168.220.189
yarn.resourcemanager.ha.rm.ids=${ip1},${ip2}
# reousrcemanager path
yarn.application.status.address=http://${ip1}:8088/ws/v1/cluster/apps/%s
< / code > < / pre > < p > 定 时 器 配 置 文 件 < / p >
< ul >
< li > quartz.properties< / li >
< / ul >
< pre > < code > #please replace the content in ${xxx}
#============================================================================
# Configure Main Scheduler Properties
#============================================================================
org.quartz.scheduler.instanceName = EasyScheduler
org.quartz.scheduler.instanceId = AUTO
org.quartz.scheduler.makeSchedulerThreadDaemon = true
org.quartz.jobStore.useProperties = false
#============================================================================
# Configure ThreadPool
#============================================================================
org.quartz.threadPool.class = org.quartz.simpl.SimpleThreadPool
org.quartz.threadPool.makeThreadsDaemons = true
org.quartz.threadPool.threadCount = 25
org.quartz.threadPool.threadPriority = 5
#============================================================================
# Configure JobStore
#============================================================================
org.quartz.jobStore.class = org.quartz.impl.jdbcjobstore.JobStoreTX
org.quartz.jobStore.driverDelegateClass = org.quartz.impl.jdbcjobstore.StdJDBCDelegate
org.quartz.jobStore.tablePrefix = QRTZ_
org.quartz.jobStore.isClustered = true
org.quartz.jobStore.misfireThreshold = 60000
org.quartz.jobStore.clusterCheckinInterval = 5000
org.quartz.jobStore.dataSource = myDs
#============================================================================
# Configure Datasources
#============================================================================
org.quartz.dataSource.myDs.driver = com.mysql.jdbc.Driver
org.quartz.dataSource.myDs.URL = jdbc:mysql://${ip}:3306/escheduler?characterEncoding=utf8& useSSL=false
org.quartz.dataSource.myDs.user = ${username}
org.quartz.dataSource.myDs.password = ${password}
org.quartz.dataSource.myDs.maxConnections = 10
org.quartz.dataSource.myDs.validationQuery = select 1
< / code > < / pre > < p > zookeeper 配 置 文 件 < / p >
< ul >
< li > zookeeper.properties< / li >
< / ul >
< pre > < code > #zookeeper cluster. eg. 192.168.220.188:2181,192.168.220.189:2181,192.168.220.190:2181
zookeeper.quorum=${ip1}:2181,${ip2}:2181,${ip3}:2181
#zookeeper server dirctory
zookeeper.escheduler.master=/escheduler/masters
zookeeper.escheduler.worker=/escheduler/workers
#zookeeper lock dirctory
zookeeper.escheduler.lock.master=/escheduler/lock/master
zookeeper.escheduler.lock.worker=/escheduler/lock/worker
#escheduler root directory
zookeeper.escheduler.root=/escheduler
#escheduler failover directory
zookeeper.escheduler.lock.master.failover=/escheduler/lock/failover/master
zookeeper.escheduler.lock.worker.failover=/escheduler/lock/failover/worker
#escheduler failover directory
zookeeper.session.timeout=300
zookeeper.connection.timeout=300
zookeeper.retry.sleep=1000
zookeeper.retry.maxtime=5
< / code > < / pre > < h3 id = "escheduler-dao" > escheduler-dao< / h3 >
< p > dao数 据 源 配 置 < / p >
< ul >
< li > dao/data_source.properties< / li >
< / ul >
< pre > < code > #please replace the content in ${xxx}
# base spring data source configuration
spring.datasource.type=com.alibaba.druid.pool.DruidDataSource
spring.datasource.driver-class-name=com.mysql.jdbc.Driver
spring.datasource.url=jdbc:mysql://${ip}:3306/escheduler?characterEncoding=UTF-8
spring.datasource.username=${username}
spring.datasource.password=${password}
# connection configuration
spring.datasource.initialSize=5
spring.datasource.minIdle=5
spring.datasource.maxActive=20
# max seconds wait connection timeout
spring.datasource.maxWait=60000
# milliseconds for check to close free connections
spring.datasource.timeBetweenEvictionRunsMillis=60000
# connection minimum survival time(milliseconds)
spring.datasource.minEvictableIdleTimeMillis=300000
spring.datasource.validationQuery=SELECT 1
spring.datasource.validationQueryTimeout=3
spring.datasource.testWhileIdle=true
spring.datasource.testOnBorrow=true
spring.datasource.testOnReturn=false
spring.datasource.defaultAutoCommit=true
# open PSCache, specify count PSCache for every connection
spring.datasource.poolPreparedStatements=true
spring.datasource.maxPoolPreparedStatementPerConnectionSize=20
# data quality analysis is not currently in use. please ignore the following configuration
# task record flag
task.record.flag=false
task.record.datasource.url=jdbc:mysql://${ip}:3306/etl?characterEncoding=UTF-8
task.record.datasource.username=etl
task.record.datasource.password=xxxxx
< / code > < / pre > < h3 id = "escheduler-server" > escheduler-server< / h3 >
< p > master配 置 文 件 < / p >
< ul >
< li > master.properties< / li >
< / ul >
< pre > < code > # master execute thread num
master.exec.threads=100
# master execute task number in parallel
master.exec.task.number=20
# master heartbeat interval
master.heartbeat.interval=8
# master commit task retry times
master.task.commit.retryTimes=5
# master commit task interval
master.task.commit.interval=100
# only less than cpu avg load, master server can work. default value : the number of cpu cores * 2
master.max.cpuload.avg=10
# only larger than reserved memory, master server can work. default value : physical memory * 1/10, unit is G.
master.reserved.memory=1
< / code > < / pre > < p > master日 志 文 件 < / p >
< p > < strong > 注 意 ： < / strong > 对 MASTERLOGFILE， 自 定 义 了 MasterLogFilter< / p >
< ul >
< li > master_logback.xml< / li >
< / ul >
< pre > < code > < !-- Logback configuration. See http://logback.qos.ch/manual/index.html -->
< configuration scan=" true" scanPeriod=" 120 seconds" > < !--debug=" true" -->
< property name=" log.base" value=" logs" />
< appender name=" STDOUT" class=" ch.qos.logback.core.ConsoleAppender" >
< encoder>
< pattern>
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
< /pattern>
< charset> UTF-8< /charset>
< /encoder>
< /appender>
< appender name=" MASTERLOGFILE" class=" ch.qos.logback.core.rolling.RollingFileAppender" >
< file> ${log.base}/escheduler-master.log< /file>
< filter class=" cn.escheduler.server.master.log.MasterLogFilter" >
< level> INFO< /level>
< /filter>
< rollingPolicy class=" ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy" >
< fileNamePattern> ${log.base}/escheduler-master.%d{yyyy-MM-dd_HH}.%i.log< /fileNamePattern>
< maxHistory> 20< /maxHistory>
< maxFileSize> 200MB< /maxFileSize>
< /rollingPolicy>
< encoder>
< pattern>
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
< /pattern>
< charset> UTF-8< /charset>
< /encoder>
< /appender>
< root level=" INFO" >
< appender-ref ref=" MASTERLOGFILE" />
< /root>
< /configuration>
< / code > < / pre > < p > worker配 置 文 件 < / p >
< ul >
< li > worker.properties< / li >
< / ul >
< pre > < code > # worker execute thread num
worker.exec.threads=100
# worker heartbeat interval
worker.heartbeat.interval=8
# submit the number of tasks at a time
worker.fetch.task.num = 10
# only less than cpu avg load, worker server can work. default value : the number of cpu cores * 2
worker.max.cpuload.avg=10
# only larger than reserved memory, worker server can work. default value : physical memory * 1/6, unit is G.
worker.reserved.memory=1
< / code > < / pre > < p > worker日 志 文 件 < / p >
< p > < strong > 注 意 ： < / strong > 对 WORKERLOGFILE， 自 定 义 了 WorkerLogFilter< / p >
< p > 对 于 TASKLOGFILE ， 自 定 义 了 TaskLogAppender和 TaskLogFilter< / p >
< ul >
< li > worker_logback.xml< / li >
< / ul >
< pre > < code > < !-- Logback configuration. See http://logback.qos.ch/manual/index.html -->
< configuration scan=" true" scanPeriod=" 120 seconds" >
< property name=" log.base" value=" logs" />
< appender name=" STDOUT" class=" ch.qos.logback.core.ConsoleAppender" >
< encoder>
< pattern>
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
< /pattern>
< charset> UTF-8< /charset>
< /encoder>
< /appender>
< appender name=" TASKLOGFILE" class=" cn.escheduler.server.worker.log.TaskLogAppender" >
< filter class=" ch.qos.logback.classic.filter.ThresholdFilter" >
< level> INFO< /level>
< /filter>
< filter class=" cn.escheduler.server.worker.log.TaskLogFilter" > < /filter>
< file> ${log.base}/{processDefinitionId}/{processInstanceId}/{taskInstanceId}.log< /file>
< encoder>
< pattern>
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
< /pattern>
< charset> UTF-8< /charset>
< /encoder>
< append> true< /append>
< /appender>
< appender name=" WORKERLOGFILE" class=" ch.qos.logback.core.rolling.RollingFileAppender" >
< file> ${log.base}/escheduler-worker.log< /file>
< filter class=" cn.escheduler.server.worker.log.WorkerLogFilter" >
< level> INFO< /level>
< /filter>
< rollingPolicy class=" ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy" >
< fileNamePattern> ${log.base}/escheduler-worker.%d{yyyy-MM-dd_HH}.%i.log< /fileNamePattern>
< maxHistory> 20< /maxHistory>
< maxFileSize> 200MB< /maxFileSize>
< /rollingPolicy>
　 　 　 　 　
< encoder>
< pattern>
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
< /pattern>
< charset> UTF-8< /charset>
< /encoder>
　 　
< /appender>
< root level=" INFO" >
< appender-ref ref=" TASKLOGFILE" />
< appender-ref ref=" WORKERLOGFILE" />
< /root>
< /configuration>
< / code > < / pre > < h3 id = "escheduler-web" > escheduler-web< / h3 >
< p > web配 置 文 件 < / p >
< ul >
< li > application.properties< / li >
< / ul >
< pre > < code > # server port
server.port=12345
# session config
server.session.timeout=7200
server.context-path=/escheduler/
# file size limit for upload
spring.http.multipart.max-file-size=1024MB
spring.http.multipart.max-request-size=1024MB
#post content
server.max-http-post-size=5000000
< / code > < / pre > < p > web日 志 文 件 < / p >
< ul >
< li > webserver_logback.xml< / li >
< / ul >
< pre > < code > < !-- Logback configuration. See http://logback.qos.ch/manual/index.html -->
< configuration scan=" true" scanPeriod=" 120 seconds" >
< logger name=" org.apache.zookeeper" level=" WARN" />
< logger name=" org.apache.hbase" level=" WARN" />
< logger name=" org.apache.hadoop" level=" WARN" />
< property name=" log.base" value=" logs" />
< appender name=" STDOUT" class=" ch.qos.logback.core.ConsoleAppender" >
< encoder>
< pattern>
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
< /pattern>
< charset> UTF-8< /charset>
< /encoder>
< /appender>
< appender name=" WEBSERVERLOGFILE" class=" ch.qos.logback.core.rolling.RollingFileAppender" >
< !-- Log level filter -->
< filter class=" ch.qos.logback.classic.filter.ThresholdFilter" >
< level> INFO< /level>
< /filter>
< file> ${log.base}/escheduler-web-server.log< /file>
< rollingPolicy class=" ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy" >
< fileNamePattern> ${log.base}/escheduler-web-server.%d{yyyy-MM-dd_HH}.%i.log< /fileNamePattern>
< maxHistory> 20< /maxHistory>
< maxFileSize> 64MB< /maxFileSize>
< /rollingPolicy>
< encoder>
< pattern>
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
< /pattern>
< charset> UTF-8< /charset>
< /encoder>
< /appender>
< root level=" INFO" >
< appender-ref ref=" WEBSERVERLOGFILE" />
< /root>
< /configuration>
< / code > < / pre > < h2 id = "启动停止命令" > 启 动 停 止 命 令 < / h2 >
< ul >
< li > 启 停 Master< / li >
< / ul >
< pre > < code > sh ./bin/escheduler-daemon.sh start master-server
sh ./bin/escheduler-daemon.sh stop master-server
< / code > < / pre > < ul >
< li > 启 停 Worker< / li >
< / ul >
< pre > < code > sh ./bin/escheduler-daemon.sh start worker-server
sh ./bin/escheduler-daemon.sh stop worker-server
< / code > < / pre > < ul >
< li > 启 停 Web< / li >
< / ul >
< pre > < code > sh ./bin/escheduler-daemon.sh start web-server
sh ./bin/escheduler-daemon.sh stop web-server
< / code > < / pre > < ul >
< li > 启 停 Logger< pre > < code > sh ./bin/escheduler-daemon.sh start logger-server
sh ./bin/escheduler-daemon.sh stop logger-server
< / code > < / pre > < / li >
< li > 启 停 Alert< pre > < code > sh ./bin/escheduler-daemon.sh start alert-server
sh ./bin/escheduler-daemon.sh stop alert-server
< / code > < / pre > < / li >
< / ul >
< h2 id = "一键启停脚本" > 一 键 启 停 脚 本 < / h2 >
< ul >
< li > < p > 部 署 用 户 配 置 < / p >
< ol >
< li > < p > 创 建 部 署 用 户 < / p >
< p > target/escheduler-{version}-SNAPSHOT/script/init_deploy_user.sh< / p >
< / li >
< li > < p > 配 置 < / p >
< p > 因 为 escheduler worker 都 是 以 sudo -u {linux-user} 方 式 来 执 行 作 业 ， 所 以 部 署 用 户 需 要 有 sudo 权 限 ， 而 且 是 免 密 的 < / p >
< / li >
< / ol >
< / li >
< / ul >
< pre > < code > vi /etc/sudoers
# 部 署 用 户 是 escheduler 账 号
escheduler ALL=(ALL) NOPASSWD: NOPASSWD: ALL
# 并 且 需 要 注 释 掉 Default requiretty 一 行
#Default requiretty
< / code > < / pre > < ul >
< li > 初 始 化 hdfs< / li >
< / ul >
< p > ​ target/escheduler-{version}-SNAPSHOT/script/init_hdfs.sh< / p >
< ul >
< li > 安 装 配 置 文 件 install_config< / li >
< / ul >
< pre > < code > # 项 目 所 在 目 录
BASE_PATH=/opt/soft/program
# 部 署 的 机 器
IPS=ark0,ark1,ark2,ark3,ark4
< / code > < / pre > < ul >
< li > < p > 运 行 配 置 文 件 run_config< / p >
< pre > < code > # master服 务 所 在 机 器 ,> =1个
MASTERS=ark0,ark1
# worker服 务 所 在 机 器 ,> =1个
WORKERS=ark2,ark3,ark4
# alert服 务 所 在 机 器 ,1个
ALERTS=ark3
# web服 务 所 在 机 器 ,1个
WEBSERVER=ark1
< / code > < / pre > < / li >
< li > < p > 初 始 化 安 装 目 录 < / p >
< p > target/escheduler-{version}-SNAPSHOT/script/init_install_path.sh< / p >
< / li >
< li > < p > 将 target/escheduler-{version}-SNAPSHOT 下 配 置 好 的 conf文 件 夹 和 编 译 好 的 escheduler-{version}-SNAPSHOT.tar.gz 复 制 到 < strong > 主 机 器 < / strong > 的 BASE_PATH 目 录 下 < / p >
< p > 说 明 ： < strong > 主 机 器 < / strong > 需 要 能 免 密 ssh登 录 到 其 它 机 器 上 < / p >
< / li >
< li > < p > 启 动 所 有 服 务 < / p >
< / li >
< / ul >
< pre > < code class = "lang-启动所有服务" > sh ./deploy/start_all.sh
< / code > < / pre >
< ul >
< li > 关 闭 所 有 服 务 < / li >
< / ul >
< pre > < code class = "lang-关闭所有服务" > sh ./deploy/stop_all.sh
< / code > < / pre >
< h2 id = "监控服务" > 监 控 服 务 < / h2 >
< p > monitor_server.py 脚 本 是 监 听 ， master和 worker服 务 挂 掉 重 启 的 脚 本 < / p >
< p > 注 意 ： 在 全 部 服 务 都 启 动 之 后 启 动 < / p >
< p > nohup python -u monitor_server.py > nohup.out 2> & 1 & < / p >
< h2 id = "日志查看" > 日 志 查 看 < / h2 >
< p > 日 志 统 一 存 放 于 指 定 文 件 夹 内 < / p >
< pre > < code class = "lang-日志路径" > logs/
├ ─ ─ escheduler-alert-server.log
├ ─ ─ escheduler-master-server.log
|— — escheduler-worker-server.log
|— — escheduler-web-server.log
|— — escheduler-logger-server.log
< / code > < / pre >
< / section >
< / div >
< div class = "search-results" >
< div class = "has-results" >
< h1 class = "search-results-title" > < span class = 'search-results-count' > < / span > results matching "< span class = 'search-query' > < / span > "< / h1 >
< ul class = "search-results-list" > < / ul >
< / div >
< div class = "no-results" >
< h1 class = "search-results-title" > No results matching "< span class = 'search-query' > < / span > "< / h1 >
< / div >
< / div >
< / div >
< / div >
< / div >
< / div >
< a href = "deploy-foreground.html#问题" class = "navigation navigation-prev " aria-label = "Previous page: 问题" >
< i class = "fa fa-angle-left" > < / i >
< / a >
< a href = "deploy-background.html#基础软件安装" class = "navigation navigation-next " aria-label = "Next page: 基础软件安装" >
< i class = "fa fa-angle-right" > < / i >
< / a >
< / div >
< script >
var gitbook = gitbook || [];
gitbook.push(function() {
gitbook.page.hasChanged({"page":{"title":"后端部署","level":"1.3.2","depth":2,"next":{"title":"基础软件安装","level":"1.3.2.1","depth":3,"anchor":"#基础软件安装","path":"pages/deploy-background.md","ref":"pages/deploy-background.md#基础软件安装","articles":[]},"previous":{"title":"问题","level":"1.3.1.5","depth":3,"anchor":"#问题","path":"pages/deploy-foreground.md","ref":"pages/deploy-foreground.md#问题","articles":[]},"dir":"ltr"},"config":{"plugins":["expandable-chapters","insert-logo-link","livereload"],"styles":{"website":"./styles/website.css"},"pluginsConfig":{"livereload":{},"insert-logo-link":{"src":"../images/logo.png","url":"/"},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"fontsettings":{"theme":"white","family":"sans","size":2},"highlight":{},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false},"expandable-chapters":{}},"theme":"default","author":"YIGUAN","pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"variables":{},"title":"调度系统-EasyScheduler","language":"zh-hans","gitbook":"3.2.3","description":"调度系统"},"file":{"path":"pages/deploy-background.md","mtime":"2019-03-07T08:53:26.537Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-03-07T11:30:18.635Z"},"basePath":"..","book":{"language":""}});
});
< / script >
< / div >
< script src = "../gitbook/gitbook.js" > < / script >
< script src = "../gitbook/theme.js" > < / script >
< script src = "../gitbook/gitbook-plugin-expandable-chapters/expandable-chapters.js" > < / script >
< script src = "../gitbook/gitbook-plugin-insert-logo-link/plugin.js" > < / script >
< script src = "../gitbook/gitbook-plugin-livereload/plugin.js" > < / script >
< script src = "../gitbook/gitbook-plugin-search/search-engine.js" > < / script >
< script src = "../gitbook/gitbook-plugin-search/search.js" > < / script >
< script src = "../gitbook/gitbook-plugin-lunr/lunr.min.js" > < / script >
< script src = "../gitbook/gitbook-plugin-lunr/search-lunr.js" > < / script >
< script src = "../gitbook/gitbook-plugin-sharing/buttons.js" > < / script >
< script src = "../gitbook/gitbook-plugin-fontsettings/fontsettings.js" > < / script >
< / body >
< / html >
