You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
610 lines
27 KiB
610 lines
27 KiB
|
|
<!DOCTYPE HTML> |
|
<html lang="zh-hans" > |
|
<head> |
|
<meta charset="UTF-8"> |
|
<meta content="text/html; charset=utf-8" http-equiv="Content-Type"> |
|
<title>Easyscheduler简介 · 调度系统-EasyScheduler</title> |
|
<meta http-equiv="X-UA-Compatible" content="IE=edge" /> |
|
<meta name="description" content=""> |
|
<meta name="generator" content="GitBook 3.2.3"> |
|
<meta name="author" content="YIGUAN"> |
|
|
|
|
|
|
|
<link rel="stylesheet" href="gitbook/style.css"> |
|
|
|
|
|
|
|
|
|
<link rel="stylesheet" href="gitbook/gitbook-plugin-expandable-chapters/expandable-chapters.css"> |
|
|
|
|
|
|
|
<link rel="stylesheet" href="gitbook/gitbook-plugin-insert-logo-link/plugin.css"> |
|
|
|
|
|
|
|
<link rel="stylesheet" href="gitbook/gitbook-plugin-highlight/website.css"> |
|
|
|
|
|
|
|
<link rel="stylesheet" href="gitbook/gitbook-plugin-search/search.css"> |
|
|
|
|
|
|
|
<link rel="stylesheet" href="gitbook/gitbook-plugin-fontsettings/website.css"> |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<link rel="stylesheet" href="styles/website.css"> |
|
|
|
|
|
|
|
|
|
|
|
|
|
<meta name="HandheldFriendly" content="true"/> |
|
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no"> |
|
<meta name="apple-mobile-web-app-capable" content="yes"> |
|
<meta name="apple-mobile-web-app-status-bar-style" content="black"> |
|
<link rel="apple-touch-icon-precomposed" sizes="152x152" href="gitbook/images/apple-touch-icon-precomposed-152.png"> |
|
<link rel="shortcut icon" href="gitbook/images/favicon.ico" type="image/x-icon"> |
|
|
|
|
|
|
|
|
|
</head> |
|
<body> |
|
|
|
<div class="book"> |
|
<div class="book-summary"> |
|
|
|
|
|
<div id="book-search-input" role="search"> |
|
<input type="text" placeholder="输入并搜索" /> |
|
</div> |
|
|
|
|
|
<nav role="navigation"> |
|
|
|
|
|
|
|
<ul class="summary"> |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="chapter active" data-level="1.1" data-path="./"> |
|
|
|
<a href="./"> |
|
|
|
|
|
Easyscheduler简介 |
|
|
|
</a> |
|
|
|
|
|
|
|
</li> |
|
|
|
<li class="chapter " data-level="1.2" > |
|
|
|
<span> |
|
|
|
|
|
前端文档 |
|
|
|
</span> |
|
|
|
|
|
|
|
<ul class="articles"> |
|
|
|
|
|
<li class="chapter " data-level="1.2.1" data-path="frontend-deploy.html"> |
|
|
|
<a href="frontend-deploy.html#前端项目环境构建及编译"> |
|
|
|
|
|
环境搭建 |
|
|
|
</a> |
|
|
|
|
|
|
|
</li> |
|
|
|
<li class="chapter " data-level="1.2.2" data-path="frontend-deploy.html"> |
|
|
|
<a href="frontend-deploy.html#安装及配置"> |
|
|
|
|
|
安装及配置 |
|
|
|
</a> |
|
|
|
|
|
|
|
</li> |
|
|
|
<li class="chapter " data-level="1.2.3" data-path="frontend-deploy.html"> |
|
|
|
<a href="frontend-deploy.html#项目生产环境配置"> |
|
|
|
|
|
项目生产环境Nginx配置 |
|
|
|
</a> |
|
|
|
|
|
|
|
</li> |
|
|
|
<li class="chapter " data-level="1.2.4" data-path="frontend-deploy.html"> |
|
|
|
<a href="frontend-deploy.html#前端项目发布"> |
|
|
|
|
|
前端项目发布 |
|
|
|
</a> |
|
|
|
|
|
|
|
</li> |
|
|
|
<li class="chapter " data-level="1.2.5" data-path="frontend-deploy.html"> |
|
|
|
<a href="frontend-deploy.html#问题"> |
|
|
|
|
|
问题 |
|
|
|
</a> |
|
|
|
|
|
|
|
</li> |
|
|
|
<li class="chapter " data-level="1.2.6" data-path="frontend-development.html"> |
|
|
|
<a href="frontend-development.html#项目目录结构"> |
|
|
|
|
|
项目目录结构 |
|
|
|
</a> |
|
|
|
|
|
|
|
</li> |
|
|
|
<li class="chapter " data-level="1.2.7" data-path="frontend-development.html"> |
|
|
|
<a href="frontend-development.html#系统功能模块"> |
|
|
|
|
|
系统功能模块 |
|
|
|
</a> |
|
|
|
|
|
|
|
</li> |
|
|
|
<li class="chapter " data-level="1.2.8" data-path="frontend-development.html"> |
|
|
|
<a href="frontend-development.html#路由和状态管理"> |
|
|
|
|
|
路由和状态管理 |
|
|
|
</a> |
|
|
|
|
|
|
|
</li> |
|
|
|
<li class="chapter " data-level="1.2.9" data-path="frontend-development.html"> |
|
|
|
<a href="frontend-development.html#规范"> |
|
|
|
|
|
规范 |
|
|
|
</a> |
|
|
|
|
|
|
|
</li> |
|
|
|
<li class="chapter " data-level="1.2.10" data-path="frontend-development.html"> |
|
|
|
<a href="frontend-development.html#接口"> |
|
|
|
|
|
接口 |
|
|
|
</a> |
|
|
|
|
|
|
|
</li> |
|
|
|
<li class="chapter " data-level="1.2.11" data-path="frontend-development.html"> |
|
|
|
<a href="frontend-development.html#扩展开发"> |
|
|
|
|
|
扩展开发 |
|
|
|
</a> |
|
|
|
|
|
|
|
</li> |
|
|
|
|
|
</ul> |
|
|
|
</li> |
|
|
|
<li class="chapter " data-level="1.3" > |
|
|
|
<span> |
|
|
|
|
|
后端文档 |
|
|
|
</span> |
|
|
|
|
|
|
|
<ul class="articles"> |
|
|
|
|
|
<li class="chapter " data-level="1.3.1" data-path="architecture-design.html"> |
|
|
|
<a href="architecture-design.html#调度系统架构设计"> |
|
|
|
|
|
系统架构设计 |
|
|
|
</a> |
|
|
|
|
|
|
|
</li> |
|
|
|
<li class="chapter " data-level="1.3.2" data-path="backend-deploy.html"> |
|
|
|
<a href="backend-deploy.html#部署文档"> |
|
|
|
|
|
部署文档 |
|
|
|
</a> |
|
|
|
|
|
|
|
</li> |
|
|
|
<li class="chapter " data-level="1.3.3" data-path="task-plugin-development.html"> |
|
|
|
<a href="task-plugin-development.html#任务插件开发"> |
|
|
|
|
|
自定义任务插件文档 |
|
|
|
</a> |
|
|
|
|
|
|
|
</li> |
|
|
|
|
|
</ul> |
|
|
|
</li> |
|
|
|
<li class="chapter " data-level="1.4" data-path="user-manual.md"> |
|
|
|
<span> |
|
|
|
|
|
使用说明文档 |
|
|
|
</a> |
|
|
|
|
|
|
|
</li> |
|
|
|
|
|
|
|
|
|
<li class="divider"></li> |
|
|
|
<li> |
|
<a href="https://www.gitbook.com" target="blank" class="gitbook-link"> |
|
本书使用 GitBook 发布 |
|
</a> |
|
</li> |
|
</ul> |
|
|
|
|
|
</nav> |
|
|
|
|
|
</div> |
|
|
|
<div class="book-body"> |
|
|
|
<div class="body-inner"> |
|
|
|
|
|
|
|
<div class="book-header" role="navigation"> |
|
|
|
|
|
<!-- Title --> |
|
<h1> |
|
<i class="fa fa-circle-o-notch fa-spin"></i> |
|
<a href="." >Easyscheduler简介</a> |
|
</h1> |
|
</div> |
|
|
|
|
|
|
|
|
|
<div class="page-wrapper" tabindex="-1" role="main"> |
|
<div class="page-inner"> |
|
|
|
<div id="book-search-results"> |
|
<div class="search-noresults"> |
|
|
|
<section class="normal markdown-section"> |
|
|
|
<h3 id="easy-scheduler">Easy Scheduler</h3> |
|
<p><a href="https://www.apache.org/licenses/LICENSE-2.0.html" target="_blank"><img src="https://img.shields.io/badge/license-Apache%202-4EB1BA.svg" alt="License"></a></p> |
|
<blockquote> |
|
<p>Easy Scheduler for Big Data</p> |
|
</blockquote> |
|
<p><strong>设计特点：</strong> 一个分布式易扩展的可视化DAG工作流任务调度系统。致力于解决数据处理流程中错综复杂的依赖关系，使调度系统在数据处理流程中<code>开箱即用</code>。 |
|
其主要目标如下：</p> |
|
<ul> |
|
<li>以DAG图的方式将Task按照任务的依赖关系关联起来，可实时可视化监控任务的运行状态</li> |
|
<li>支持丰富的任务类型：Shell、MR、Spark、SQL(mysql、postgresql、hive、sparksql),Python,Sub_Process、Procedure等</li> |
|
<li>支持工作流定时调度、依赖调度、手动调度、手动暂停/停止/恢复，同时支持失败重试/告警、从指定节点恢复失败、Kill任务等操作</li> |
|
<li>支持工作流优先级、任务优先级及任务的故障转移及任务超时告警/失败</li> |
|
<li>支持工作流全局参数及节点自定义参数设置</li> |
|
<li>支持资源文件的在线上传/下载，管理等，支持在线文件创建、编辑</li> |
|
<li>支持任务日志在线查看及滚动、在线下载日志等</li> |
|
<li>实现集群HA，通过Zookeeper实现Master集群和Worker集群去中心化</li> |
|
<li>支持对<code>Master/Worker</code> cpu load，memory，cpu在线查看</li> |
|
<li>支持工作流运行历史树形/甘特图展示、支持任务状态统计、流程状态统计</li> |
|
<li>支持补数</li> |
|
<li>支持多租户</li> |
|
<li>支持国际化</li> |
|
<li>还有更多等待伙伴们探索</li> |
|
</ul> |
|
<h3 id="与同类调度系统的对比">与同类调度系统的对比</h3> |
|
<style> |
|
table th:first-of-type { |
|
width: 200px; |
|
} |
|
</style> |
|
|
|
<table> |
|
<thead> |
|
<tr> |
|
<th style="text-align:left"></th> |
|
<th style="text-align:left">EasyScheduler</th> |
|
<th style="text-align:left">Azkaban</th> |
|
<th style="text-align:left">Airflow</th> |
|
</tr> |
|
</thead> |
|
<tbody> |
|
<tr> |
|
<td style="text-align:left"><h6 id="稳定性">稳定性</h6></td> |
|
<td style="text-align:left"></td> |
|
<td style="text-align:left"></td> |
|
<td style="text-align:left"></td> |
|
</tr> |
|
<tr> |
|
<td style="text-align:left">单点故障</td> |
|
<td style="text-align:left">去中心化的多Master和多Worker</td> |
|
<td style="text-align:left">是 <br> 单个Web和调度程序组合</td> |
|
<td style="text-align:left">是<br> 单一调度程序</td> |
|
</tr> |
|
<tr> |
|
<td style="text-align:left">HA额外要求</td> |
|
<td style="text-align:left">不需要(本身就支持HA)</td> |
|
<td style="text-align:left">DB</td> |
|
<td style="text-align:left">Celery / Dask / Mesos + Load Balancer + DB</td> |
|
</tr> |
|
<tr> |
|
<td style="text-align:left">过载处理</td> |
|
<td style="text-align:left">任务队列机制，单个机器上可调度的任务数量可以灵活配置，当任务过多时会缓存在任务队列中，不会造成机器卡死</td> |
|
<td style="text-align:left">任务太多时会卡死服务器</td> |
|
<td style="text-align:left">任务太多时会卡死服务器</td> |
|
</tr> |
|
<tr> |
|
<td style="text-align:left"><h6 id="易用性">易用性</h6></td> |
|
<td style="text-align:left"></td> |
|
<td style="text-align:left"></td> |
|
<td style="text-align:left"></td> |
|
</tr> |
|
<tr> |
|
<td style="text-align:left">DAG监控界面</td> |
|
<td style="text-align:left">任务状态、任务类型、重试次数、任务运行机器、可视化变量等关键信息一目了然</td> |
|
<td style="text-align:left">只能看到任务状态</td> |
|
<td style="text-align:left">不能直观区分任务类型</td> |
|
</tr> |
|
<tr> |
|
<td style="text-align:left">可视化流程定义</td> |
|
<td style="text-align:left">是 <br> 所有流程定义操作都是可视化的，通过拖拽任务来绘制DAG,配置数据源及资源。同时对于第三方系统，提供api方式的操作。</td> |
|
<td style="text-align:left">否 <br> 通过自定义DSL绘制DAG并打包上传</td> |
|
<td style="text-align:left">否 <br> 通过python代码来绘制DAG，使用不便，特别是对不会写代码的业务人员基本无法使用。</td> |
|
</tr> |
|
<tr> |
|
<td style="text-align:left">快速部署</td> |
|
<td style="text-align:left">一键部署</td> |
|
<td style="text-align:left">集群化部署复杂</td> |
|
<td style="text-align:left">集群化部署复杂</td> |
|
</tr> |
|
<tr> |
|
<td style="text-align:left"><h6 id="功能">功能</h6></td> |
|
<td style="text-align:left"></td> |
|
<td style="text-align:left"></td> |
|
<td style="text-align:left"></td> |
|
</tr> |
|
<tr> |
|
<td style="text-align:left">是否能暂停和恢复</td> |
|
<td style="text-align:left">支持暂停，恢复操作</td> |
|
<td style="text-align:left">否 <br> 需将工作流杀死再运行</td> |
|
<td style="text-align:left">否 <br> 需将工作流杀死再运行</td> |
|
</tr> |
|
<tr> |
|
<td style="text-align:left">是否支持多租户</td> |
|
<td style="text-align:left">支持 <br> easyscheduler上的用户可以通过租户和hadoop用户实现多对一或一对一的映射关系，这对大数据作业的调度是非常重要的。</td> |
|
<td style="text-align:left">否</td> |
|
<td style="text-align:left">否</td> |
|
</tr> |
|
<tr> |
|
<td style="text-align:left">任务类型</td> |
|
<td style="text-align:left">支持传统的shell任务，同时支持大数据平台任务调度： MR、Spark、SQL(mysql、postgresql、hive、sparksql)、Python、Procedure、Sub_Process</td> |
|
<td style="text-align:left">shell、gobblin、hadoopJava、java、hive、pig、spark、hdfsToTeradata、teradataToHdfs</td> |
|
<td style="text-align:left">BashOperator、DummyOperator、MySqlOperator、HiveOperator、EmailOperator、HTTPOperator、SqlOperator</td> |
|
</tr> |
|
<tr> |
|
<td style="text-align:left">契合度</td> |
|
<td style="text-align:left">支持大数据作业spark,hive,mr的调度，同时由于支持多租户，与大数据业务更加契合</td> |
|
<td style="text-align:left">由于不支持多租户，在大数据平台业务使用不够灵活</td> |
|
<td style="text-align:left">由于不支持多租户，在大数据平台业务使用不够灵活</td> |
|
</tr> |
|
<tr> |
|
<td style="text-align:left"><h6 id="扩展性">扩展性</h6></td> |
|
<td style="text-align:left"></td> |
|
<td style="text-align:left"></td> |
|
<td style="text-align:left"></td> |
|
</tr> |
|
<tr> |
|
<td style="text-align:left"><center>是否支持自定义任务类型</center></td> |
|
<td style="text-align:left">是</td> |
|
<td style="text-align:left">是</td> |
|
<td style="text-align:left">是</td> |
|
</tr> |
|
<tr> |
|
<td style="text-align:left">是否支持集群扩展</td> |
|
<td style="text-align:left">是 <br> 调度器使用分布式调度，整体的调度能力会随便集群的规模线性增长，Master和Worker支持动态上下线</td> |
|
<td style="text-align:left">是，但是复杂 <br> Executor水平扩展</td> |
|
<td style="text-align:left">是，但是复杂 <br> Executor水平扩展</td> |
|
</tr> |
|
</tbody> |
|
</table> |
|
<h3 id="系统部分截图">系统部分截图</h3> |
|
<p><img src="http://geek.analysys.cn/static/upload/47/2019-03-06/76db3013-8e3b-4d17-b167-2aa1e6a6b0ad.jpeg" alt=""></p> |
|
<p><img src="http://geek.analysys.cn/static/upload/47/2019-03-06/08b79a19-4aa0-4a73-a71b-81ad210513fb.jpeg" alt=""></p> |
|
<p><img src="http://geek.analysys.cn/static/upload/47/2019-03-06/384dd8a3-4cf8-4e3e-944d-1185ba198f75.jpeg" alt=""></p> |
|
<h3 id="文档">文档</h3> |
|
<ul> |
|
<li>部署文档 |
|
<a href="https://analysys.github.io/EasyScheduler/pages/deploy-background.html" target="_blank">后端部署文档</a></li> |
|
</ul> |
|
<p><a href="https://analysys.github.io/EasyScheduler/pages/deploy-foreground.html" target="_blank">前端部署文档</a></p> |
|
<p><a href="https://analysys.github.io/EasyScheduler/pages/guide-manual.html?_blank" title="使用手册" target="_blank"><strong>使用手册</strong></a> </p> |
|
<p>更多文档请参考 XXX</p> |
|
<h3 id="帮助">帮助</h3> |
|
<p>The fastest way to get response from our developers is to submit issues, or add our wechat : 510570367</p> |
|
|
|
|
|
</section> |
|
|
|
</div> |
|
<div class="search-results"> |
|
<div class="has-results"> |
|
|
|
<h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1> |
|
<ul class="search-results-list"></ul> |
|
|
|
</div> |
|
<div class="no-results"> |
|
|
|
<h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1> |
|
|
|
</div> |
|
</div> |
|
</div> |
|
|
|
</div> |
|
</div> |
|
|
|
</div> |
|
|
|
|
|
|
|
|
|
|
|
|
|
</div> |
|
|
|
<script> |
|
var gitbook = gitbook || []; |
|
gitbook.push(function() { |
|
gitbook.page.hasChanged({"page":{"title":"Easyscheduler简介","level":"1.1","depth":1,"next":{"title":"前端文档","level":"1.2","depth":1,"ref":"","articles":[{"title":"环境搭建","level":"1.2.1","depth":2,"anchor":"#前端项目环境构建及编译","path":"frontend-deploy.md","ref":"frontend-deploy.md#前端项目环境构建及编译","articles":[]},{"title":"安装及配置","level":"1.2.2","depth":2,"anchor":"#安装及配置","path":"frontend-deploy.md","ref":"frontend-deploy.md#安装及配置","articles":[]},{"title":"项目生产环境Nginx配置","level":"1.2.3","depth":2,"anchor":"#项目生产环境配置","path":"frontend-deploy.md","ref":"frontend-deploy.md#项目生产环境配置","articles":[]},{"title":"前端项目发布","level":"1.2.4","depth":2,"anchor":"#前端项目发布","path":"frontend-deploy.md","ref":"frontend-deploy.md#前端项目发布","articles":[]},{"title":"问题","level":"1.2.5","depth":2,"anchor":"#问题","path":"frontend-deploy.md","ref":"frontend-deploy.md#问题","articles":[]},{"title":"项目目录结构","level":"1.2.6","depth":2,"anchor":"#项目目录结构","path":"frontend-development.md","ref":"frontend-development.md#项目目录结构","articles":[]},{"title":"系统功能模块","level":"1.2.7","depth":2,"anchor":"#系统功能模块","path":"frontend-development.md","ref":"frontend-development.md#系统功能模块","articles":[]},{"title":"路由和状态管理","level":"1.2.8","depth":2,"anchor":"#路由和状态管理","path":"frontend-development.md","ref":"frontend-development.md#路由和状态管理","articles":[]},{"title":"规范","level":"1.2.9","depth":2,"anchor":"#规范","path":"frontend-development.md","ref":"frontend-development.md#规范","articles":[]},{"title":"接口","level":"1.2.10","depth":2,"anchor":"#接口","path":"frontend-development.md","ref":"frontend-development.md#接口","articles":[]},{"title":"扩展开发","level":"1.2.11","depth":2,"anchor":"#扩展开发","path":"frontend-development.md","ref":"frontend-development.md#扩展开发","articles":[]}]},"dir":"ltr"},"config":{"plugins":["expandable-chapters","insert-logo-link"],"styles":{"website":"./styles/website.css"},"pluginsConfig":{"insert-logo-link":{"src":"../images/logo.png","url":"/"},"expandable-chapters":{},"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"theme":"default","author":"YIGUAN","pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"variables":{},"title":"调度系统-EasyScheduler","language":"zh-hans","gitbook":"3.2.3","description":"调度系统"},"file":{"path":"README.md","mtime":"2019-03-28T10:57:25.346Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-03-28T11:15:13.750Z"},"basePath":".","book":{"language":""}}); |
|
}); |
|
</script> |
|
</div> |
|
|
|
|
|
<script src="gitbook/gitbook.js"></script> |
|
<script src="gitbook/theme.js"></script> |
|
|
|
|
|
<script src="gitbook/gitbook-plugin-expandable-chapters/expandable-chapters.js"></script> |
|
|
|
|
|
|
|
<script src="gitbook/gitbook-plugin-insert-logo-link/plugin.js"></script> |
|
|
|
|
|
|
|
<script src="gitbook/gitbook-plugin-search/search-engine.js"></script> |
|
|
|
|
|
|
|
<script src="gitbook/gitbook-plugin-search/search.js"></script> |
|
|
|
|
|
|
|
<script src="gitbook/gitbook-plugin-lunr/lunr.min.js"></script> |
|
|
|
|
|
|
|
<script src="gitbook/gitbook-plugin-lunr/search-lunr.js"></script> |
|
|
|
|
|
|
|
<script src="gitbook/gitbook-plugin-sharing/buttons.js"></script> |
|
|
|
|
|
|
|
<script src="gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script> |
|
|
|
|
|
|
|
</body> |
|
</html> |
|
|
|
|