dailidong
4 years ago
committed by
GitHub
111 changed files with 2776 additions and 3817 deletions
@ -0,0 +1,68 @@ |
|||||||
|
### Dolphin Scheduler的Ambari插件使用说明 |
||||||
|
|
||||||
|
##### 备注 |
||||||
|
|
||||||
|
1. 本文档适用于对Ambari中基本了解的用户 |
||||||
|
2. 本文档是对已安装Ambari服务添加Dolphin Scheduler(1.3.0版本)服务的说明 |
||||||
|
|
||||||
|
##### 一 安装准备 |
||||||
|
|
||||||
|
1. 准备RPM包 |
||||||
|
|
||||||
|
- 在源码dolphinscheduler-dist目录下执行命令```mvn -U clean install rpm:attached-rpm -Prpmbuild -Dmaven.test.skip=true -X```即可生成(在目录 dolphinscheduler-dist/target/rpm/apache-dolphinscheduler-incubating/RPMS/noarch 下) |
||||||
|
|
||||||
|
2. 创建DS的安装用户--权限 |
||||||
|
|
||||||
|
3. 初始化数据库信息 |
||||||
|
|
||||||
|
``` |
||||||
|
-- 创建Dolphin Scheduler的数据库：dolphinscheduler |
||||||
|
CREATE DATABASE dolphinscheduler DEFAULT CHARACTER SET utf8 DEFAULT COLLATE |
||||||
|
utf8_general_ci; |
||||||
|
|
||||||
|
-- 初始化dolphinscheduler数据库的用户和密码，并分配权限 |
||||||
|
-- 替换下面sql语句中的{user}为dolphinscheduler数据库的用户 |
||||||
|
GRANT ALL PRIVILEGES ON dolphinscheduler.* TO '{user}'@'%' IDENTIFIED BY '{password}'; |
||||||
|
GRANT ALL PRIVILEGES ON dolphinscheduler.* TO '{user}'@'localhost' IDENTIFIED BY |
||||||
|
'{password}'; |
||||||
|
flush privileges; |
||||||
|
``` |
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
##### 二 Ambari安装Dolphin Scheduler |
||||||
|
|
||||||
|
1. Ambari界面安装Dolphin Scheduler |
||||||
|
|
||||||
|
 |
||||||
|
|
||||||
|
2. 选择Dolphin Scheduler的Master安装的节点 |
||||||
|
|
||||||
|
 |
||||||
|
|
||||||
|
3. 配置Dolphin Scheduler的Worker、Api、Logger、Alert安装的节点 |
||||||
|
|
||||||
|
 |
||||||
|
|
||||||
|
4. 设置Dolphin Scheduler服务的安装用户（**步骤一中创建的**）及所属的用户组 |
||||||
|
|
||||||
|
 |
||||||
|
|
||||||
|
5. 配置数据库的信息（和步骤一中初始化数据库中一致） |
||||||
|
|
||||||
|
 |
||||||
|
|
||||||
|
6. 配置其它的信息--如果需要的话 |
||||||
|
|
||||||
|
 |
||||||
|
|
||||||
|
 |
||||||
|
|
||||||
|
7. 正常执行接下来的步骤 |
||||||
|
|
||||||
|
 |
||||||
|
|
||||||
|
8. 安装成功后的界面 |
||||||
|
|
||||||
|
 |
||||||
|
|
||||||
@ -1,164 +0,0 @@ |
|||||||
{ |
|
||||||
"DOLPHIN": { |
|
||||||
"service": [], |
|
||||||
"DOLPHIN_API": [ |
|
||||||
{ |
|
||||||
"name": "dolphin_api_port_check", |
|
||||||
"label": "dolphin_api_port_check", |
|
||||||
"description": "dolphin_api_port_check.", |
|
||||||
"interval": 10, |
|
||||||
"scope": "ANY", |
|
||||||
"source": { |
|
||||||
"type": "PORT", |
|
||||||
"uri": "{{dolphin-application-api/server.port}}", |
|
||||||
"default_port": 12345, |
|
||||||
"reporting": { |
|
||||||
"ok": { |
|
||||||
"text": "TCP OK - {0:.3f}s response on port {1}" |
|
||||||
}, |
|
||||||
"warning": { |
|
||||||
"text": "TCP OK - {0:.3f}s response on port {1}", |
|
||||||
"value": 1.5 |
|
||||||
}, |
|
||||||
"critical": { |
|
||||||
"text": "Connection failed: {0} to {1}:{2}", |
|
||||||
"value": 5.0 |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
], |
|
||||||
"DOLPHIN_MASTER": [ |
|
||||||
{ |
|
||||||
"name": "DOLPHIN_MASTER_CHECK", |
|
||||||
"label": "check dolphin scheduler master status", |
|
||||||
"description": "", |
|
||||||
"interval":10, |
|
||||||
"scope": "HOST", |
|
||||||
"enabled": true, |
|
||||||
"source": { |
|
||||||
"type": "SCRIPT", |
|
||||||
"path": "DOLPHIN/2.0.0/package/alerts/alert_dolphin_scheduler_status.py", |
|
||||||
"parameters": [ |
|
||||||
|
|
||||||
{ |
|
||||||
"name": "connection.timeout", |
|
||||||
"display_name": "Connection Timeout", |
|
||||||
"value": 5.0, |
|
||||||
"type": "NUMERIC", |
|
||||||
"description": "The maximum time before this alert is considered to be CRITICAL", |
|
||||||
"units": "seconds", |
|
||||||
"threshold": "CRITICAL" |
|
||||||
}, |
|
||||||
{ |
|
||||||
"name": "alertName", |
|
||||||
"display_name": "alertName", |
|
||||||
"value": "DOLPHIN_MASTER", |
|
||||||
"type": "STRING", |
|
||||||
"description": "alert name" |
|
||||||
} |
|
||||||
] |
|
||||||
} |
|
||||||
} |
|
||||||
], |
|
||||||
"DOLPHIN_WORKER": [ |
|
||||||
{ |
|
||||||
"name": "DOLPHIN_WORKER_CHECK", |
|
||||||
"label": "check dolphin scheduler worker status", |
|
||||||
"description": "", |
|
||||||
"interval":10, |
|
||||||
"scope": "HOST", |
|
||||||
"enabled": true, |
|
||||||
"source": { |
|
||||||
"type": "SCRIPT", |
|
||||||
"path": "DOLPHIN/2.0.0/package/alerts/alert_dolphin_scheduler_status.py", |
|
||||||
"parameters": [ |
|
||||||
|
|
||||||
{ |
|
||||||
"name": "connection.timeout", |
|
||||||
"display_name": "Connection Timeout", |
|
||||||
"value": 5.0, |
|
||||||
"type": "NUMERIC", |
|
||||||
"description": "The maximum time before this alert is considered to be CRITICAL", |
|
||||||
"units": "seconds", |
|
||||||
"threshold": "CRITICAL" |
|
||||||
}, |
|
||||||
{ |
|
||||||
"name": "alertName", |
|
||||||
"display_name": "alertName", |
|
||||||
"value": "DOLPHIN_WORKER", |
|
||||||
"type": "STRING", |
|
||||||
"description": "alert name" |
|
||||||
} |
|
||||||
] |
|
||||||
} |
|
||||||
} |
|
||||||
], |
|
||||||
"DOLPHIN_ALERT": [ |
|
||||||
{ |
|
||||||
"name": "DOLPHIN_DOLPHIN_ALERT_CHECK", |
|
||||||
"label": "check dolphin scheduler alert status", |
|
||||||
"description": "", |
|
||||||
"interval":10, |
|
||||||
"scope": "HOST", |
|
||||||
"enabled": true, |
|
||||||
"source": { |
|
||||||
"type": "SCRIPT", |
|
||||||
"path": "DOLPHIN/2.0.0/package/alerts/alert_dolphin_scheduler_status.py", |
|
||||||
"parameters": [ |
|
||||||
|
|
||||||
{ |
|
||||||
"name": "connection.timeout", |
|
||||||
"display_name": "Connection Timeout", |
|
||||||
"value": 5.0, |
|
||||||
"type": "NUMERIC", |
|
||||||
"description": "The maximum time before this alert is considered to be CRITICAL", |
|
||||||
"units": "seconds", |
|
||||||
"threshold": "CRITICAL" |
|
||||||
}, |
|
||||||
{ |
|
||||||
"name": "alertName", |
|
||||||
"display_name": "alertName", |
|
||||||
"value": "DOLPHIN_ALERT", |
|
||||||
"type": "STRING", |
|
||||||
"description": "alert name" |
|
||||||
} |
|
||||||
] |
|
||||||
} |
|
||||||
} |
|
||||||
], |
|
||||||
"DOLPHIN_ALERT": [ |
|
||||||
{ |
|
||||||
"name": "DOLPHIN_DOLPHIN_LOGGER_CHECK", |
|
||||||
"label": "check dolphin scheduler alert status", |
|
||||||
"description": "", |
|
||||||
"interval":10, |
|
||||||
"scope": "HOST", |
|
||||||
"enabled": true, |
|
||||||
"source": { |
|
||||||
"type": "SCRIPT", |
|
||||||
"path": "DOLPHIN/2.0.0/package/alerts/alert_dolphin_scheduler_status.py", |
|
||||||
"parameters": [ |
|
||||||
|
|
||||||
{ |
|
||||||
"name": "connection.timeout", |
|
||||||
"display_name": "Connection Timeout", |
|
||||||
"value": 5.0, |
|
||||||
"type": "NUMERIC", |
|
||||||
"description": "The maximum time before this alert is considered to be CRITICAL", |
|
||||||
"units": "seconds", |
|
||||||
"threshold": "CRITICAL" |
|
||||||
}, |
|
||||||
{ |
|
||||||
"name": "alertName", |
|
||||||
"display_name": "alertName", |
|
||||||
"value": "DOLPHIN_LOGGER", |
|
||||||
"type": "STRING", |
|
||||||
"description": "alert name" |
|
||||||
} |
|
||||||
] |
|
||||||
} |
|
||||||
} |
|
||||||
] |
|
||||||
} |
|
||||||
} |
|
||||||
@ -1,143 +0,0 @@ |
|||||||
<!-- |
|
||||||
~ Licensed to the Apache Software Foundation (ASF) under one or more |
|
||||||
~ contributor license agreements. See the NOTICE file distributed with |
|
||||||
~ this work for additional information regarding copyright ownership. |
|
||||||
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
|
||||||
~ (the "License"); you may not use this file except in compliance with |
|
||||||
~ the License. You may obtain a copy of the License at |
|
||||||
~ |
|
||||||
~ http://www.apache.org/licenses/LICENSE-2.0 |
|
||||||
~ |
|
||||||
~ Unless required by applicable law or agreed to in writing, software |
|
||||||
~ distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
~ See the License for the specific language governing permissions and |
|
||||||
~ limitations under the License. |
|
||||||
--> |
|
||||||
<configuration> |
|
||||||
<property> |
|
||||||
<name>alert.type</name> |
|
||||||
<value>EMAIL</value> |
|
||||||
<description>alert type is EMAIL/SMS</description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>alert.template</name> |
|
||||||
<value>html</value> |
|
||||||
<description>alter msg template, default is html template</description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>mail.protocol</name> |
|
||||||
<value>SMTP</value> |
|
||||||
<description></description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>mail.server.host</name> |
|
||||||
<value>xxx.xxx.com</value> |
|
||||||
<description></description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>mail.server.port</name> |
|
||||||
<value>25</value> |
|
||||||
<value-attributes> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
<description></description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>mail.sender</name> |
|
||||||
<value>admin</value> |
|
||||||
<description></description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>mail.user</name> |
|
||||||
<value>admin</value> |
|
||||||
<description></description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>mail.passwd</name> |
|
||||||
<value>000000</value> |
|
||||||
<description></description> |
|
||||||
<property-type>PASSWORD</property-type> |
|
||||||
<value-attributes> |
|
||||||
<type>password</type> |
|
||||||
</value-attributes> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
|
|
||||||
<property> |
|
||||||
<name>mail.smtp.starttls.enable</name> |
|
||||||
<value>true</value> |
|
||||||
<value-attributes> |
|
||||||
<type>boolean</type> |
|
||||||
</value-attributes> |
|
||||||
<description></description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>mail.smtp.ssl.enable</name> |
|
||||||
<value>true</value> |
|
||||||
<value-attributes> |
|
||||||
<type>boolean</type> |
|
||||||
</value-attributes> |
|
||||||
<description></description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>mail.smtp.ssl.trust</name> |
|
||||||
<value>xxx.xxx.com</value> |
|
||||||
<description></description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
|
|
||||||
<property> |
|
||||||
<name>enterprise.wechat.enable</name> |
|
||||||
<value>false</value> |
|
||||||
<description></description> |
|
||||||
<value-attributes> |
|
||||||
<type>value-list</type> |
|
||||||
<entries> |
|
||||||
<entry> |
|
||||||
<value>true</value> |
|
||||||
<label>Enabled</label> |
|
||||||
</entry> |
|
||||||
<entry> |
|
||||||
<value>false</value> |
|
||||||
<label>Disabled</label> |
|
||||||
</entry> |
|
||||||
</entries> |
|
||||||
<selection-cardinality>1</selection-cardinality> |
|
||||||
</value-attributes> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>enterprise.wechat.corp.id</name> |
|
||||||
<value>wechatId</value> |
|
||||||
<description></description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>enterprise.wechat.secret</name> |
|
||||||
<value>secret</value> |
|
||||||
<description></description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>enterprise.wechat.agent.id</name> |
|
||||||
<value>agentId</value> |
|
||||||
<description></description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>enterprise.wechat.users</name> |
|
||||||
<value>wechatUsers</value> |
|
||||||
<description></description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
</configuration> |
|
||||||
@ -1,87 +0,0 @@ |
|||||||
<!-- |
|
||||||
~ Licensed to the Apache Software Foundation (ASF) under one or more |
|
||||||
~ contributor license agreements. See the NOTICE file distributed with |
|
||||||
~ this work for additional information regarding copyright ownership. |
|
||||||
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
|
||||||
~ (the "License"); you may not use this file except in compliance with |
|
||||||
~ the License. You may obtain a copy of the License at |
|
||||||
~ |
|
||||||
~ http://www.apache.org/licenses/LICENSE-2.0 |
|
||||||
~ |
|
||||||
~ Unless required by applicable law or agreed to in writing, software |
|
||||||
~ distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
~ See the License for the specific language governing permissions and |
|
||||||
~ limitations under the License. |
|
||||||
--> |
|
||||||
<configuration> |
|
||||||
<property> |
|
||||||
<name>server.port</name> |
|
||||||
<value>12345</value> |
|
||||||
<description> |
|
||||||
server port |
|
||||||
</description> |
|
||||||
<value-attributes> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>server.servlet.session.timeout</name> |
|
||||||
<value>7200</value> |
|
||||||
<value-attributes> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
<description> |
|
||||||
</description> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>server.servlet.context-path</name> |
|
||||||
<value>/dolphinscheduler/</value> |
|
||||||
<description> |
|
||||||
</description> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>spring.servlet.multipart.max-file-size</name> |
|
||||||
<value>1024</value> |
|
||||||
<value-attributes> |
|
||||||
<unit>MB</unit> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
<description> |
|
||||||
</description> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>spring.servlet.multipart.max-request-size</name> |
|
||||||
<value>1024</value> |
|
||||||
<value-attributes> |
|
||||||
<unit>MB</unit> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
<description> |
|
||||||
</description> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>server.jetty.max-http-post-size</name> |
|
||||||
<value>5000000</value> |
|
||||||
<value-attributes> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
<description> |
|
||||||
</description> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>spring.messages.encoding</name> |
|
||||||
<value>UTF-8</value> |
|
||||||
<description></description> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>spring.messages.basename</name> |
|
||||||
<value>i18n/messages</value> |
|
||||||
<description></description> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>security.authentication.type</name> |
|
||||||
<value>PASSWORD</value> |
|
||||||
<description></description> |
|
||||||
</property> |
|
||||||
</configuration> |
|
||||||
@ -1,158 +0,0 @@ |
|||||||
<!-- |
|
||||||
~ Licensed to the Apache Software Foundation (ASF) under one or more |
|
||||||
~ contributor license agreements. See the NOTICE file distributed with |
|
||||||
~ this work for additional information regarding copyright ownership. |
|
||||||
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
|
||||||
~ (the "License"); you may not use this file except in compliance with |
|
||||||
~ the License. You may obtain a copy of the License at |
|
||||||
~ |
|
||||||
~ http://www.apache.org/licenses/LICENSE-2.0 |
|
||||||
~ |
|
||||||
~ Unless required by applicable law or agreed to in writing, software |
|
||||||
~ distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
~ See the License for the specific language governing permissions and |
|
||||||
~ limitations under the License. |
|
||||||
--> |
|
||||||
<configuration> |
|
||||||
<property> |
|
||||||
<name>resource.storage.type</name> |
|
||||||
<display-name>Choose Resource Upload Startup Type</display-name> |
|
||||||
<description> |
|
||||||
Resource upload startup type : HDFS,S3,NONE |
|
||||||
</description> |
|
||||||
<value>NONE</value> |
|
||||||
<value-attributes> |
|
||||||
<type>value-list</type> |
|
||||||
<entries> |
|
||||||
<entry> |
|
||||||
<value>HDFS</value> |
|
||||||
<label>HDFS</label> |
|
||||||
</entry> |
|
||||||
<entry> |
|
||||||
<value>S3</value> |
|
||||||
<label>S3</label> |
|
||||||
</entry> |
|
||||||
<entry> |
|
||||||
<value>NONE</value> |
|
||||||
<label>NONE</label> |
|
||||||
</entry> |
|
||||||
</entries> |
|
||||||
<selection-cardinality>1</selection-cardinality> |
|
||||||
</value-attributes> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>resource.upload.path</name> |
|
||||||
<value>/dolphinscheduler</value> |
|
||||||
<description> |
|
||||||
resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended |
|
||||||
</description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>data.basedir.path</name> |
|
||||||
<value>/tmp/dolphinscheduler</value> |
|
||||||
<description> |
|
||||||
user data local directory path, please make sure the directory exists and have read write permissions |
|
||||||
</description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
|
|
||||||
<property> |
|
||||||
<name>hadoop.security.authentication.startup.state</name> |
|
||||||
<value>false</value> |
|
||||||
<value-attributes> |
|
||||||
<type>value-list</type> |
|
||||||
<entries> |
|
||||||
<entry> |
|
||||||
<value>true</value> |
|
||||||
<label>Enabled</label> |
|
||||||
</entry> |
|
||||||
<entry> |
|
||||||
<value>false</value> |
|
||||||
<label>Disabled</label> |
|
||||||
</entry> |
|
||||||
</entries> |
|
||||||
<selection-cardinality>1</selection-cardinality> |
|
||||||
</value-attributes> |
|
||||||
<description>whether kerberos starts</description> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>java.security.krb5.conf.path</name> |
|
||||||
<value>/opt/krb5.conf</value> |
|
||||||
<description> |
|
||||||
java.security.krb5.conf path |
|
||||||
</description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>login.user.keytab.username</name> |
|
||||||
<value>hdfs-mycluster@ESZ.COM</value> |
|
||||||
<description> |
|
||||||
LoginUserFromKeytab user |
|
||||||
</description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>login.user.keytab.path</name> |
|
||||||
<value>/opt/hdfs.headless.keytab</value> |
|
||||||
<description> |
|
||||||
LoginUserFromKeytab path |
|
||||||
</description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>resource.view.suffixs</name> |
|
||||||
<value>txt,log,sh,conf,cfg,py,java,sql,hql,xml,properties</value> |
|
||||||
<description></description> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>hdfs.root.user</name> |
|
||||||
<value>hdfs</value> |
|
||||||
<description> |
|
||||||
Users who have permission to create directories under the HDFS root path |
|
||||||
</description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>fs.defaultFS</name> |
|
||||||
<value>hdfs://mycluster:8020</value> |
|
||||||
<description> |
|
||||||
HA or single namenode, |
|
||||||
If namenode ha needs to copy core-site.xml and hdfs-site.xml to the conf directory， |
|
||||||
support s3，for example : s3a://dolphinscheduler |
|
||||||
</description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>fs.s3a.endpoint</name> |
|
||||||
<value>http://host:9010</value> |
|
||||||
<description> |
|
||||||
s3 need，s3 endpoint |
|
||||||
</description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>fs.s3a.access.key</name> |
|
||||||
<value>A3DXS30FO22544RE</value> |
|
||||||
<description> |
|
||||||
s3 need，s3 access key |
|
||||||
</description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>fs.s3a.secret.key</name> |
|
||||||
<value>OloCLq3n+8+sdPHUhJ21XrSxTC+JK</value> |
|
||||||
<description> |
|
||||||
s3 need，s3 secret key |
|
||||||
</description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>kerberos.expire.time</name> |
|
||||||
<value>7</value> |
|
||||||
<description></description> |
|
||||||
</property> |
|
||||||
|
|
||||||
</configuration> |
|
||||||
@ -1,467 +0,0 @@ |
|||||||
<!-- |
|
||||||
~ Licensed to the Apache Software Foundation (ASF) under one or more |
|
||||||
~ contributor license agreements. See the NOTICE file distributed with |
|
||||||
~ this work for additional information regarding copyright ownership. |
|
||||||
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
|
||||||
~ (the "License"); you may not use this file except in compliance with |
|
||||||
~ the License. You may obtain a copy of the License at |
|
||||||
~ |
|
||||||
~ http://www.apache.org/licenses/LICENSE-2.0 |
|
||||||
~ |
|
||||||
~ Unless required by applicable law or agreed to in writing, software |
|
||||||
~ distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
~ See the License for the specific language governing permissions and |
|
||||||
~ limitations under the License. |
|
||||||
--> |
|
||||||
<configuration> |
|
||||||
<property> |
|
||||||
<name>spring.datasource.initialSize</name> |
|
||||||
<value>5</value> |
|
||||||
<description> |
|
||||||
Init connection number |
|
||||||
</description> |
|
||||||
<value-attributes> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>spring.datasource.minIdle</name> |
|
||||||
<value>5</value> |
|
||||||
<description> |
|
||||||
Min connection number |
|
||||||
</description> |
|
||||||
<value-attributes> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>spring.datasource.maxActive</name> |
|
||||||
<value>50</value> |
|
||||||
<description> |
|
||||||
Max connection number |
|
||||||
</description> |
|
||||||
<value-attributes> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>spring.datasource.maxWait</name> |
|
||||||
<value>60000</value> |
|
||||||
<description> |
|
||||||
Max wait time for get a connection in milliseconds. |
|
||||||
If configuring maxWait, fair locks are enabled by default and concurrency efficiency decreases. |
|
||||||
If necessary, unfair locks can be used by configuring the useUnfairLock attribute to true. |
|
||||||
</description> |
|
||||||
<value-attributes> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>spring.datasource.timeBetweenEvictionRunsMillis</name> |
|
||||||
<value>60000</value> |
|
||||||
<description> |
|
||||||
Milliseconds for check to close free connections |
|
||||||
</description> |
|
||||||
<value-attributes> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>spring.datasource.timeBetweenConnectErrorMillis</name> |
|
||||||
<value>60000</value> |
|
||||||
<description> |
|
||||||
The Destroy thread detects the connection interval and closes the physical connection in milliseconds |
|
||||||
if the connection idle time is greater than or equal to minEvictableIdleTimeMillis. |
|
||||||
</description> |
|
||||||
<value-attributes> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>spring.datasource.minEvictableIdleTimeMillis</name> |
|
||||||
<value>300000</value> |
|
||||||
<description> |
|
||||||
The longest time a connection remains idle without being evicted, in milliseconds |
|
||||||
</description> |
|
||||||
<value-attributes> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>spring.datasource.validationQuery</name> |
|
||||||
<value>SELECT 1</value> |
|
||||||
<description> |
|
||||||
The SQL used to check whether the connection is valid requires a query statement. |
|
||||||
If validation Query is null, testOnBorrow, testOnReturn, and testWhileIdle will not work. |
|
||||||
</description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>spring.datasource.validationQueryTimeout</name> |
|
||||||
<value>3</value> |
|
||||||
<value-attributes> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
<description> |
|
||||||
Check whether the connection is valid for timeout, in seconds |
|
||||||
</description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>spring.datasource.testWhileIdle</name> |
|
||||||
<value>true</value> |
|
||||||
<value-attributes> |
|
||||||
<type>boolean</type> |
|
||||||
</value-attributes> |
|
||||||
<description> |
|
||||||
When applying for a connection, |
|
||||||
if it is detected that the connection is idle longer than time Between Eviction Runs Millis, |
|
||||||
validation Query is performed to check whether the connection is valid |
|
||||||
</description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>spring.datasource.testOnBorrow</name> |
|
||||||
<value>true</value> |
|
||||||
<value-attributes> |
|
||||||
<type>boolean</type> |
|
||||||
</value-attributes> |
|
||||||
<description> |
|
||||||
Execute validation to check if the connection is valid when applying for a connection |
|
||||||
</description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>spring.datasource.testOnReturn</name> |
|
||||||
<value>false</value> |
|
||||||
<value-attributes> |
|
||||||
<type>boolean</type> |
|
||||||
</value-attributes> |
|
||||||
<description> |
|
||||||
Execute validation to check if the connection is valid when the connection is returned |
|
||||||
</description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>spring.datasource.defaultAutoCommit</name> |
|
||||||
<value>true</value> |
|
||||||
<value-attributes> |
|
||||||
<type>boolean</type> |
|
||||||
</value-attributes> |
|
||||||
<description> |
|
||||||
</description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>spring.datasource.keepAlive</name> |
|
||||||
<value>false</value> |
|
||||||
<value-attributes> |
|
||||||
<type>boolean</type> |
|
||||||
</value-attributes> |
|
||||||
<description> |
|
||||||
</description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
|
|
||||||
<property> |
|
||||||
<name>spring.datasource.poolPreparedStatements</name> |
|
||||||
<value>true</value> |
|
||||||
<value-attributes> |
|
||||||
<type>boolean</type> |
|
||||||
</value-attributes> |
|
||||||
<description> |
|
||||||
Open PSCache, specify count PSCache for every connection |
|
||||||
</description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>spring.datasource.maxPoolPreparedStatementPerConnectionSize</name> |
|
||||||
<value>20</value> |
|
||||||
<value-attributes> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
<description></description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>spring.datasource.spring.datasource.filters</name> |
|
||||||
<value>stat,wall,log4j</value> |
|
||||||
<description></description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>spring.datasource.connectionProperties</name> |
|
||||||
<value>druid.stat.mergeSql=true;druid.stat.slowSqlMillis=5000</value> |
|
||||||
<description></description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
|
|
||||||
<property> |
|
||||||
<name>mybatis-plus.mapper-locations</name> |
|
||||||
<value>classpath*:/org.apache.dolphinscheduler.dao.mapper/*.xml</value> |
|
||||||
<description></description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>mybatis-plus.typeEnumsPackage</name> |
|
||||||
<value>org.apache.dolphinscheduler.*.enums</value> |
|
||||||
<description></description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>mybatis-plus.typeAliasesPackage</name> |
|
||||||
<value>org.apache.dolphinscheduler.dao.entity</value> |
|
||||||
<description> |
|
||||||
Entity scan, where multiple packages are separated by a comma or semicolon |
|
||||||
</description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>mybatis-plus.global-config.db-config.id-type</name> |
|
||||||
<value>AUTO</value> |
|
||||||
<value-attributes> |
|
||||||
<type>value-list</type> |
|
||||||
<entries> |
|
||||||
<entry> |
|
||||||
<value>AUTO</value> |
|
||||||
<label>AUTO</label> |
|
||||||
</entry> |
|
||||||
<entry> |
|
||||||
<value>INPUT</value> |
|
||||||
<label>INPUT</label> |
|
||||||
</entry> |
|
||||||
<entry> |
|
||||||
<value>ID_WORKER</value> |
|
||||||
<label>ID_WORKER</label> |
|
||||||
</entry> |
|
||||||
<entry> |
|
||||||
<value>UUID</value> |
|
||||||
<label>UUID</label> |
|
||||||
</entry> |
|
||||||
</entries> |
|
||||||
<selection-cardinality>1</selection-cardinality> |
|
||||||
</value-attributes> |
|
||||||
<description> |
|
||||||
Primary key type AUTO:" database ID AUTO ", |
|
||||||
INPUT:" user INPUT ID", |
|
||||||
ID_WORKER:" global unique ID (numeric type unique ID)", |
|
||||||
UUID:" global unique ID UUID"; |
|
||||||
</description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>mybatis-plus.global-config.db-config.field-strategy</name> |
|
||||||
<value>NOT_NULL</value> |
|
||||||
<value-attributes> |
|
||||||
<type>value-list</type> |
|
||||||
<entries> |
|
||||||
<entry> |
|
||||||
<value>IGNORED</value> |
|
||||||
<label>IGNORED</label> |
|
||||||
</entry> |
|
||||||
<entry> |
|
||||||
<value>NOT_NULL</value> |
|
||||||
<label>NOT_NULL</label> |
|
||||||
</entry> |
|
||||||
<entry> |
|
||||||
<value>NOT_EMPTY</value> |
|
||||||
<label>NOT_EMPTY</label> |
|
||||||
</entry> |
|
||||||
</entries> |
|
||||||
<selection-cardinality>1</selection-cardinality> |
|
||||||
</value-attributes> |
|
||||||
<description> |
|
||||||
Field policy IGNORED:" ignore judgment ", |
|
||||||
NOT_NULL:" not NULL judgment "), |
|
||||||
NOT_EMPTY:" not NULL judgment" |
|
||||||
</description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>mybatis-plus.global-config.db-config.column-underline</name> |
|
||||||
<value>true</value> |
|
||||||
<value-attributes> |
|
||||||
<type>boolean</type> |
|
||||||
</value-attributes> |
|
||||||
<description></description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>mybatis-plus.global-config.db-config.logic-delete-value</name> |
|
||||||
<value>1</value> |
|
||||||
<value-attributes> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
<description></description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>mybatis-plus.global-config.db-config.logic-not-delete-value</name> |
|
||||||
<value>0</value> |
|
||||||
<value-attributes> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
<description></description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>mybatis-plus.global-config.db-config.banner</name> |
|
||||||
<value>true</value> |
|
||||||
<value-attributes> |
|
||||||
<type>boolean</type> |
|
||||||
</value-attributes> |
|
||||||
<description></description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
|
|
||||||
<property> |
|
||||||
<name>mybatis-plus.configuration.map-underscore-to-camel-case</name> |
|
||||||
<value>true</value> |
|
||||||
<value-attributes> |
|
||||||
<type>boolean</type> |
|
||||||
</value-attributes> |
|
||||||
<description></description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>mybatis-plus.configuration.cache-enabled</name> |
|
||||||
<value>false</value> |
|
||||||
<value-attributes> |
|
||||||
<type>boolean</type> |
|
||||||
</value-attributes> |
|
||||||
<description></description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>mybatis-plus.configuration.call-setters-on-nulls</name> |
|
||||||
<value>true</value> |
|
||||||
<value-attributes> |
|
||||||
<type>boolean</type> |
|
||||||
</value-attributes> |
|
||||||
<description></description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>mybatis-plus.configuration.jdbc-type-for-null</name> |
|
||||||
<value>null</value> |
|
||||||
<description></description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>master.exec.threads</name> |
|
||||||
<value>100</value> |
|
||||||
<value-attributes> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
<description></description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>master.exec.task.num</name> |
|
||||||
<value>20</value> |
|
||||||
<value-attributes> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
<description></description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>master.heartbeat.interval</name> |
|
||||||
<value>10</value> |
|
||||||
<value-attributes> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
<description></description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>master.task.commit.retryTimes</name> |
|
||||||
<value>5</value> |
|
||||||
<value-attributes> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
<description></description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>master.task.commit.interval</name> |
|
||||||
<value>1000</value> |
|
||||||
<value-attributes> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
<description></description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>master.max.cpuload.avg</name> |
|
||||||
<value>100</value> |
|
||||||
<value-attributes> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
<description></description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>master.reserved.memory</name> |
|
||||||
<value>0.1</value> |
|
||||||
<value-attributes> |
|
||||||
<type>float</type> |
|
||||||
</value-attributes> |
|
||||||
<description></description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>worker.exec.threads</name> |
|
||||||
<value>100</value> |
|
||||||
<value-attributes> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
<description></description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>worker.heartbeat.interval</name> |
|
||||||
<value>10</value> |
|
||||||
<value-attributes> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
<description></description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>worker.fetch.task.num</name> |
|
||||||
<value>3</value> |
|
||||||
<value-attributes> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
<description></description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>worker.max.cpuload.avg</name> |
|
||||||
<value>100</value> |
|
||||||
<value-attributes> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
<description></description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>worker.reserved.memory</name> |
|
||||||
<value>0.1</value> |
|
||||||
<value-attributes> |
|
||||||
<type>float</type> |
|
||||||
</value-attributes> |
|
||||||
<description></description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
|
|
||||||
</configuration> |
|
||||||
@ -1,123 +0,0 @@ |
|||||||
<!-- |
|
||||||
~ Licensed to the Apache Software Foundation (ASF) under one or more |
|
||||||
~ contributor license agreements. See the NOTICE file distributed with |
|
||||||
~ this work for additional information regarding copyright ownership. |
|
||||||
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
|
||||||
~ (the "License"); you may not use this file except in compliance with |
|
||||||
~ the License. You may obtain a copy of the License at |
|
||||||
~ |
|
||||||
~ http://www.apache.org/licenses/LICENSE-2.0 |
|
||||||
~ |
|
||||||
~ Unless required by applicable law or agreed to in writing, software |
|
||||||
~ distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
~ See the License for the specific language governing permissions and |
|
||||||
~ limitations under the License. |
|
||||||
--> |
|
||||||
<configuration> |
|
||||||
<property> |
|
||||||
<name>dolphin.database.type</name> |
|
||||||
<value>mysql</value> |
|
||||||
<description>Dolphin Scheduler DataBase Type Which Is Select</description> |
|
||||||
<display-name>Dolphin Database Type</display-name> |
|
||||||
<value-attributes> |
|
||||||
<type>value-list</type> |
|
||||||
<entries> |
|
||||||
<entry> |
|
||||||
<value>mysql</value> |
|
||||||
<label>Mysql</label> |
|
||||||
</entry> |
|
||||||
<entry> |
|
||||||
<value>postgresql</value> |
|
||||||
<label>Postgresql</label> |
|
||||||
</entry> |
|
||||||
</entries> |
|
||||||
<selection-cardinality>1</selection-cardinality> |
|
||||||
</value-attributes> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
|
|
||||||
<property> |
|
||||||
<name>dolphin.database.host</name> |
|
||||||
<value></value> |
|
||||||
<display-name>Dolphin Database Host</display-name> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
|
|
||||||
<property> |
|
||||||
<name>dolphin.database.port</name> |
|
||||||
<value></value> |
|
||||||
<display-name>Dolphin Database Port</display-name> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
|
|
||||||
<property> |
|
||||||
<name>dolphin.database.username</name> |
|
||||||
<value></value> |
|
||||||
<display-name>Dolphin Database Username</display-name> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
|
|
||||||
<property> |
|
||||||
<name>dolphin.database.password</name> |
|
||||||
<value></value> |
|
||||||
<display-name>Dolphin Database Password</display-name> |
|
||||||
<property-type>PASSWORD</property-type> |
|
||||||
<value-attributes> |
|
||||||
<type>password</type> |
|
||||||
</value-attributes> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
|
|
||||||
<property> |
|
||||||
<name>dolphin.user</name> |
|
||||||
<value></value> |
|
||||||
<description>Which user to install and admin dolphin scheduler</description> |
|
||||||
<display-name>Deploy User</display-name> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>dolphin.group</name> |
|
||||||
<value></value> |
|
||||||
<description>Which user to install and admin dolphin scheduler</description> |
|
||||||
<display-name>Deploy Group</display-name> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
|
|
||||||
<property> |
|
||||||
<name>dolphinscheduler-env-content</name> |
|
||||||
<display-name>Dolphinscheduler Env template</display-name> |
|
||||||
<description>This is the jinja template for dolphinscheduler.env.sh file</description> |
|
||||||
<value># |
|
||||||
# Licensed to the Apache Software Foundation (ASF) under one or more |
|
||||||
# contributor license agreements. See the NOTICE file distributed with |
|
||||||
# this work for additional information regarding copyright ownership. |
|
||||||
# The ASF licenses this file to You under the Apache License, Version 2.0 |
|
||||||
# (the "License"); you may not use this file except in compliance with |
|
||||||
# the License. You may obtain a copy of the License at |
|
||||||
# |
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0 |
|
||||||
# |
|
||||||
# Unless required by applicable law or agreed to in writing, software |
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
# See the License for the specific language governing permissions and |
|
||||||
# limitations under the License. |
|
||||||
# |
|
||||||
|
|
||||||
export HADOOP_HOME=/opt/soft/hadoop |
|
||||||
export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop |
|
||||||
export SPARK_HOME1=/opt/soft/spark1 |
|
||||||
export SPARK_HOME2=/opt/soft/spark2 |
|
||||||
export PYTHON_HOME=/opt/soft/python |
|
||||||
export JAVA_HOME=/opt/soft/java |
|
||||||
export HIVE_HOME=/opt/soft/hive |
|
||||||
export FLINK_HOME=/opt/soft/flink</value> |
|
||||||
<value-attributes> |
|
||||||
<type>content</type> |
|
||||||
<empty-value-valid>false</empty-value-valid> |
|
||||||
<show-property-name>false</show-property-name> |
|
||||||
</value-attributes> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
</configuration> |
|
||||||
@ -1,88 +0,0 @@ |
|||||||
<!-- |
|
||||||
~ Licensed to the Apache Software Foundation (ASF) under one or more |
|
||||||
~ contributor license agreements. See the NOTICE file distributed with |
|
||||||
~ this work for additional information regarding copyright ownership. |
|
||||||
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
|
||||||
~ (the "License"); you may not use this file except in compliance with |
|
||||||
~ the License. You may obtain a copy of the License at |
|
||||||
~ |
|
||||||
~ http://www.apache.org/licenses/LICENSE-2.0 |
|
||||||
~ |
|
||||||
~ Unless required by applicable law or agreed to in writing, software |
|
||||||
~ distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
~ See the License for the specific language governing permissions and |
|
||||||
~ limitations under the License. |
|
||||||
--> |
|
||||||
<configuration> |
|
||||||
<property> |
|
||||||
<name>master.exec.threads</name> |
|
||||||
<value>100</value> |
|
||||||
<value-attributes> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
<description>master execute thread num</description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>master.exec.task.num</name> |
|
||||||
<value>20</value> |
|
||||||
<value-attributes> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
<description>master execute task number in parallel</description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>master.heartbeat.interval</name> |
|
||||||
<value>10</value> |
|
||||||
<value-attributes> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
<description>master heartbeat interval</description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>master.task.commit.retryTimes</name> |
|
||||||
<value>5</value> |
|
||||||
<value-attributes> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
<description>master commit task retry times</description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>master.task.commit.interval</name> |
|
||||||
<value>1000</value> |
|
||||||
<value-attributes> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
<description>master commit task interval</description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>master.max.cpuload.avg</name> |
|
||||||
<value>100</value> |
|
||||||
<value-attributes> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
<description>only less than cpu avg load, master server can work. default value : the number of cpu cores * 2</description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>master.reserved.memory</name> |
|
||||||
<value>0.3</value> |
|
||||||
<description>only larger than reserved memory, master server can work. default value : physical memory * 1/10, unit is G.</description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
|
|
||||||
<property> |
|
||||||
<name>master.listen.port</name> |
|
||||||
<value>5678</value> |
|
||||||
<value-attributes> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
<description>master listen port</description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
</configuration> |
|
||||||
@ -1,126 +0,0 @@ |
|||||||
<!-- |
|
||||||
~ Licensed to the Apache Software Foundation (ASF) under one or more |
|
||||||
~ contributor license agreements. See the NOTICE file distributed with |
|
||||||
~ this work for additional information regarding copyright ownership. |
|
||||||
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
|
||||||
~ (the "License"); you may not use this file except in compliance with |
|
||||||
~ the License. You may obtain a copy of the License at |
|
||||||
~ |
|
||||||
~ http://www.apache.org/licenses/LICENSE-2.0 |
|
||||||
~ |
|
||||||
~ Unless required by applicable law or agreed to in writing, software |
|
||||||
~ distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
~ See the License for the specific language governing permissions and |
|
||||||
~ limitations under the License. |
|
||||||
--> |
|
||||||
<configuration> |
|
||||||
<property> |
|
||||||
<name>org.quartz.scheduler.instanceName</name> |
|
||||||
<value>DolphinScheduler</value> |
|
||||||
<description></description> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<!-- 列举枚举值 --> |
|
||||||
<name>org.quartz.scheduler.instanceId</name> |
|
||||||
<value>AUTO</value> |
|
||||||
<description></description> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>org.quartz.scheduler.makeSchedulerThreadDaemon</name> |
|
||||||
<value>true</value> |
|
||||||
<value-attributes> |
|
||||||
<type>boolean</type> |
|
||||||
</value-attributes> |
|
||||||
<description></description> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>org.quartz.jobStore.useProperties</name> |
|
||||||
<value>false</value> |
|
||||||
<value-attributes> |
|
||||||
<type>boolean</type> |
|
||||||
</value-attributes> |
|
||||||
<description></description> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>org.quartz.threadPool.class</name> |
|
||||||
<value>org.quartz.simpl.SimpleThreadPool</value> |
|
||||||
<description></description> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>org.quartz.threadPool.makeThreadsDaemons</name> |
|
||||||
<value>true</value> |
|
||||||
<value-attributes> |
|
||||||
<type>boolean</type> |
|
||||||
</value-attributes> |
|
||||||
<description></description> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>org.quartz.threadPool.threadCount</name> |
|
||||||
<value>25</value> |
|
||||||
<value-attributes> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
<description></description> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>org.quartz.threadPool.threadPriority</name> |
|
||||||
<value>5</value> |
|
||||||
<value-attributes> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
<description></description> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>org.quartz.jobStore.class</name> |
|
||||||
<value>org.quartz.impl.jdbcjobstore.JobStoreTX</value> |
|
||||||
<description></description> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>org.quartz.jobStore.tablePrefix</name> |
|
||||||
<value>QRTZ_</value> |
|
||||||
<description></description> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>org.quartz.jobStore.isClustered</name> |
|
||||||
<value>true</value> |
|
||||||
<value-attributes> |
|
||||||
<type>boolean</type> |
|
||||||
</value-attributes> |
|
||||||
<description></description> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>org.quartz.jobStore.misfireThreshold</name> |
|
||||||
<value>60000</value> |
|
||||||
<value-attributes> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
<description></description> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>org.quartz.jobStore.clusterCheckinInterval</name> |
|
||||||
<value>5000</value> |
|
||||||
<value-attributes> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
<description></description> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>org.quartz.jobStore.acquireTriggersWithinLock</name> |
|
||||||
<value>true</value> |
|
||||||
<value-attributes> |
|
||||||
<type>boolean</type> |
|
||||||
</value-attributes> |
|
||||||
<description></description> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>org.quartz.jobStore.dataSource</name> |
|
||||||
<value>myDs</value> |
|
||||||
<description></description> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>org.quartz.dataSource.myDs.connectionProvider.class</name> |
|
||||||
<value>org.apache.dolphinscheduler.service.quartz.DruidConnectionProvider</value> |
|
||||||
<description></description> |
|
||||||
</property> |
|
||||||
</configuration> |
|
||||||
@ -1,76 +0,0 @@ |
|||||||
<!-- |
|
||||||
~ Licensed to the Apache Software Foundation (ASF) under one or more |
|
||||||
~ contributor license agreements. See the NOTICE file distributed with |
|
||||||
~ this work for additional information regarding copyright ownership. |
|
||||||
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
|
||||||
~ (the "License"); you may not use this file except in compliance with |
|
||||||
~ the License. You may obtain a copy of the License at |
|
||||||
~ |
|
||||||
~ http://www.apache.org/licenses/LICENSE-2.0 |
|
||||||
~ |
|
||||||
~ Unless required by applicable law or agreed to in writing, software |
|
||||||
~ distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
~ See the License for the specific language governing permissions and |
|
||||||
~ limitations under the License. |
|
||||||
--> |
|
||||||
<configuration> |
|
||||||
<property> |
|
||||||
<name>worker.exec.threads</name> |
|
||||||
<value>100</value> |
|
||||||
<value-attributes> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
<description>worker execute thread num</description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>worker.heartbeat.interval</name> |
|
||||||
<value>10</value> |
|
||||||
<value-attributes> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
<description>worker heartbeat interval</description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>worker.fetch.task.num</name> |
|
||||||
<value>3</value> |
|
||||||
<value-attributes> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
<description>submit the number of tasks at a time</description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>worker.max.cpuload.avg</name> |
|
||||||
<value>100</value> |
|
||||||
<value-attributes> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
<description>only less than cpu avg load, worker server can work. default value : the number of cpu cores * 2</description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>worker.reserved.memory</name> |
|
||||||
<value>0.3</value> |
|
||||||
<description>only larger than reserved memory, worker server can work. default value : physical memory * 1/10, unit is G.</description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
|
|
||||||
<property> |
|
||||||
<name>worker.listen.port</name> |
|
||||||
<value>1234</value> |
|
||||||
<value-attributes> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
<description>worker listen port</description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>worker.group</name> |
|
||||||
<value>default</value> |
|
||||||
<description>default worker group</description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
</configuration> |
|
||||||
@ -1,84 +0,0 @@ |
|||||||
<!-- |
|
||||||
~ Licensed to the Apache Software Foundation (ASF) under one or more |
|
||||||
~ contributor license agreements. See the NOTICE file distributed with |
|
||||||
~ this work for additional information regarding copyright ownership. |
|
||||||
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
|
||||||
~ (the "License"); you may not use this file except in compliance with |
|
||||||
~ the License. You may obtain a copy of the License at |
|
||||||
~ |
|
||||||
~ http://www.apache.org/licenses/LICENSE-2.0 |
|
||||||
~ |
|
||||||
~ Unless required by applicable law or agreed to in writing, software |
|
||||||
~ distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
~ See the License for the specific language governing permissions and |
|
||||||
~ limitations under the License. |
|
||||||
--> |
|
||||||
<configuration> |
|
||||||
<property> |
|
||||||
<name>dolphinscheduler.queue.impl</name> |
|
||||||
<value>zookeeper</value> |
|
||||||
<description> |
|
||||||
Task queue implementation, default "zookeeper" |
|
||||||
</description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>zookeeper.dolphinscheduler.root</name> |
|
||||||
<value>/dolphinscheduler</value> |
|
||||||
<description> |
|
||||||
dolphinscheduler root directory |
|
||||||
</description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>zookeeper.session.timeout</name> |
|
||||||
<value>300</value> |
|
||||||
<value-attributes> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
<description> |
|
||||||
</description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>zookeeper.connection.timeout</name> |
|
||||||
<value>300</value> |
|
||||||
<value-attributes> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
<description> |
|
||||||
</description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>zookeeper.retry.base.sleep</name> |
|
||||||
<value>100</value> |
|
||||||
<value-attributes> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
<description> |
|
||||||
</description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>zookeeper.retry.max.sleep</name> |
|
||||||
<value>30000</value> |
|
||||||
<value-attributes> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
<description> |
|
||||||
</description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
<property> |
|
||||||
<name>zookeeper.retry.maxtime</name> |
|
||||||
<value>5</value> |
|
||||||
<value-attributes> |
|
||||||
<type>int</type> |
|
||||||
</value-attributes> |
|
||||||
<description> |
|
||||||
</description> |
|
||||||
<on-ambari-upgrade add="true"/> |
|
||||||
</property> |
|
||||||
</configuration> |
|
||||||
@ -1,137 +0,0 @@ |
|||||||
<?xml version="1.0"?> |
|
||||||
<!-- |
|
||||||
Licensed to the Apache Software Foundation (ASF) under one or more |
|
||||||
contributor license agreements. See the NOTICE file distributed with |
|
||||||
this work for additional information regarding copyright ownership. |
|
||||||
The ASF licenses this file to You under the Apache License, Version 2.0 |
|
||||||
(the "License"); you may not use this file except in compliance with |
|
||||||
the License. You may obtain a copy of the License at |
|
||||||
|
|
||||||
http://www.apache.org/licenses/LICENSE-2.0 |
|
||||||
|
|
||||||
Unless required by applicable law or agreed to in writing, software |
|
||||||
distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
See the License for the specific language governing permissions and |
|
||||||
limitations under the License. |
|
||||||
--> |
|
||||||
<metainfo> |
|
||||||
<schemaVersion>2.0</schemaVersion> |
|
||||||
<services> |
|
||||||
<service> |
|
||||||
<name>DOLPHIN</name> |
|
||||||
<displayName>Dolphin Scheduler</displayName> |
|
||||||
<comment>分布式易扩展的可视化DAG工作流任务调度系统</comment> |
|
||||||
<version>2.0.0</version> |
|
||||||
<components> |
|
||||||
<component> |
|
||||||
<name>DOLPHIN_MASTER</name> |
|
||||||
<displayName>DS Master</displayName> |
|
||||||
<category>MASTER</category> |
|
||||||
<cardinality>1+</cardinality> |
|
||||||
<commandScript> |
|
||||||
<script>scripts/dolphin_master_service.py</script> |
|
||||||
<scriptType>PYTHON</scriptType> |
|
||||||
<timeout>600</timeout> |
|
||||||
</commandScript> |
|
||||||
</component> |
|
||||||
|
|
||||||
<component> |
|
||||||
<name>DOLPHIN_LOGGER</name> |
|
||||||
<displayName>DS Logger</displayName> |
|
||||||
<category>SLAVE</category> |
|
||||||
<cardinality>1+</cardinality> |
|
||||||
<commandScript> |
|
||||||
<script>scripts/dolphin_logger_service.py</script> |
|
||||||
<scriptType>PYTHON</scriptType> |
|
||||||
<timeout>600</timeout> |
|
||||||
</commandScript> |
|
||||||
</component> |
|
||||||
|
|
||||||
<component> |
|
||||||
<name>DOLPHIN_WORKER</name> |
|
||||||
<displayName>DS Worker</displayName> |
|
||||||
<category>SLAVE</category> |
|
||||||
<cardinality>1+</cardinality> |
|
||||||
<dependencies> |
|
||||||
<dependency> |
|
||||||
<name>DOLPHIN/DOLPHIN_LOGGER</name> |
|
||||||
<scope>host</scope> |
|
||||||
<auto-deploy> |
|
||||||
<enabled>true</enabled> |
|
||||||
</auto-deploy> |
|
||||||
</dependency> |
|
||||||
</dependencies> |
|
||||||
<commandScript> |
|
||||||
<script>scripts/dolphin_worker_service.py</script> |
|
||||||
<scriptType>PYTHON</scriptType> |
|
||||||
<timeout>600</timeout> |
|
||||||
</commandScript> |
|
||||||
</component> |
|
||||||
|
|
||||||
<component> |
|
||||||
<name>DOLPHIN_ALERT</name> |
|
||||||
<displayName>DS Alert</displayName> |
|
||||||
<category>SLAVE</category> |
|
||||||
<cardinality>1</cardinality> |
|
||||||
<commandScript> |
|
||||||
<script>scripts/dolphin_alert_service.py</script> |
|
||||||
<scriptType>PYTHON</scriptType> |
|
||||||
<timeout>600</timeout> |
|
||||||
</commandScript> |
|
||||||
</component> |
|
||||||
|
|
||||||
<component> |
|
||||||
<name>DOLPHIN_API</name> |
|
||||||
<displayName>DS_Api</displayName> |
|
||||||
<category>SLAVE</category> |
|
||||||
<cardinality>1</cardinality> |
|
||||||
<commandScript> |
|
||||||
<script>scripts/dolphin_api_service.py</script> |
|
||||||
<scriptType>PYTHON</scriptType> |
|
||||||
<timeout>600</timeout> |
|
||||||
</commandScript> |
|
||||||
</component> |
|
||||||
</components> |
|
||||||
|
|
||||||
<requiredServices> |
|
||||||
<service>ZOOKEEPER</service> |
|
||||||
</requiredServices> |
|
||||||
|
|
||||||
<osSpecifics> |
|
||||||
<osSpecific> |
|
||||||
<osFamily>any</osFamily> |
|
||||||
<packages> |
|
||||||
<package> |
|
||||||
<name>apache-dolphinscheduler-incubating-1.2.1*</name> |
|
||||||
</package> |
|
||||||
</packages> |
|
||||||
</osSpecific> |
|
||||||
</osSpecifics> |
|
||||||
|
|
||||||
<configuration-dependencies> |
|
||||||
<config-type>dolphin-alert</config-type> |
|
||||||
<config-type>dolphin-app-api</config-type> |
|
||||||
<config-type>dolphin-app-dao</config-type> |
|
||||||
<config-type>dolphin-common</config-type> |
|
||||||
<config-type>dolphin-env</config-type> |
|
||||||
<config-type>dolphin-quartz</config-type> |
|
||||||
</configuration-dependencies> |
|
||||||
|
|
||||||
<themes> |
|
||||||
<theme> |
|
||||||
<fileName>theme.json</fileName> |
|
||||||
<default>true</default> |
|
||||||
</theme> |
|
||||||
</themes> |
|
||||||
|
|
||||||
<quickLinksConfigurations-dir>quicklinks</quickLinksConfigurations-dir> |
|
||||||
<quickLinksConfigurations> |
|
||||||
<quickLinksConfiguration> |
|
||||||
<fileName>quicklinks.json</fileName> |
|
||||||
<default>true</default> |
|
||||||
</quickLinksConfiguration> |
|
||||||
</quickLinksConfigurations> |
|
||||||
</service> |
|
||||||
</services> |
|
||||||
</metainfo> |
|
||||||
@ -1,124 +0,0 @@ |
|||||||
""" |
|
||||||
Licensed to the Apache Software Foundation (ASF) under one |
|
||||||
or more contributor license agreements. See the NOTICE file |
|
||||||
distributed with this work for additional information |
|
||||||
regarding copyright ownership. The ASF licenses this file |
|
||||||
to you under the Apache License, Version 2.0 (the |
|
||||||
"License"); you may not use this file except in compliance |
|
||||||
with the License. You may obtain a copy of the License at |
|
||||||
|
|
||||||
http://www.apache.org/licenses/LICENSE-2.0 |
|
||||||
|
|
||||||
Unless required by applicable law or agreed to in writing, software |
|
||||||
distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
See the License for the specific language governing permissions and |
|
||||||
limitations under the License. |
|
||||||
""" |
|
||||||
|
|
||||||
import socket |
|
||||||
import urllib2 |
|
||||||
import os |
|
||||||
import logging |
|
||||||
import ambari_simplejson as json |
|
||||||
from resource_management.libraries.script.script import Script |
|
||||||
import sys |
|
||||||
reload(sys) |
|
||||||
sys.setdefaultencoding('utf-8') |
|
||||||
|
|
||||||
logger = logging.getLogger('ambari_alerts') |
|
||||||
|
|
||||||
config = Script.get_config() |
|
||||||
|
|
||||||
|
|
||||||
def get_tokens(): |
|
||||||
""" |
|
||||||
Returns a tuple of tokens in the format {{site/property}} that will be used |
|
||||||
to build the dictionary passed into execute |
|
||||||
|
|
||||||
:rtype tuple |
|
||||||
""" |
|
||||||
|
|
||||||
def get_info(url, connection_timeout): |
|
||||||
response = None |
|
||||||
|
|
||||||
try: |
|
||||||
response = urllib2.urlopen(url, timeout=connection_timeout) |
|
||||||
json_data = response.read() |
|
||||||
return json_data |
|
||||||
finally: |
|
||||||
if response is not None: |
|
||||||
try: |
|
||||||
response.close() |
|
||||||
except: |
|
||||||
pass |
|
||||||
|
|
||||||
|
|
||||||
def execute(configurations={}, parameters={}, host_name=None): |
|
||||||
""" |
|
||||||
Returns a tuple containing the result code and a pre-formatted result label |
|
||||||
|
|
||||||
Keyword arguments: |
|
||||||
configurations : a mapping of configuration key to value |
|
||||||
parameters : a mapping of script parameter key to value |
|
||||||
host_name : the name of this host where the alert is running |
|
||||||
|
|
||||||
:type configurations dict |
|
||||||
:type parameters dict |
|
||||||
:type host_name str |
|
||||||
""" |
|
||||||
|
|
||||||
alert_name = parameters['alertName'] |
|
||||||
|
|
||||||
dolphin_pidfile_dir = "/opt/soft/run/dolphinscheduler" |
|
||||||
|
|
||||||
pid = "0" |
|
||||||
|
|
||||||
|
|
||||||
from resource_management.core import sudo |
|
||||||
|
|
||||||
is_running = True |
|
||||||
pid_file_path = "" |
|
||||||
if alert_name == 'DOLPHIN_MASTER': |
|
||||||
pid_file_path = dolphin_pidfile_dir + "/master-server.pid" |
|
||||||
elif alert_name == 'DOLPHIN_WORKER': |
|
||||||
pid_file_path = dolphin_pidfile_dir + "/worker-server.pid" |
|
||||||
elif alert_name == 'DOLPHIN_ALERT': |
|
||||||
pid_file_path = dolphin_pidfile_dir + "/alert-server.pid" |
|
||||||
elif alert_name == 'DOLPHIN_LOGGER': |
|
||||||
pid_file_path = dolphin_pidfile_dir + "/logger-server.pid" |
|
||||||
elif alert_name == 'DOLPHIN_API': |
|
||||||
pid_file_path = dolphin_pidfile_dir + "/api-server.pid" |
|
||||||
|
|
||||||
if not pid_file_path or not os.path.isfile(pid_file_path): |
|
||||||
is_running = False |
|
||||||
|
|
||||||
try: |
|
||||||
pid = int(sudo.read_file(pid_file_path)) |
|
||||||
except: |
|
||||||
is_running = False |
|
||||||
|
|
||||||
try: |
|
||||||
# Kill will not actually kill the process |
|
||||||
# From the doc: |
|
||||||
# If sig is 0, then no signal is sent, but error checking is still |
|
||||||
# performed; this can be used to check for the existence of a |
|
||||||
# process ID or process group ID. |
|
||||||
sudo.kill(pid, 0) |
|
||||||
except OSError: |
|
||||||
is_running = False |
|
||||||
|
|
||||||
if host_name is None: |
|
||||||
host_name = socket.getfqdn() |
|
||||||
|
|
||||||
if not is_running: |
|
||||||
result_code = "CRITICAL" |
|
||||||
else: |
|
||||||
result_code = "OK" |
|
||||||
|
|
||||||
label = "The comment {0} of DOLPHIN_SCHEDULER on {1} is {2}".format(alert_name, host_name, result_code) |
|
||||||
|
|
||||||
return ((result_code, [label])) |
|
||||||
|
|
||||||
if __name__ == "__main__": |
|
||||||
pass |
|
||||||
@ -1,61 +0,0 @@ |
|||||||
""" |
|
||||||
Licensed to the Apache Software Foundation (ASF) under one |
|
||||||
or more contributor license agreements. See the NOTICE file |
|
||||||
distributed with this work for additional information |
|
||||||
regarding copyright ownership. The ASF licenses this file |
|
||||||
to you under the Apache License, Version 2.0 (the |
|
||||||
"License"); you may not use this file except in compliance |
|
||||||
with the License. You may obtain a copy of the License at |
|
||||||
|
|
||||||
http://www.apache.org/licenses/LICENSE-2.0 |
|
||||||
|
|
||||||
Unless required by applicable law or agreed to in writing, software |
|
||||||
distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
See the License for the specific language governing permissions and |
|
||||||
limitations under the License. |
|
||||||
""" |
|
||||||
import time |
|
||||||
from resource_management import * |
|
||||||
|
|
||||||
from dolphin_env import dolphin_env |
|
||||||
|
|
||||||
|
|
||||||
class DolphinAlertService(Script): |
|
||||||
def install(self, env): |
|
||||||
import params |
|
||||||
env.set_params(params) |
|
||||||
self.install_packages(env) |
|
||||||
Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True) |
|
||||||
|
|
||||||
def configure(self, env): |
|
||||||
import params |
|
||||||
params.pika_slave = True |
|
||||||
env.set_params(params) |
|
||||||
|
|
||||||
dolphin_env() |
|
||||||
|
|
||||||
def start(self, env): |
|
||||||
import params |
|
||||||
env.set_params(params) |
|
||||||
self.configure(env) |
|
||||||
no_op_test = format("ls {dolphin_pidfile_dir}/alert-server.pid >/dev/null 2>&1 && ps `cat {dolphin_pidfile_dir}/alert-server.pid` | grep `cat {dolphin_pidfile_dir}/alert-server.pid` >/dev/null 2>&1") |
|
||||||
|
|
||||||
start_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh start alert-server") |
|
||||||
Execute(start_cmd, user=params.dolphin_user, not_if=no_op_test) |
|
||||||
|
|
||||||
def stop(self, env): |
|
||||||
import params |
|
||||||
env.set_params(params) |
|
||||||
stop_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh stop alert-server") |
|
||||||
Execute(stop_cmd, user=params.dolphin_user) |
|
||||||
time.sleep(5) |
|
||||||
|
|
||||||
def status(self, env): |
|
||||||
import status_params |
|
||||||
env.set_params(status_params) |
|
||||||
check_process_status(status_params.dolphin_run_dir + "alert-server.pid") |
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__": |
|
||||||
DolphinAlertService().execute() |
|
||||||
@ -1,70 +0,0 @@ |
|||||||
""" |
|
||||||
Licensed to the Apache Software Foundation (ASF) under one |
|
||||||
or more contributor license agreements. See the NOTICE file |
|
||||||
distributed with this work for additional information |
|
||||||
regarding copyright ownership. The ASF licenses this file |
|
||||||
to you under the Apache License, Version 2.0 (the |
|
||||||
"License"); you may not use this file except in compliance |
|
||||||
with the License. You may obtain a copy of the License at |
|
||||||
|
|
||||||
http://www.apache.org/licenses/LICENSE-2.0 |
|
||||||
|
|
||||||
Unless required by applicable law or agreed to in writing, software |
|
||||||
distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
See the License for the specific language governing permissions and |
|
||||||
limitations under the License. |
|
||||||
""" |
|
||||||
import time |
|
||||||
from resource_management import * |
|
||||||
|
|
||||||
from dolphin_env import dolphin_env |
|
||||||
|
|
||||||
|
|
||||||
class DolphinApiService(Script): |
|
||||||
def install(self, env): |
|
||||||
import params |
|
||||||
env.set_params(params) |
|
||||||
self.install_packages(env) |
|
||||||
Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True) |
|
||||||
|
|
||||||
def configure(self, env): |
|
||||||
import params |
|
||||||
params.pika_slave = True |
|
||||||
env.set_params(params) |
|
||||||
|
|
||||||
dolphin_env() |
|
||||||
|
|
||||||
def start(self, env): |
|
||||||
import params |
|
||||||
env.set_params(params) |
|
||||||
self.configure(env) |
|
||||||
|
|
||||||
#init |
|
||||||
init_cmd=format("sh " + params.dolphin_home + "/script/create-dolphinscheduler.sh") |
|
||||||
Execute(init_cmd, user=params.dolphin_user) |
|
||||||
|
|
||||||
#upgrade |
|
||||||
upgrade_cmd=format("sh " + params.dolphin_home + "/script/upgrade-dolphinscheduler.sh") |
|
||||||
Execute(upgrade_cmd, user=params.dolphin_user) |
|
||||||
|
|
||||||
no_op_test = format("ls {dolphin_pidfile_dir}/api-server.pid >/dev/null 2>&1 && ps `cat {dolphin_pidfile_dir}/api-server.pid` | grep `cat {dolphin_pidfile_dir}/api-server.pid` >/dev/null 2>&1") |
|
||||||
|
|
||||||
start_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh start api-server") |
|
||||||
Execute(start_cmd, user=params.dolphin_user, not_if=no_op_test) |
|
||||||
|
|
||||||
def stop(self, env): |
|
||||||
import params |
|
||||||
env.set_params(params) |
|
||||||
stop_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh stop api-server") |
|
||||||
Execute(stop_cmd, user=params.dolphin_user) |
|
||||||
time.sleep(5) |
|
||||||
|
|
||||||
def status(self, env): |
|
||||||
import status_params |
|
||||||
env.set_params(status_params) |
|
||||||
check_process_status(status_params.dolphin_run_dir + "api-server.pid") |
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__": |
|
||||||
DolphinApiService().execute() |
|
||||||
@ -1,123 +0,0 @@ |
|||||||
""" |
|
||||||
Licensed to the Apache Software Foundation (ASF) under one |
|
||||||
or more contributor license agreements. See the NOTICE file |
|
||||||
distributed with this work for additional information |
|
||||||
regarding copyright ownership. The ASF licenses this file |
|
||||||
to you under the Apache License, Version 2.0 (the |
|
||||||
"License"); you may not use this file except in compliance |
|
||||||
with the License. You may obtain a copy of the License at |
|
||||||
|
|
||||||
http://www.apache.org/licenses/LICENSE-2.0 |
|
||||||
|
|
||||||
Unless required by applicable law or agreed to in writing, software |
|
||||||
distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
See the License for the specific language governing permissions and |
|
||||||
limitations under the License. |
|
||||||
|
|
||||||
""" |
|
||||||
|
|
||||||
from resource_management import * |
|
||||||
|
|
||||||
|
|
||||||
def dolphin_env(): |
|
||||||
import params |
|
||||||
|
|
||||||
Directory(params.dolphin_pidfile_dir, |
|
||||||
mode=0777, |
|
||||||
owner=params.dolphin_user, |
|
||||||
group=params.dolphin_group, |
|
||||||
create_parents=True |
|
||||||
) |
|
||||||
Directory(params.dolphin_log_dir, |
|
||||||
mode=0777, |
|
||||||
owner=params.dolphin_user, |
|
||||||
group=params.dolphin_group, |
|
||||||
create_parents=True |
|
||||||
) |
|
||||||
Directory(params.dolphin_conf_dir, |
|
||||||
mode=0777, |
|
||||||
owner=params.dolphin_user, |
|
||||||
group=params.dolphin_group, |
|
||||||
create_parents=True |
|
||||||
) |
|
||||||
|
|
||||||
Directory(params.dolphin_common_map['data.basedir.path'], |
|
||||||
mode=0777, |
|
||||||
owner=params.dolphin_user, |
|
||||||
group=params.dolphin_group, |
|
||||||
create_parents=True |
|
||||||
) |
|
||||||
|
|
||||||
|
|
||||||
File(format(params.dolphin_env_path), |
|
||||||
mode=0777, |
|
||||||
content=InlineTemplate(params.dolphin_env_content), |
|
||||||
owner=params.dolphin_user, |
|
||||||
group=params.dolphin_group |
|
||||||
) |
|
||||||
|
|
||||||
|
|
||||||
File(format(params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh"), |
|
||||||
mode=0755, |
|
||||||
content=Template("dolphin-daemon.sh.j2"), |
|
||||||
owner=params.dolphin_user, |
|
||||||
group=params.dolphin_group |
|
||||||
) |
|
||||||
|
|
||||||
File(format(params.dolphin_conf_dir + "/master.properties"), |
|
||||||
mode=0755, |
|
||||||
content=Template("master.properties.j2"), |
|
||||||
owner=params.dolphin_user, |
|
||||||
group=params.dolphin_group |
|
||||||
) |
|
||||||
|
|
||||||
File(format(params.dolphin_conf_dir + "/worker.properties"), |
|
||||||
mode=0755, |
|
||||||
content=Template("worker.properties.j2"), |
|
||||||
owner=params.dolphin_user, |
|
||||||
group=params.dolphin_group |
|
||||||
) |
|
||||||
|
|
||||||
|
|
||||||
File(format(params.dolphin_conf_dir + "/alert.properties"), |
|
||||||
mode=0755, |
|
||||||
content=Template("alert.properties.j2"), |
|
||||||
owner=params.dolphin_user, |
|
||||||
group=params.dolphin_group |
|
||||||
) |
|
||||||
|
|
||||||
File(format(params.dolphin_conf_dir + "/datasource.properties"), |
|
||||||
mode=0755, |
|
||||||
content=Template("datasource.properties.j2"), |
|
||||||
owner=params.dolphin_user, |
|
||||||
group=params.dolphin_group |
|
||||||
) |
|
||||||
|
|
||||||
File(format(params.dolphin_conf_dir + "/application-api.properties"), |
|
||||||
mode=0755, |
|
||||||
content=Template("application-api.properties.j2"), |
|
||||||
owner=params.dolphin_user, |
|
||||||
group=params.dolphin_group |
|
||||||
) |
|
||||||
|
|
||||||
File(format(params.dolphin_conf_dir + "/common.properties"), |
|
||||||
mode=0755, |
|
||||||
content=Template("common.properties.j2"), |
|
||||||
owner=params.dolphin_user, |
|
||||||
group=params.dolphin_group |
|
||||||
) |
|
||||||
|
|
||||||
File(format(params.dolphin_conf_dir + "/quartz.properties"), |
|
||||||
mode=0755, |
|
||||||
content=Template("quartz.properties.j2"), |
|
||||||
owner=params.dolphin_user, |
|
||||||
group=params.dolphin_group |
|
||||||
) |
|
||||||
|
|
||||||
File(format(params.dolphin_conf_dir + "/zookeeper.properties"), |
|
||||||
mode=0755, |
|
||||||
content=Template("zookeeper.properties.j2"), |
|
||||||
owner=params.dolphin_user, |
|
||||||
group=params.dolphin_group |
|
||||||
) |
|
||||||
@ -1,61 +0,0 @@ |
|||||||
""" |
|
||||||
Licensed to the Apache Software Foundation (ASF) under one |
|
||||||
or more contributor license agreements. See the NOTICE file |
|
||||||
distributed with this work for additional information |
|
||||||
regarding copyright ownership. The ASF licenses this file |
|
||||||
to you under the Apache License, Version 2.0 (the |
|
||||||
"License"); you may not use this file except in compliance |
|
||||||
with the License. You may obtain a copy of the License at |
|
||||||
|
|
||||||
http://www.apache.org/licenses/LICENSE-2.0 |
|
||||||
|
|
||||||
Unless required by applicable law or agreed to in writing, software |
|
||||||
distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
See the License for the specific language governing permissions and |
|
||||||
limitations under the License. |
|
||||||
""" |
|
||||||
import time |
|
||||||
from resource_management import * |
|
||||||
|
|
||||||
from dolphin_env import dolphin_env |
|
||||||
|
|
||||||
|
|
||||||
class DolphinLoggerService(Script): |
|
||||||
def install(self, env): |
|
||||||
import params |
|
||||||
env.set_params(params) |
|
||||||
self.install_packages(env) |
|
||||||
Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True) |
|
||||||
|
|
||||||
def configure(self, env): |
|
||||||
import params |
|
||||||
params.pika_slave = True |
|
||||||
env.set_params(params) |
|
||||||
|
|
||||||
dolphin_env() |
|
||||||
|
|
||||||
def start(self, env): |
|
||||||
import params |
|
||||||
env.set_params(params) |
|
||||||
self.configure(env) |
|
||||||
no_op_test = format("ls {dolphin_pidfile_dir}/logger-server.pid >/dev/null 2>&1 && ps `cat {dolphin_pidfile_dir}/logger-server.pid` | grep `cat {dolphin_pidfile_dir}/logger-server.pid` >/dev/null 2>&1") |
|
||||||
|
|
||||||
start_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh start logger-server") |
|
||||||
Execute(start_cmd, user=params.dolphin_user, not_if=no_op_test) |
|
||||||
|
|
||||||
def stop(self, env): |
|
||||||
import params |
|
||||||
env.set_params(params) |
|
||||||
stop_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh stop logger-server") |
|
||||||
Execute(stop_cmd, user=params.dolphin_user) |
|
||||||
time.sleep(5) |
|
||||||
|
|
||||||
def status(self, env): |
|
||||||
import status_params |
|
||||||
env.set_params(status_params) |
|
||||||
check_process_status(status_params.dolphin_run_dir + "logger-server.pid") |
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__": |
|
||||||
DolphinLoggerService().execute() |
|
||||||
@ -1,61 +0,0 @@ |
|||||||
# -*- coding: utf-8 -*- |
|
||||||
""" |
|
||||||
Licensed to the Apache Software Foundation (ASF) under one |
|
||||||
or more contributor license agreements. See the NOTICE file |
|
||||||
distributed with this work for additional information |
|
||||||
regarding copyright ownership. The ASF licenses this file |
|
||||||
to you under the Apache License, Version 2.0 (the |
|
||||||
"License"); you may not use this file except in compliance |
|
||||||
with the License. You may obtain a copy of the License at |
|
||||||
|
|
||||||
http://www.apache.org/licenses/LICENSE-2.0 |
|
||||||
|
|
||||||
Unless required by applicable law or agreed to in writing, software |
|
||||||
distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
See the License for the specific language governing permissions and |
|
||||||
limitations under the License. |
|
||||||
""" |
|
||||||
import time |
|
||||||
from resource_management import * |
|
||||||
|
|
||||||
from dolphin_env import dolphin_env |
|
||||||
|
|
||||||
|
|
||||||
class DolphinMasterService(Script): |
|
||||||
def install(self, env): |
|
||||||
import params |
|
||||||
env.set_params(params) |
|
||||||
self.install_packages(env) |
|
||||||
Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True) |
|
||||||
|
|
||||||
def configure(self, env): |
|
||||||
import params |
|
||||||
params.pika_slave = True |
|
||||||
env.set_params(params) |
|
||||||
|
|
||||||
dolphin_env() |
|
||||||
|
|
||||||
def start(self, env): |
|
||||||
import params |
|
||||||
env.set_params(params) |
|
||||||
self.configure(env) |
|
||||||
no_op_test = format("ls {dolphin_pidfile_dir}/master-server.pid >/dev/null 2>&1 && ps `cat {dolphin_pidfile_dir}/master-server.pid` | grep `cat {dolphin_pidfile_dir}/master-server.pid` >/dev/null 2>&1") |
|
||||||
start_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh start master-server") |
|
||||||
Execute(start_cmd, user=params.dolphin_user, not_if=no_op_test) |
|
||||||
|
|
||||||
def stop(self, env): |
|
||||||
import params |
|
||||||
env.set_params(params) |
|
||||||
stop_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh stop master-server") |
|
||||||
Execute(stop_cmd, user=params.dolphin_user) |
|
||||||
time.sleep(5) |
|
||||||
|
|
||||||
def status(self, env): |
|
||||||
import status_params |
|
||||||
env.set_params(status_params) |
|
||||||
check_process_status(status_params.dolphin_run_dir + "master-server.pid") |
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__": |
|
||||||
DolphinMasterService().execute() |
|
||||||
@ -1,60 +0,0 @@ |
|||||||
""" |
|
||||||
Licensed to the Apache Software Foundation (ASF) under one |
|
||||||
or more contributor license agreements. See the NOTICE file |
|
||||||
distributed with this work for additional information |
|
||||||
regarding copyright ownership. The ASF licenses this file |
|
||||||
to you under the Apache License, Version 2.0 (the |
|
||||||
"License"); you may not use this file except in compliance |
|
||||||
with the License. You may obtain a copy of the License at |
|
||||||
|
|
||||||
http://www.apache.org/licenses/LICENSE-2.0 |
|
||||||
|
|
||||||
Unless required by applicable law or agreed to in writing, software |
|
||||||
distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
See the License for the specific language governing permissions and |
|
||||||
limitations under the License. |
|
||||||
""" |
|
||||||
import time |
|
||||||
from resource_management import * |
|
||||||
|
|
||||||
from dolphin_env import dolphin_env |
|
||||||
|
|
||||||
|
|
||||||
class DolphinWorkerService(Script): |
|
||||||
def install(self, env): |
|
||||||
import params |
|
||||||
env.set_params(params) |
|
||||||
self.install_packages(env) |
|
||||||
Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True) |
|
||||||
|
|
||||||
def configure(self, env): |
|
||||||
import params |
|
||||||
params.pika_slave = True |
|
||||||
env.set_params(params) |
|
||||||
|
|
||||||
dolphin_env() |
|
||||||
|
|
||||||
def start(self, env): |
|
||||||
import params |
|
||||||
env.set_params(params) |
|
||||||
self.configure(env) |
|
||||||
no_op_test = format("ls {dolphin_pidfile_dir}/worker-server.pid >/dev/null 2>&1 && ps `cat {dolphin_pidfile_dir}/worker-server.pid` | grep `cat {dolphin_pidfile_dir}/worker-server.pid` >/dev/null 2>&1") |
|
||||||
start_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh start worker-server") |
|
||||||
Execute(start_cmd, user=params.dolphin_user, not_if=no_op_test) |
|
||||||
|
|
||||||
def stop(self, env): |
|
||||||
import params |
|
||||||
env.set_params(params) |
|
||||||
stop_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh stop worker-server") |
|
||||||
Execute(stop_cmd, user=params.dolphin_user) |
|
||||||
time.sleep(5) |
|
||||||
|
|
||||||
def status(self, env): |
|
||||||
import status_params |
|
||||||
env.set_params(status_params) |
|
||||||
check_process_status(status_params.dolphin_run_dir + "worker-server.pid") |
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__": |
|
||||||
DolphinWorkerService().execute() |
|
||||||
@ -1,154 +0,0 @@ |
|||||||
""" |
|
||||||
Licensed to the Apache Software Foundation (ASF) under one |
|
||||||
or more contributor license agreements. See the NOTICE file |
|
||||||
distributed with this work for additional information |
|
||||||
regarding copyright ownership. The ASF licenses this file |
|
||||||
to you under the Apache License, Version 2.0 (the |
|
||||||
"License"); you may not use this file except in compliance |
|
||||||
with the License. You may obtain a copy of the License at |
|
||||||
|
|
||||||
http://www.apache.org/licenses/LICENSE-2.0 |
|
||||||
|
|
||||||
Unless required by applicable law or agreed to in writing, software |
|
||||||
distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
See the License for the specific language governing permissions and |
|
||||||
limitations under the License. |
|
||||||
""" |
|
||||||
|
|
||||||
|
|
||||||
import sys |
|
||||||
from resource_management import * |
|
||||||
from resource_management.core.logger import Logger |
|
||||||
from resource_management.libraries.functions import default |
|
||||||
|
|
||||||
Logger.initialize_logger() |
|
||||||
reload(sys) |
|
||||||
sys.setdefaultencoding('utf-8') |
|
||||||
|
|
||||||
# server configurations |
|
||||||
config = Script.get_config() |
|
||||||
|
|
||||||
# conf_dir = "/etc/" |
|
||||||
dolphin_home = "/opt/soft/dolphinscheduler" |
|
||||||
dolphin_conf_dir = dolphin_home + "/conf" |
|
||||||
dolphin_log_dir = dolphin_home + "/logs" |
|
||||||
dolphin_bin_dir = dolphin_home + "/bin" |
|
||||||
dolphin_lib_jars = dolphin_home + "/lib/*" |
|
||||||
dolphin_pidfile_dir = "/opt/soft/run/dolphinscheduler" |
|
||||||
|
|
||||||
rmHosts = default("/clusterHostInfo/rm_host", []) |
|
||||||
|
|
||||||
# dolphin-env |
|
||||||
dolphin_env_map = {} |
|
||||||
dolphin_env_map.update(config['configurations']['dolphin-env']) |
|
||||||
|
|
||||||
# which user to install and admin dolphin scheduler |
|
||||||
dolphin_user = dolphin_env_map['dolphin.user'] |
|
||||||
dolphin_group = dolphin_env_map['dolphin.group'] |
|
||||||
|
|
||||||
# .dolphinscheduler_env.sh |
|
||||||
dolphin_env_path = dolphin_conf_dir + '/env/dolphinscheduler_env.sh' |
|
||||||
dolphin_env_content = dolphin_env_map['dolphinscheduler-env-content'] |
|
||||||
|
|
||||||
# database config |
|
||||||
dolphin_database_config = {} |
|
||||||
dolphin_database_config['dolphin_database_type'] = dolphin_env_map['dolphin.database.type'] |
|
||||||
dolphin_database_config['dolphin_database_username'] = dolphin_env_map['dolphin.database.username'] |
|
||||||
dolphin_database_config['dolphin_database_password'] = dolphin_env_map['dolphin.database.password'] |
|
||||||
if 'mysql' == dolphin_database_config['dolphin_database_type']: |
|
||||||
dolphin_database_config['dolphin_database_driver'] = 'com.mysql.jdbc.Driver' |
|
||||||
dolphin_database_config['driverDelegateClass'] = 'org.quartz.impl.jdbcjobstore.StdJDBCDelegate' |
|
||||||
dolphin_database_config['dolphin_database_url'] = 'jdbc:mysql://' + dolphin_env_map['dolphin.database.host'] \ |
|
||||||
+ ':' + dolphin_env_map['dolphin.database.port'] \ |
|
||||||
+ '/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8' |
|
||||||
else: |
|
||||||
dolphin_database_config['dolphin_database_driver'] = 'org.postgresql.Driver' |
|
||||||
dolphin_database_config['driverDelegateClass'] = 'org.quartz.impl.jdbcjobstore.PostgreSQLDelegate' |
|
||||||
dolphin_database_config['dolphin_database_url'] = 'jdbc:postgresql://' + dolphin_env_map['dolphin.database.host'] \ |
|
||||||
+ ':' + dolphin_env_map['dolphin.database.port'] \ |
|
||||||
+ '/dolphinscheduler' |
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
# application-alert.properties |
|
||||||
dolphin_alert_map = {} |
|
||||||
wechat_push_url = 'https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=$token' |
|
||||||
wechat_token_url = 'https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid=$corpId&corpsecret=$secret' |
|
||||||
wechat_team_send_msg = '{\"toparty\":\"$toParty\",\"agentid\":\"$agentId\",\"msgtype\":\"text\",\"text\":{\"content\":\"$msg\"},\"safe\":\"0\"}' |
|
||||||
wechat_user_send_msg = '{\"touser\":\"$toUser\",\"agentid\":\"$agentId\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"$msg\"}}' |
|
||||||
|
|
||||||
dolphin_alert_config_map = config['configurations']['dolphin-alert'] |
|
||||||
|
|
||||||
if dolphin_alert_config_map['enterprise.wechat.enable']: |
|
||||||
dolphin_alert_map['enterprise.wechat.push.ur'] = wechat_push_url |
|
||||||
dolphin_alert_map['enterprise.wechat.token.url'] = wechat_token_url |
|
||||||
dolphin_alert_map['enterprise.wechat.team.send.msg'] = wechat_team_send_msg |
|
||||||
dolphin_alert_map['enterprise.wechat.user.send.msg'] = wechat_user_send_msg |
|
||||||
|
|
||||||
dolphin_alert_map.update(dolphin_alert_config_map) |
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
# application-api.properties |
|
||||||
dolphin_app_api_map = {} |
|
||||||
dolphin_app_api_map.update(config['configurations']['dolphin-application-api']) |
|
||||||
|
|
||||||
|
|
||||||
# common.properties |
|
||||||
dolphin_common_map = {} |
|
||||||
|
|
||||||
if 'yarn-site' in config['configurations'] and \ |
|
||||||
'yarn.resourcemanager.webapp.address' in config['configurations']['yarn-site']: |
|
||||||
yarn_resourcemanager_webapp_address = config['configurations']['yarn-site']['yarn.resourcemanager.webapp.address'] |
|
||||||
yarn_application_status_address = 'http://' + yarn_resourcemanager_webapp_address + '/ws/v1/cluster/apps/%s' |
|
||||||
dolphin_common_map['yarn.application.status.address'] = yarn_application_status_address |
|
||||||
|
|
||||||
rmHosts = default("/clusterHostInfo/rm_host", []) |
|
||||||
if len(rmHosts) > 1: |
|
||||||
dolphin_common_map['yarn.resourcemanager.ha.rm.ids'] = ','.join(rmHosts) |
|
||||||
else: |
|
||||||
dolphin_common_map['yarn.resourcemanager.ha.rm.ids'] = '' |
|
||||||
|
|
||||||
dolphin_common_map_tmp = config['configurations']['dolphin-common'] |
|
||||||
data_basedir_path = dolphin_common_map_tmp['data.basedir.path'] |
|
||||||
process_exec_basepath = data_basedir_path + '/exec' |
|
||||||
data_download_basedir_path = data_basedir_path + '/download' |
|
||||||
dolphin_common_map['process.exec.basepath'] = process_exec_basepath |
|
||||||
dolphin_common_map['data.download.basedir.path'] = data_download_basedir_path |
|
||||||
dolphin_common_map['dolphinscheduler.env.path'] = dolphin_env_path |
|
||||||
dolphin_common_map.update(config['configurations']['dolphin-common']) |
|
||||||
|
|
||||||
# datasource.properties |
|
||||||
dolphin_datasource_map = {} |
|
||||||
dolphin_datasource_map['spring.datasource.type'] = 'com.alibaba.druid.pool.DruidDataSource' |
|
||||||
dolphin_datasource_map['spring.datasource.driver-class-name'] = dolphin_database_config['dolphin_database_driver'] |
|
||||||
dolphin_datasource_map['spring.datasource.url'] = dolphin_database_config['dolphin_database_url'] |
|
||||||
dolphin_datasource_map['spring.datasource.username'] = dolphin_database_config['dolphin_database_username'] |
|
||||||
dolphin_datasource_map['spring.datasource.password'] = dolphin_database_config['dolphin_database_password'] |
|
||||||
dolphin_datasource_map.update(config['configurations']['dolphin-datasource']) |
|
||||||
|
|
||||||
# master.properties |
|
||||||
dolphin_master_map = config['configurations']['dolphin-master'] |
|
||||||
|
|
||||||
# quartz.properties |
|
||||||
dolphin_quartz_map = {} |
|
||||||
dolphin_quartz_map['org.quartz.jobStore.driverDelegateClass'] = dolphin_database_config['driverDelegateClass'] |
|
||||||
dolphin_quartz_map.update(config['configurations']['dolphin-quartz']) |
|
||||||
|
|
||||||
# worker.properties |
|
||||||
dolphin_worker_map = config['configurations']['dolphin-worker'] |
|
||||||
|
|
||||||
# zookeeper.properties |
|
||||||
dolphin_zookeeper_map={} |
|
||||||
zookeeperHosts = default("/clusterHostInfo/zookeeper_hosts", []) |
|
||||||
if len(zookeeperHosts) > 0 and "clientPort" in config['configurations']['zoo.cfg']: |
|
||||||
clientPort = config['configurations']['zoo.cfg']['clientPort'] |
|
||||||
zookeeperPort = ":" + clientPort + "," |
|
||||||
dolphin_zookeeper_map['zookeeper.quorum'] = zookeeperPort.join(zookeeperHosts) + ":" + clientPort |
|
||||||
dolphin_zookeeper_map.update(config['configurations']['dolphin-zookeeper']) |
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@ -1,31 +0,0 @@ |
|||||||
""" |
|
||||||
Licensed to the Apache Software Foundation (ASF) under one |
|
||||||
or more contributor license agreements. See the NOTICE file |
|
||||||
distributed with this work for additional information |
|
||||||
regarding copyright ownership. The ASF licenses this file |
|
||||||
to you under the Apache License, Version 2.0 (the |
|
||||||
"License"); you may not use this file except in compliance |
|
||||||
with the License. You may obtain a copy of the License at |
|
||||||
|
|
||||||
http://www.apache.org/licenses/LICENSE-2.0 |
|
||||||
|
|
||||||
Unless required by applicable law or agreed to in writing, software |
|
||||||
distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
See the License for the specific language governing permissions and |
|
||||||
limitations under the License. |
|
||||||
|
|
||||||
""" |
|
||||||
|
|
||||||
from resource_management import * |
|
||||||
from resource_management.libraries.functions import get_unique_id_and_date |
|
||||||
|
|
||||||
class ServiceCheck(Script): |
|
||||||
def service_check(self, env): |
|
||||||
import params |
|
||||||
#env.set_params(params) |
|
||||||
|
|
||||||
# Execute(format("which pika_server")) |
|
||||||
|
|
||||||
if __name__ == "__main__": |
|
||||||
ServiceCheck().execute() |
|
||||||
@ -1,23 +0,0 @@ |
|||||||
""" |
|
||||||
Licensed to the Apache Software Foundation (ASF) under one |
|
||||||
or more contributor license agreements. See the NOTICE file |
|
||||||
distributed with this work for additional information |
|
||||||
regarding copyright ownership. The ASF licenses this file |
|
||||||
to you under the Apache License, Version 2.0 (the |
|
||||||
"License"); you may not use this file except in compliance |
|
||||||
with the License. You may obtain a copy of the License at |
|
||||||
|
|
||||||
http://www.apache.org/licenses/LICENSE-2.0 |
|
||||||
|
|
||||||
Unless required by applicable law or agreed to in writing, software |
|
||||||
distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
See the License for the specific language governing permissions and |
|
||||||
limitations under the License. |
|
||||||
""" |
|
||||||
|
|
||||||
from resource_management import * |
|
||||||
|
|
||||||
config = Script.get_config() |
|
||||||
|
|
||||||
dolphin_run_dir = "/opt/soft/run/dolphinscheduler/" |
|
||||||
@ -1,20 +0,0 @@ |
|||||||
# |
|
||||||
# Licensed to the Apache Software Foundation (ASF) under one or more |
|
||||||
# contributor license agreements. See the NOTICE file distributed with |
|
||||||
# this work for additional information regarding copyright ownership. |
|
||||||
# The ASF licenses this file to You under the Apache License, Version 2.0 |
|
||||||
# (the "License"); you may not use this file except in compliance with |
|
||||||
# the License. You may obtain a copy of the License at |
|
||||||
# |
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0 |
|
||||||
# |
|
||||||
# Unless required by applicable law or agreed to in writing, software |
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
# See the License for the specific language governing permissions and |
|
||||||
# limitations under the License. |
|
||||||
# |
|
||||||
|
|
||||||
{% for key, value in dolphin_app_api_map.iteritems() -%} |
|
||||||
{{key}}={{value}} |
|
||||||
{% endfor %} |
|
||||||
@ -1,20 +0,0 @@ |
|||||||
# |
|
||||||
# Licensed to the Apache Software Foundation (ASF) under one or more |
|
||||||
# contributor license agreements. See the NOTICE file distributed with |
|
||||||
# this work for additional information regarding copyright ownership. |
|
||||||
# The ASF licenses this file to You under the Apache License, Version 2.0 |
|
||||||
# (the "License"); you may not use this file except in compliance with |
|
||||||
# the License. You may obtain a copy of the License at |
|
||||||
# |
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0 |
|
||||||
# |
|
||||||
# Unless required by applicable law or agreed to in writing, software |
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
# See the License for the specific language governing permissions and |
|
||||||
# limitations under the License. |
|
||||||
# |
|
||||||
|
|
||||||
{% for key, value in dolphin_common_map.iteritems() -%} |
|
||||||
{{key}}={{value}} |
|
||||||
{% endfor %} |
|
||||||
@ -1,20 +0,0 @@ |
|||||||
# |
|
||||||
# Licensed to the Apache Software Foundation (ASF) under one or more |
|
||||||
# contributor license agreements. See the NOTICE file distributed with |
|
||||||
# this work for additional information regarding copyright ownership. |
|
||||||
# The ASF licenses this file to You under the Apache License, Version 2.0 |
|
||||||
# (the "License"); you may not use this file except in compliance with |
|
||||||
# the License. You may obtain a copy of the License at |
|
||||||
# |
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0 |
|
||||||
# |
|
||||||
# Unless required by applicable law or agreed to in writing, software |
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
# See the License for the specific language governing permissions and |
|
||||||
# limitations under the License. |
|
||||||
# |
|
||||||
|
|
||||||
{% for key, value in dolphin_datasource_map.iteritems() -%} |
|
||||||
{{key}}={{value}} |
|
||||||
{% endfor %} |
|
||||||
@ -1,116 +0,0 @@ |
|||||||
#!/bin/sh |
|
||||||
# |
|
||||||
# Licensed to the Apache Software Foundation (ASF) under one or more |
|
||||||
# contributor license agreements. See the NOTICE file distributed with |
|
||||||
# this work for additional information regarding copyright ownership. |
|
||||||
# The ASF licenses this file to You under the Apache License, Version 2.0 |
|
||||||
# (the "License"); you may not use this file except in compliance with |
|
||||||
# the License. You may obtain a copy of the License at |
|
||||||
# |
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0 |
|
||||||
# |
|
||||||
# Unless required by applicable law or agreed to in writing, software |
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
# See the License for the specific language governing permissions and |
|
||||||
# limitations under the License. |
|
||||||
# |
|
||||||
|
|
||||||
usage="Usage: dolphinscheduler-daemon.sh (start|stop) <command> " |
|
||||||
|
|
||||||
# if no args specified, show usage |
|
||||||
if [ $# -le 1 ]; then |
|
||||||
echo $usage |
|
||||||
exit 1 |
|
||||||
fi |
|
||||||
|
|
||||||
startStop=$1 |
|
||||||
shift |
|
||||||
command=$1 |
|
||||||
shift |
|
||||||
|
|
||||||
echo "Begin $startStop $command......" |
|
||||||
|
|
||||||
BIN_DIR=`dirname $0` |
|
||||||
BIN_DIR=`cd "$BIN_DIR"; pwd` |
|
||||||
DOLPHINSCHEDULER_HOME=$BIN_DIR/.. |
|
||||||
|
|
||||||
export HOSTNAME=`hostname` |
|
||||||
|
|
||||||
DOLPHINSCHEDULER_LIB_JARS={{dolphin_lib_jars}} |
|
||||||
|
|
||||||
DOLPHINSCHEDULER_OPTS="-server -Xmx16g -Xms4g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70" |
|
||||||
STOP_TIMEOUT=5 |
|
||||||
|
|
||||||
log={{dolphin_log_dir}}/dolphinscheduler-$command-$HOSTNAME.out |
|
||||||
pid={{dolphin_pidfile_dir}}/$command.pid |
|
||||||
|
|
||||||
cd $DOLPHINSCHEDULER_HOME |
|
||||||
|
|
||||||
if [ "$command" = "api-server" ]; then |
|
||||||
LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/logback-api.xml -Dspring.profiles.active=api" |
|
||||||
CLASS=org.apache.dolphinscheduler.api.ApiApplicationServer |
|
||||||
elif [ "$command" = "master-server" ]; then |
|
||||||
LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/logback-master.xml -Ddruid.mysql.usePingMethod=false" |
|
||||||
CLASS=org.apache.dolphinscheduler.server.master.MasterServer |
|
||||||
elif [ "$command" = "worker-server" ]; then |
|
||||||
LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/logback-worker.xml -Ddruid.mysql.usePingMethod=false" |
|
||||||
CLASS=org.apache.dolphinscheduler.server.worker.WorkerServer |
|
||||||
elif [ "$command" = "alert-server" ]; then |
|
||||||
LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/logback-alert.xml" |
|
||||||
CLASS=org.apache.dolphinscheduler.alert.AlertServer |
|
||||||
elif [ "$command" = "logger-server" ]; then |
|
||||||
CLASS=org.apache.dolphinscheduler.server.log.LoggerServer |
|
||||||
else |
|
||||||
echo "Error: No command named \`$command' was found." |
|
||||||
exit 1 |
|
||||||
fi |
|
||||||
|
|
||||||
case $startStop in |
|
||||||
(start) |
|
||||||
|
|
||||||
if [ -f $pid ]; then |
|
||||||
if kill -0 `cat $pid` > /dev/null 2>&1; then |
|
||||||
echo $command running as process `cat $pid`. Stop it first. |
|
||||||
exit 1 |
|
||||||
fi |
|
||||||
fi |
|
||||||
|
|
||||||
echo starting $command, logging to $log |
|
||||||
|
|
||||||
exec_command="$LOG_FILE $DOLPHINSCHEDULER_OPTS -classpath {{dolphin_conf_dir}}:{{dolphin_lib_jars}} $CLASS" |
|
||||||
|
|
||||||
echo "nohup java $exec_command > $log 2>&1 < /dev/null &" |
|
||||||
nohup java $exec_command > $log 2>&1 < /dev/null & |
|
||||||
echo $! > $pid |
|
||||||
;; |
|
||||||
|
|
||||||
(stop) |
|
||||||
|
|
||||||
if [ -f $pid ]; then |
|
||||||
TARGET_PID=`cat $pid` |
|
||||||
if kill -0 $TARGET_PID > /dev/null 2>&1; then |
|
||||||
echo stopping $command |
|
||||||
kill $TARGET_PID |
|
||||||
sleep $STOP_TIMEOUT |
|
||||||
if kill -0 $TARGET_PID > /dev/null 2>&1; then |
|
||||||
echo "$command did not stop gracefully after $STOP_TIMEOUT seconds: killing with kill -9" |
|
||||||
kill -9 $TARGET_PID |
|
||||||
fi |
|
||||||
else |
|
||||||
echo no $command to stop |
|
||||||
fi |
|
||||||
rm -f $pid |
|
||||||
else |
|
||||||
echo no $command to stop |
|
||||||
fi |
|
||||||
;; |
|
||||||
|
|
||||||
(*) |
|
||||||
echo $usage |
|
||||||
exit 1 |
|
||||||
;; |
|
||||||
|
|
||||||
esac |
|
||||||
|
|
||||||
echo "End $startStop $command." |
|
||||||
@ -1,20 +0,0 @@ |
|||||||
# |
|
||||||
# Licensed to the Apache Software Foundation (ASF) under one or more |
|
||||||
# contributor license agreements. See the NOTICE file distributed with |
|
||||||
# this work for additional information regarding copyright ownership. |
|
||||||
# The ASF licenses this file to You under the Apache License, Version 2.0 |
|
||||||
# (the "License"); you may not use this file except in compliance with |
|
||||||
# the License. You may obtain a copy of the License at |
|
||||||
# |
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0 |
|
||||||
# |
|
||||||
# Unless required by applicable law or agreed to in writing, software |
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
# See the License for the specific language governing permissions and |
|
||||||
# limitations under the License. |
|
||||||
# |
|
||||||
|
|
||||||
{% for key, value in dolphin_master_map.iteritems() -%} |
|
||||||
{{key}}={{value}} |
|
||||||
{% endfor %} |
|
||||||
@ -1,20 +0,0 @@ |
|||||||
# |
|
||||||
# Licensed to the Apache Software Foundation (ASF) under one or more |
|
||||||
# contributor license agreements. See the NOTICE file distributed with |
|
||||||
# this work for additional information regarding copyright ownership. |
|
||||||
# The ASF licenses this file to You under the Apache License, Version 2.0 |
|
||||||
# (the "License"); you may not use this file except in compliance with |
|
||||||
# the License. You may obtain a copy of the License at |
|
||||||
# |
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0 |
|
||||||
# |
|
||||||
# Unless required by applicable law or agreed to in writing, software |
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
# See the License for the specific language governing permissions and |
|
||||||
# limitations under the License. |
|
||||||
# |
|
||||||
|
|
||||||
{% for key, value in dolphin_quartz_map.iteritems() -%} |
|
||||||
{{key}}={{value}} |
|
||||||
{% endfor %} |
|
||||||
@ -1,20 +0,0 @@ |
|||||||
# |
|
||||||
# Licensed to the Apache Software Foundation (ASF) under one or more |
|
||||||
# contributor license agreements. See the NOTICE file distributed with |
|
||||||
# this work for additional information regarding copyright ownership. |
|
||||||
# The ASF licenses this file to You under the Apache License, Version 2.0 |
|
||||||
# (the "License"); you may not use this file except in compliance with |
|
||||||
# the License. You may obtain a copy of the License at |
|
||||||
# |
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0 |
|
||||||
# |
|
||||||
# Unless required by applicable law or agreed to in writing, software |
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
# See the License for the specific language governing permissions and |
|
||||||
# limitations under the License. |
|
||||||
# |
|
||||||
|
|
||||||
{% for key, value in dolphin_worker_map.iteritems() -%} |
|
||||||
{{key}}={{value}} |
|
||||||
{% endfor %} |
|
||||||
@ -1,20 +0,0 @@ |
|||||||
# |
|
||||||
# Licensed to the Apache Software Foundation (ASF) under one or more |
|
||||||
# contributor license agreements. See the NOTICE file distributed with |
|
||||||
# this work for additional information regarding copyright ownership. |
|
||||||
# The ASF licenses this file to You under the Apache License, Version 2.0 |
|
||||||
# (the "License"); you may not use this file except in compliance with |
|
||||||
# the License. You may obtain a copy of the License at |
|
||||||
# |
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0 |
|
||||||
# |
|
||||||
# Unless required by applicable law or agreed to in writing, software |
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
# See the License for the specific language governing permissions and |
|
||||||
# limitations under the License. |
|
||||||
# |
|
||||||
|
|
||||||
{% for key, value in dolphin_zookeeper_map.iteritems() -%} |
|
||||||
{{key}}={{value}} |
|
||||||
{% endfor %} |
|
||||||
@ -1,26 +0,0 @@ |
|||||||
{ |
|
||||||
"name": "default", |
|
||||||
"description": "default quick links configuration", |
|
||||||
"configuration": { |
|
||||||
"protocol": |
|
||||||
{ |
|
||||||
"type":"http" |
|
||||||
}, |
|
||||||
|
|
||||||
"links": [ |
|
||||||
{ |
|
||||||
"name": "dolphin-application-ui", |
|
||||||
"label": "DolphinApplication UI", |
|
||||||
"requires_user_name": "false", |
|
||||||
"component_name": "DOLPHIN_API", |
|
||||||
"url": "%@://%@:%@/dolphinscheduler/ui/view/login/index.html", |
|
||||||
"port":{ |
|
||||||
"http_property": "server.port", |
|
||||||
"http_default_port": "12345", |
|
||||||
"regex": "^(\\d+)$", |
|
||||||
"site": "dolphin-application-api" |
|
||||||
} |
|
||||||
} |
|
||||||
] |
|
||||||
} |
|
||||||
} |
|
||||||
@ -1,661 +0,0 @@ |
|||||||
{ |
|
||||||
"name": "default", |
|
||||||
"description": "Default theme for Dolphin Scheduler service", |
|
||||||
"configuration": { |
|
||||||
"layouts": [ |
|
||||||
{ |
|
||||||
"name": "default", |
|
||||||
"tabs": [ |
|
||||||
{ |
|
||||||
"name": "settings", |
|
||||||
"display-name": "Settings", |
|
||||||
"layout": { |
|
||||||
"tab-rows": "3", |
|
||||||
"tab-columns": "3", |
|
||||||
"sections": [ |
|
||||||
{ |
|
||||||
"name": "dolphin-env-config", |
|
||||||
"display-name": "Dolphin Env Config", |
|
||||||
"row-index": "0", |
|
||||||
"column-index": "0", |
|
||||||
"row-span": "1", |
|
||||||
"column-span": "2", |
|
||||||
"section-rows": "1", |
|
||||||
"section-columns": "2", |
|
||||||
"subsections": [ |
|
||||||
{ |
|
||||||
"name": "env-row1-col1", |
|
||||||
"display-name": "Deploy User Info", |
|
||||||
"row-index": "0", |
|
||||||
"column-index": "0", |
|
||||||
"row-span": "1", |
|
||||||
"column-span": "1" |
|
||||||
}, |
|
||||||
{ |
|
||||||
"name": "env-row1-col2", |
|
||||||
"display-name": "System Env Optimization", |
|
||||||
"row-index": "0", |
|
||||||
"column-index": "1", |
|
||||||
"row-span": "1", |
|
||||||
"column-span": "1" |
|
||||||
} |
|
||||||
] |
|
||||||
}, |
|
||||||
{ |
|
||||||
"name": "dolphin-database-config", |
|
||||||
"display-name": "Database Config", |
|
||||||
"row-index": "1", |
|
||||||
"column-index": "0", |
|
||||||
"row-span": "1", |
|
||||||
"column-span": "2", |
|
||||||
"section-rows": "1", |
|
||||||
"section-columns": "3", |
|
||||||
"subsections": [ |
|
||||||
{ |
|
||||||
"name": "database-row1-col1", |
|
||||||
"row-index": "0", |
|
||||||
"column-index": "0", |
|
||||||
"row-span": "1", |
|
||||||
"column-span": "1" |
|
||||||
}, |
|
||||||
{ |
|
||||||
"name": "database-row1-col2", |
|
||||||
"row-index": "0", |
|
||||||
"column-index": "1", |
|
||||||
"row-span": "1", |
|
||||||
"column-span": "1" |
|
||||||
}, |
|
||||||
{ |
|
||||||
"name": "database-row1-col3", |
|
||||||
"row-index": "0", |
|
||||||
"column-index": "2", |
|
||||||
"row-span": "1", |
|
||||||
"column-span": "1" |
|
||||||
} |
|
||||||
] |
|
||||||
}, |
|
||||||
{ |
|
||||||
"name": "dynamic-config", |
|
||||||
"row-index": "2", |
|
||||||
"column-index": "0", |
|
||||||
"row-span": "1", |
|
||||||
"column-span": "2", |
|
||||||
"section-rows": "1", |
|
||||||
"section-columns": "3", |
|
||||||
"subsections": [ |
|
||||||
{ |
|
||||||
"name": "dynamic-row1-col1", |
|
||||||
"display-name": "Resource FS Config", |
|
||||||
"row-index": "0", |
|
||||||
"column-index": "0", |
|
||||||
"row-span": "1", |
|
||||||
"column-span": "1" |
|
||||||
}, |
|
||||||
{ |
|
||||||
"name": "dynamic-row1-col2", |
|
||||||
"display-name": "Kerberos Info", |
|
||||||
"row-index": "0", |
|
||||||
"column-index": "1", |
|
||||||
"row-span": "1", |
|
||||||
"column-span": "1" |
|
||||||
}, |
|
||||||
{ |
|
||||||
"name": "dynamic-row1-col3", |
|
||||||
"display-name": "Wechat Info", |
|
||||||
"row-index": "0", |
|
||||||
"column-index": "1", |
|
||||||
"row-span": "1", |
|
||||||
"column-span": "1" |
|
||||||
} |
|
||||||
] |
|
||||||
} |
|
||||||
] |
|
||||||
} |
|
||||||
} |
|
||||||
] |
|
||||||
} |
|
||||||
], |
|
||||||
"placement": { |
|
||||||
"configuration-layout": "default", |
|
||||||
"configs": [ |
|
||||||
{ |
|
||||||
"config": "dolphin-env/dolphin.database.type", |
|
||||||
"subsection-name": "database-row1-col1" |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-env/dolphin.database.host", |
|
||||||
"subsection-name": "database-row1-col2" |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-env/dolphin.database.port", |
|
||||||
"subsection-name": "database-row1-col2" |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-env/dolphin.database.username", |
|
||||||
"subsection-name": "database-row1-col3" |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-env/dolphin.database.password", |
|
||||||
"subsection-name": "database-row1-col3" |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-env/dolphin.user", |
|
||||||
"subsection-name": "env-row1-col1" |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-env/dolphin.group", |
|
||||||
"subsection-name": "env-row1-col1" |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-env/dolphinscheduler-env-content", |
|
||||||
"subsection-name": "env-row1-col2" |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-common/resource.storage.type", |
|
||||||
"subsection-name": "dynamic-row1-col1" |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-common/resource.upload.path", |
|
||||||
"subsection-name": "dynamic-row1-col1", |
|
||||||
"depends-on": [ |
|
||||||
{ |
|
||||||
"configs":[ |
|
||||||
"dolphin-common/resource.storage.type" |
|
||||||
], |
|
||||||
"if": "${dolphin-common/resource.storage.type} === HDFS || ${dolphin-common/resource.storage.type} === S3", |
|
||||||
"then": { |
|
||||||
"property_value_attributes": { |
|
||||||
"visible": true |
|
||||||
} |
|
||||||
}, |
|
||||||
"else": { |
|
||||||
"property_value_attributes": { |
|
||||||
"visible": false |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
] |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-common/hdfs.root.user", |
|
||||||
"subsection-name": "dynamic-row1-col1", |
|
||||||
"depends-on": [ |
|
||||||
{ |
|
||||||
"configs":[ |
|
||||||
"dolphin-common/resource.storage.type" |
|
||||||
], |
|
||||||
"if": "${dolphin-common/resource.storage.type} === HDFS", |
|
||||||
"then": { |
|
||||||
"property_value_attributes": { |
|
||||||
"visible": true |
|
||||||
} |
|
||||||
}, |
|
||||||
"else": { |
|
||||||
"property_value_attributes": { |
|
||||||
"visible": false |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
] |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-common/data.store2hdfs.basepath", |
|
||||||
"subsection-name": "dynamic-row1-col1", |
|
||||||
"depends-on": [ |
|
||||||
{ |
|
||||||
"configs":[ |
|
||||||
"dolphin-common/resource.storage.type" |
|
||||||
], |
|
||||||
"if": "${dolphin-common/resource.storage.type} === HDFS", |
|
||||||
"then": { |
|
||||||
"property_value_attributes": { |
|
||||||
"visible": true |
|
||||||
} |
|
||||||
}, |
|
||||||
"else": { |
|
||||||
"property_value_attributes": { |
|
||||||
"visible": false |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
] |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-common/fs.defaultFS", |
|
||||||
"subsection-name": "dynamic-row1-col1", |
|
||||||
"depends-on": [ |
|
||||||
{ |
|
||||||
"configs":[ |
|
||||||
"dolphin-common/resource.storage.type" |
|
||||||
], |
|
||||||
"if": "${dolphin-common/resource.storage.type} === HDFS", |
|
||||||
"then": { |
|
||||||
"property_value_attributes": { |
|
||||||
"visible": true |
|
||||||
} |
|
||||||
}, |
|
||||||
"else": { |
|
||||||
"property_value_attributes": { |
|
||||||
"visible": false |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
] |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-common/fs.s3a.endpoint", |
|
||||||
"subsection-name": "dynamic-row1-col1", |
|
||||||
"depends-on": [ |
|
||||||
{ |
|
||||||
"configs":[ |
|
||||||
"dolphin-common/resource.storage.type" |
|
||||||
], |
|
||||||
"if": "${dolphin-common/resource.storage.type} === S3", |
|
||||||
"then": { |
|
||||||
"property_value_attributes": { |
|
||||||
"visible": true |
|
||||||
} |
|
||||||
}, |
|
||||||
"else": { |
|
||||||
"property_value_attributes": { |
|
||||||
"visible": false |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
] |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-common/fs.s3a.access.key", |
|
||||||
"subsection-name": "dynamic-row1-col1", |
|
||||||
"depends-on": [ |
|
||||||
{ |
|
||||||
"configs":[ |
|
||||||
"dolphin-common/resource.storage.type" |
|
||||||
], |
|
||||||
"if": "${dolphin-common/resource.storage.type} === S3", |
|
||||||
"then": { |
|
||||||
"property_value_attributes": { |
|
||||||
"visible": true |
|
||||||
} |
|
||||||
}, |
|
||||||
"else": { |
|
||||||
"property_value_attributes": { |
|
||||||
"visible": false |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
] |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-common/fs.s3a.secret.key", |
|
||||||
"subsection-name": "dynamic-row1-col1", |
|
||||||
"depends-on": [ |
|
||||||
{ |
|
||||||
"configs":[ |
|
||||||
"dolphin-common/resource.storage.type" |
|
||||||
], |
|
||||||
"if": "${dolphin-common/resource.storage.type} === S3", |
|
||||||
"then": { |
|
||||||
"property_value_attributes": { |
|
||||||
"visible": true |
|
||||||
} |
|
||||||
}, |
|
||||||
"else": { |
|
||||||
"property_value_attributes": { |
|
||||||
"visible": false |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
] |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-common/hadoop.security.authentication.startup.state", |
|
||||||
"subsection-name": "dynamic-row1-col2" |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-common/java.security.krb5.conf.path", |
|
||||||
"subsection-name": "dynamic-row1-col2", |
|
||||||
"depends-on": [ |
|
||||||
{ |
|
||||||
"configs":[ |
|
||||||
"dolphin-common/hadoop.security.authentication.startup.state" |
|
||||||
], |
|
||||||
"if": "${dolphin-common/hadoop.security.authentication.startup.state}", |
|
||||||
"then": { |
|
||||||
"property_value_attributes": { |
|
||||||
"visible": true |
|
||||||
} |
|
||||||
}, |
|
||||||
"else": { |
|
||||||
"property_value_attributes": { |
|
||||||
"visible": false |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
] |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-common/login.user.keytab.username", |
|
||||||
"subsection-name": "dynamic-row1-col2", |
|
||||||
"depends-on": [ |
|
||||||
{ |
|
||||||
"configs":[ |
|
||||||
"dolphin-common/hadoop.security.authentication.startup.state" |
|
||||||
], |
|
||||||
"if": "${dolphin-common/hadoop.security.authentication.startup.state}", |
|
||||||
"then": { |
|
||||||
"property_value_attributes": { |
|
||||||
"visible": true |
|
||||||
} |
|
||||||
}, |
|
||||||
"else": { |
|
||||||
"property_value_attributes": { |
|
||||||
"visible": false |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
] |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-common/login.user.keytab.path", |
|
||||||
"subsection-name": "dynamic-row1-col2", |
|
||||||
"depends-on": [ |
|
||||||
{ |
|
||||||
"configs":[ |
|
||||||
"dolphin-common/hadoop.security.authentication.startup.state" |
|
||||||
], |
|
||||||
"if": "${dolphin-common/hadoop.security.authentication.startup.state}", |
|
||||||
"then": { |
|
||||||
"property_value_attributes": { |
|
||||||
"visible": true |
|
||||||
} |
|
||||||
}, |
|
||||||
"else": { |
|
||||||
"property_value_attributes": { |
|
||||||
"visible": false |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
] |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-common/kerberos.expire.time", |
|
||||||
"subsection-name": "dynamic-row1-col2", |
|
||||||
"depends-on": [ |
|
||||||
{ |
|
||||||
"configs":[ |
|
||||||
"dolphin-common/hadoop.security.authentication.startup.state" |
|
||||||
], |
|
||||||
"if": "${dolphin-common/hadoop.security.authentication.startup.state}", |
|
||||||
"then": { |
|
||||||
"property_value_attributes": { |
|
||||||
"visible": true |
|
||||||
} |
|
||||||
}, |
|
||||||
"else": { |
|
||||||
"property_value_attributes": { |
|
||||||
"visible": false |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
] |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-alert/enterprise.wechat.enable", |
|
||||||
"subsection-name": "dynamic-row1-col3" |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-alert/enterprise.wechat.corp.id", |
|
||||||
"subsection-name": "dynamic-row1-col3", |
|
||||||
"depends-on": [ |
|
||||||
{ |
|
||||||
"configs":[ |
|
||||||
"dolphin-alert/enterprise.wechat.enable" |
|
||||||
], |
|
||||||
"if": "${dolphin-alert/enterprise.wechat.enable}", |
|
||||||
"then": { |
|
||||||
"property_value_attributes": { |
|
||||||
"visible": true |
|
||||||
} |
|
||||||
}, |
|
||||||
"else": { |
|
||||||
"property_value_attributes": { |
|
||||||
"visible": false |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
] |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-alert/enterprise.wechat.secret", |
|
||||||
"subsection-name": "dynamic-row1-col3", |
|
||||||
"depends-on": [ |
|
||||||
{ |
|
||||||
"configs":[ |
|
||||||
"dolphin-alert/enterprise.wechat.enable" |
|
||||||
], |
|
||||||
"if": "${dolphin-alert/enterprise.wechat.enable}", |
|
||||||
"then": { |
|
||||||
"property_value_attributes": { |
|
||||||
"visible": true |
|
||||||
} |
|
||||||
}, |
|
||||||
"else": { |
|
||||||
"property_value_attributes": { |
|
||||||
"visible": false |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
] |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-alert/enterprise.wechat.agent.id", |
|
||||||
"subsection-name": "dynamic-row1-col3", |
|
||||||
"depends-on": [ |
|
||||||
{ |
|
||||||
"configs":[ |
|
||||||
"dolphin-alert/enterprise.wechat.enable" |
|
||||||
], |
|
||||||
"if": "${dolphin-alert/enterprise.wechat.enable}", |
|
||||||
"then": { |
|
||||||
"property_value_attributes": { |
|
||||||
"visible": true |
|
||||||
} |
|
||||||
}, |
|
||||||
"else": { |
|
||||||
"property_value_attributes": { |
|
||||||
"visible": false |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
] |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-alert/enterprise.wechat.users", |
|
||||||
"subsection-name": "dynamic-row1-col3", |
|
||||||
"depends-on": [ |
|
||||||
{ |
|
||||||
"configs":[ |
|
||||||
"dolphin-alert/enterprise.wechat.enable" |
|
||||||
], |
|
||||||
"if": "${dolphin-alert/enterprise.wechat.enable}", |
|
||||||
"then": { |
|
||||||
"property_value_attributes": { |
|
||||||
"visible": true |
|
||||||
} |
|
||||||
}, |
|
||||||
"else": { |
|
||||||
"property_value_attributes": { |
|
||||||
"visible": false |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
] |
|
||||||
} |
|
||||||
] |
|
||||||
}, |
|
||||||
"widgets": [ |
|
||||||
{ |
|
||||||
"config": "dolphin-env/dolphin.database.type", |
|
||||||
"widget": { |
|
||||||
"type": "combo" |
|
||||||
} |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-env/dolphin.database.host", |
|
||||||
"widget": { |
|
||||||
"type": "text-field" |
|
||||||
} |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-env/dolphin.database.port", |
|
||||||
"widget": { |
|
||||||
"type": "text-field", |
|
||||||
"units": [ |
|
||||||
{ |
|
||||||
"unit-name": "int" |
|
||||||
} |
|
||||||
] |
|
||||||
} |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-env/dolphin.database.username", |
|
||||||
"widget": { |
|
||||||
"type": "text-field" |
|
||||||
} |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-env/dolphin.database.password", |
|
||||||
"widget": { |
|
||||||
"type": "password" |
|
||||||
} |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-env/dolphin.user", |
|
||||||
"widget": { |
|
||||||
"type": "text-field" |
|
||||||
} |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-env/dolphin.group", |
|
||||||
"widget": { |
|
||||||
"type": "text-field" |
|
||||||
} |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-env/dolphinscheduler-env-content", |
|
||||||
"widget": { |
|
||||||
"type": "text-area" |
|
||||||
} |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-common/resource.storage.type", |
|
||||||
"widget": { |
|
||||||
"type": "combo" |
|
||||||
} |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-common/resource.upload.path", |
|
||||||
"widget": { |
|
||||||
"type": "text-field" |
|
||||||
} |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-common/hdfs.root.user", |
|
||||||
"widget": { |
|
||||||
"type": "text-field" |
|
||||||
} |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-common/data.store2hdfs.basepath", |
|
||||||
"widget": { |
|
||||||
"type": "text-field" |
|
||||||
} |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-common/fs.defaultFS", |
|
||||||
"widget": { |
|
||||||
"type": "text-field" |
|
||||||
} |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-common/fs.s3a.endpoint", |
|
||||||
"widget": { |
|
||||||
"type": "text-field" |
|
||||||
} |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-common/fs.s3a.access.key", |
|
||||||
"widget": { |
|
||||||
"type": "text-field" |
|
||||||
} |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-common/fs.s3a.secret.key", |
|
||||||
"widget": { |
|
||||||
"type": "text-field" |
|
||||||
} |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-common/hadoop.security.authentication.startup.state", |
|
||||||
"widget": { |
|
||||||
"type": "toggle" |
|
||||||
} |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-common/java.security.krb5.conf.path", |
|
||||||
"widget": { |
|
||||||
"type": "text-field" |
|
||||||
} |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-common/login.user.keytab.username", |
|
||||||
"widget": { |
|
||||||
"type": "text-field" |
|
||||||
} |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-common/login.user.keytab.path", |
|
||||||
"widget": { |
|
||||||
"type": "text-field" |
|
||||||
} |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-common/kerberos.expire.time", |
|
||||||
"widget": { |
|
||||||
"type": "text-field" |
|
||||||
} |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-alert/enterprise.wechat.enable", |
|
||||||
"widget": { |
|
||||||
"type": "toggle" |
|
||||||
} |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-alert/enterprise.wechat.corp.id", |
|
||||||
"widget": { |
|
||||||
"type": "text-field" |
|
||||||
} |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-alert/enterprise.wechat.secret", |
|
||||||
"widget": { |
|
||||||
"type": "text-field" |
|
||||||
} |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-alert/enterprise.wechat.agent.id", |
|
||||||
"widget": { |
|
||||||
"type": "text-field" |
|
||||||
} |
|
||||||
}, |
|
||||||
{ |
|
||||||
"config": "dolphin-alert/enterprise.wechat.users", |
|
||||||
"widget": { |
|
||||||
"type": "text-field" |
|
||||||
} |
|
||||||
} |
|
||||||
] |
|
||||||
} |
|
||||||
} |
|
||||||
@ -0,0 +1,133 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package org.apache.dolphinscheduler.alert.plugin; |
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.alert.manager.EmailManager; |
||||||
|
import org.apache.dolphinscheduler.alert.manager.EnterpriseWeChatManager; |
||||||
|
import org.apache.dolphinscheduler.alert.utils.Constants; |
||||||
|
import org.apache.dolphinscheduler.alert.utils.EnterpriseWeChatUtils; |
||||||
|
import org.apache.dolphinscheduler.common.utils.CollectionUtils; |
||||||
|
import org.apache.dolphinscheduler.common.utils.StringUtils; |
||||||
|
import org.apache.dolphinscheduler.plugin.api.AlertPlugin; |
||||||
|
import org.apache.dolphinscheduler.plugin.model.AlertData; |
||||||
|
import org.apache.dolphinscheduler.plugin.model.AlertInfo; |
||||||
|
import org.apache.dolphinscheduler.plugin.model.PluginName; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
import java.util.*; |
||||||
|
|
||||||
|
/** |
||||||
|
* EmailAlertPlugin |
||||||
|
* |
||||||
|
* This plugin is a default plugin, and mix up email and enterprise wechat, because adapt with former alert behavior |
||||||
|
*/ |
||||||
|
public class EmailAlertPlugin implements AlertPlugin { |
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(EmailAlertPlugin.class); |
||||||
|
|
||||||
|
private PluginName pluginName; |
||||||
|
|
||||||
|
private static final EmailManager emailManager = new EmailManager(); |
||||||
|
private static final EnterpriseWeChatManager weChatManager = new EnterpriseWeChatManager(); |
||||||
|
|
||||||
|
public EmailAlertPlugin() { |
||||||
|
this.pluginName = new PluginName(); |
||||||
|
this.pluginName.setEnglish(Constants.PLUGIN_DEFAULT_EMAIL_EN); |
||||||
|
this.pluginName.setChinese(Constants.PLUGIN_DEFAULT_EMAIL_CH); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public String getId() { |
||||||
|
return Constants.PLUGIN_DEFAULT_EMAIL; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public PluginName getName() { |
||||||
|
return pluginName; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
@SuppressWarnings("unchecked") |
||||||
|
public Map<String, Object> process(AlertInfo info) { |
||||||
|
Map<String, Object> retMaps = new HashMap<>(); |
||||||
|
|
||||||
|
AlertData alert = info.getAlertData(); |
||||||
|
|
||||||
|
List<String> receviersList = (List<String>) info.getProp(Constants.PLUGIN_DEFAULT_EMAIL_RECEIVERS); |
||||||
|
|
||||||
|
// receiving group list
|
||||||
|
// custom receiver
|
||||||
|
String receivers = alert.getReceivers(); |
||||||
|
if (StringUtils.isNotEmpty(receivers)) { |
||||||
|
String[] splits = receivers.split(","); |
||||||
|
receviersList.addAll(Arrays.asList(splits)); |
||||||
|
} |
||||||
|
|
||||||
|
List<String> receviersCcList = new ArrayList<>(); |
||||||
|
// Custom Copier
|
||||||
|
String receiversCc = alert.getReceiversCc(); |
||||||
|
if (StringUtils.isNotEmpty(receiversCc)) { |
||||||
|
String[] splits = receiversCc.split(","); |
||||||
|
receviersCcList.addAll(Arrays.asList(splits)); |
||||||
|
} |
||||||
|
|
||||||
|
if (CollectionUtils.isEmpty(receviersList) && CollectionUtils.isEmpty(receviersCcList)) { |
||||||
|
logger.warn("alert send error : At least one receiver address required"); |
||||||
|
retMaps.put(Constants.STATUS, "false"); |
||||||
|
retMaps.put(Constants.MESSAGE, "execution failure,At least one receiver address required."); |
||||||
|
return retMaps; |
||||||
|
} |
||||||
|
|
||||||
|
retMaps = emailManager.send(receviersList, receviersCcList, alert.getTitle(), alert.getContent(), |
||||||
|
alert.getShowType()); |
||||||
|
|
||||||
|
//send flag
|
||||||
|
boolean flag = false; |
||||||
|
|
||||||
|
if (retMaps == null) { |
||||||
|
retMaps = new HashMap<>(); |
||||||
|
retMaps.put(Constants.MESSAGE, "alert send error."); |
||||||
|
retMaps.put(Constants.STATUS, "false"); |
||||||
|
logger.info("alert send error : {}", retMaps.get(Constants.MESSAGE)); |
||||||
|
return retMaps; |
||||||
|
} |
||||||
|
|
||||||
|
flag = Boolean.parseBoolean(String.valueOf(retMaps.get(Constants.STATUS))); |
||||||
|
|
||||||
|
if (flag) { |
||||||
|
logger.info("alert send success"); |
||||||
|
retMaps.put(Constants.MESSAGE, "email send success."); |
||||||
|
if (EnterpriseWeChatUtils.isEnable()) { |
||||||
|
logger.info("Enterprise WeChat is enable!"); |
||||||
|
try { |
||||||
|
String token = EnterpriseWeChatUtils.getToken(); |
||||||
|
weChatManager.send(info, token); |
||||||
|
} catch (Exception e) { |
||||||
|
logger.error(e.getMessage(), e); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
} else { |
||||||
|
retMaps.put(Constants.MESSAGE, "alert send error."); |
||||||
|
logger.info("alert send error : {}", retMaps.get(Constants.MESSAGE)); |
||||||
|
} |
||||||
|
|
||||||
|
return retMaps; |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
@ -0,0 +1,80 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package org.apache.dolphinscheduler.alert.plugin; |
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.alert.utils.Constants; |
||||||
|
import org.apache.dolphinscheduler.common.enums.ShowType; |
||||||
|
import org.apache.dolphinscheduler.plugin.api.AlertPlugin; |
||||||
|
import org.apache.dolphinscheduler.plugin.model.AlertData; |
||||||
|
import org.apache.dolphinscheduler.plugin.model.AlertInfo; |
||||||
|
import org.apache.dolphinscheduler.plugin.model.PluginName; |
||||||
|
import org.junit.Before; |
||||||
|
import org.junit.Test; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
import java.util.ArrayList; |
||||||
|
import java.util.List; |
||||||
|
import java.util.Map; |
||||||
|
|
||||||
|
import static org.junit.Assert.*; |
||||||
|
|
||||||
|
public class EmailAlertPluginTest { |
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(EmailAlertPluginTest.class); |
||||||
|
|
||||||
|
private AlertPlugin plugin; |
||||||
|
|
||||||
|
@Before |
||||||
|
public void before() { |
||||||
|
plugin = new EmailAlertPlugin(); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void getId() { |
||||||
|
String id = plugin.getId(); |
||||||
|
assertEquals(Constants.PLUGIN_DEFAULT_EMAIL, id); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void getName() { |
||||||
|
PluginName pluginName = plugin.getName(); |
||||||
|
assertEquals(Constants.PLUGIN_DEFAULT_EMAIL_CH, pluginName.getChinese()); |
||||||
|
assertEquals(Constants.PLUGIN_DEFAULT_EMAIL_EN, pluginName.getEnglish()); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void process() { |
||||||
|
AlertInfo alertInfo = new AlertInfo(); |
||||||
|
AlertData alertData = new AlertData(); |
||||||
|
alertData.setId(1) |
||||||
|
.setAlertGroupId(1) |
||||||
|
.setContent("[\"alarm time：2018-02-05\", \"service name：MYSQL_ALTER\", \"alarm name：MYSQL_ALTER_DUMP\", " + |
||||||
|
"\"get the alarm exception.！，interface error，exception information：timed out\", \"request address：http://blog.csdn.net/dreamInTheWorld/article/details/78539286\"]") |
||||||
|
.setLog("test log") |
||||||
|
.setReceivers("xx@xx.com") |
||||||
|
.setReceiversCc("xx@xx.com") |
||||||
|
.setShowType(ShowType.TEXT.getDescp()) |
||||||
|
.setTitle("test title"); |
||||||
|
|
||||||
|
alertInfo.setAlertData(alertData); |
||||||
|
List<String> list = new ArrayList<String>(){{ add("xx@xx.com"); }}; |
||||||
|
alertInfo.addProp("receivers", list); |
||||||
|
Map<String, Object> ret = plugin.process(alertInfo); |
||||||
|
assertFalse(Boolean.parseBoolean(String.valueOf(ret.get(Constants.STATUS)))); |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,107 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package org.apache.dolphinscheduler.common.plugin; |
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.common.Constants; |
||||||
|
import org.apache.dolphinscheduler.plugin.api.AlertPlugin; |
||||||
|
import org.apache.dolphinscheduler.plugin.spi.AlertPluginProvider; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
import java.io.File; |
||||||
|
import java.net.MalformedURLException; |
||||||
|
import java.net.URL; |
||||||
|
import java.util.Map; |
||||||
|
import java.util.ServiceLoader; |
||||||
|
import java.util.concurrent.ConcurrentHashMap; |
||||||
|
|
||||||
|
/** |
||||||
|
* FilePluginManager |
||||||
|
*/ |
||||||
|
public class FilePluginManager implements PluginManager { |
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(FilePluginManager.class); |
||||||
|
|
||||||
|
private Map<String, AlertPlugin> pluginMap = new ConcurrentHashMap<>(); |
||||||
|
|
||||||
|
private Map<String, ServiceLoader<AlertPluginProvider>> pluginLoaderMap = new ConcurrentHashMap<>(); |
||||||
|
|
||||||
|
private Map<String, PluginClassLoader> classLoaderMap = new ConcurrentHashMap<>(); |
||||||
|
|
||||||
|
private String[] whitePrefixes; |
||||||
|
|
||||||
|
private String[] excludePrefixes; |
||||||
|
|
||||||
|
public FilePluginManager(String dirPath, String[] whitePrefixes, String[] excludePrefixes) { |
||||||
|
this.whitePrefixes = whitePrefixes; |
||||||
|
this.excludePrefixes = excludePrefixes; |
||||||
|
try { |
||||||
|
load(dirPath); |
||||||
|
} catch (MalformedURLException e) { |
||||||
|
logger.error("load plugins failed.", e); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
private void load(String dirPath) throws MalformedURLException { |
||||||
|
logger.info("start to load jar files in {}", dirPath); |
||||||
|
if (dirPath == null) { |
||||||
|
logger.error("not a valid path - {}", dirPath); |
||||||
|
return; |
||||||
|
} |
||||||
|
File[] files = new File(dirPath).listFiles(); |
||||||
|
if (files == null) { |
||||||
|
logger.error("not a valid path - {}", dirPath); |
||||||
|
return; |
||||||
|
} |
||||||
|
for (File file : files) { |
||||||
|
if (file.isDirectory() && !file.getPath().endsWith(Constants.PLUGIN_JAR_SUFFIX)) { |
||||||
|
continue; |
||||||
|
} |
||||||
|
String pluginName = file.getName() |
||||||
|
.substring(0, file.getName().length() - Constants.PLUGIN_JAR_SUFFIX.length()); |
||||||
|
URL[] urls = new URL[]{ file.toURI().toURL() }; |
||||||
|
PluginClassLoader classLoader = |
||||||
|
new PluginClassLoader(urls, Thread.currentThread().getContextClassLoader(), whitePrefixes, excludePrefixes); |
||||||
|
classLoaderMap.put(pluginName, classLoader); |
||||||
|
|
||||||
|
ServiceLoader<AlertPluginProvider> loader = ServiceLoader.load(AlertPluginProvider.class, classLoader); |
||||||
|
pluginLoaderMap.put(pluginName, loader); |
||||||
|
|
||||||
|
loader.forEach(provider -> { |
||||||
|
AlertPlugin plugin = provider.createPlugin(); |
||||||
|
pluginMap.put(plugin.getId(), plugin); |
||||||
|
logger.info("loaded plugin - {}", plugin.getId()); |
||||||
|
}); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public AlertPlugin findOne(String name) { |
||||||
|
return pluginMap.get(name); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public Map<String, AlertPlugin> findAll() { |
||||||
|
return pluginMap; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public void addPlugin(AlertPlugin plugin) { |
||||||
|
pluginMap.put(plugin.getId(), plugin); |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
@ -0,0 +1,154 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package org.apache.dolphinscheduler.common.plugin; |
||||||
|
|
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
import java.io.IOException; |
||||||
|
import java.net.URL; |
||||||
|
import java.net.URLClassLoader; |
||||||
|
import java.util.Enumeration; |
||||||
|
import java.util.Iterator; |
||||||
|
import java.util.LinkedList; |
||||||
|
import java.util.List; |
||||||
|
|
||||||
|
/** |
||||||
|
* Plugin Class Loader |
||||||
|
*/ |
||||||
|
public class PluginClassLoader extends URLClassLoader { |
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(PluginClassLoader.class); |
||||||
|
|
||||||
|
private static final String JAVA_PACKAGE_PREFIX = "java."; |
||||||
|
private static final String JAVAX_PACKAGE_PREFIX = "javax."; |
||||||
|
|
||||||
|
private final String[] whitePrefixes; |
||||||
|
|
||||||
|
private final String[] excludePrefixes; |
||||||
|
|
||||||
|
public PluginClassLoader(URL[] urls, ClassLoader parent, String[] whitePrefix, String[] excludePreifx) { |
||||||
|
super(urls, parent); |
||||||
|
this.whitePrefixes = whitePrefix; |
||||||
|
this.excludePrefixes = excludePreifx; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public Class<?> loadClass(String name) throws ClassNotFoundException { |
||||||
|
logger.trace("Received request to load class '{}'", name); |
||||||
|
synchronized (getClassLoadingLock(name)) { |
||||||
|
if (name.startsWith(JAVA_PACKAGE_PREFIX) || name.startsWith(JAVAX_PACKAGE_PREFIX)) { |
||||||
|
return findSystemClass(name); |
||||||
|
} |
||||||
|
|
||||||
|
boolean isWhitePrefixes = fromWhitePrefix(name); |
||||||
|
boolean isExcludePrefixed = fromExcludePrefix(name); |
||||||
|
|
||||||
|
// if the class is part of the plugin engine use parent class loader
|
||||||
|
if (!isWhitePrefixes && isExcludePrefixed) { |
||||||
|
return getParent().loadClass(name); |
||||||
|
} |
||||||
|
|
||||||
|
// check whether it's already been loaded
|
||||||
|
Class<?> loadedClass = findLoadedClass(name); |
||||||
|
if (loadedClass != null) { |
||||||
|
logger.debug("Found loaded class '{}'", name); |
||||||
|
return loadedClass; |
||||||
|
} |
||||||
|
|
||||||
|
// nope, try to load locally
|
||||||
|
try { |
||||||
|
loadedClass = findClass(name); |
||||||
|
logger.debug("Found class '{}' in plugin classpath", name); |
||||||
|
return loadedClass; |
||||||
|
} catch (ClassNotFoundException e) { |
||||||
|
// try next step
|
||||||
|
} |
||||||
|
|
||||||
|
// use the standard ClassLoader (which follows normal parent delegation)
|
||||||
|
return super.loadClass(name); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
private boolean fromWhitePrefix(String name) { |
||||||
|
if (this.whitePrefixes == null) { |
||||||
|
return false; |
||||||
|
} |
||||||
|
for (String whitePrefix : this.whitePrefixes) { |
||||||
|
if (name.startsWith(whitePrefix)) { |
||||||
|
return true; |
||||||
|
} |
||||||
|
} |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
private boolean fromExcludePrefix(String name) { |
||||||
|
if (this.excludePrefixes == null) { |
||||||
|
return false; |
||||||
|
} |
||||||
|
for (String excludePrefix : this.excludePrefixes) { |
||||||
|
if (name.startsWith(excludePrefix)) { |
||||||
|
return true; |
||||||
|
} |
||||||
|
} |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public Enumeration<URL> getResources(String name) throws IOException { |
||||||
|
List<URL> allRes = new LinkedList<>(); |
||||||
|
|
||||||
|
Enumeration<URL> thisRes = findResources(name); |
||||||
|
if (thisRes != null) { |
||||||
|
while (thisRes.hasMoreElements()) { |
||||||
|
allRes.add(thisRes.nextElement()); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
Enumeration<URL> parentRes = super.findResources(name); |
||||||
|
if (parentRes != null) { |
||||||
|
while (parentRes.hasMoreElements()) { |
||||||
|
allRes.add(parentRes.nextElement()); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
return new Enumeration<URL>() { |
||||||
|
Iterator<URL> it = allRes.iterator(); |
||||||
|
|
||||||
|
@Override |
||||||
|
public boolean hasMoreElements() { |
||||||
|
return it.hasNext(); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public URL nextElement() { |
||||||
|
return it.next(); |
||||||
|
} |
||||||
|
}; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public URL getResource(String name) { |
||||||
|
URL res = null; |
||||||
|
|
||||||
|
res = findResource(name); |
||||||
|
if (res == null) { |
||||||
|
res = super.getResource(name); |
||||||
|
} |
||||||
|
return res; |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,33 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package org.apache.dolphinscheduler.common.plugin; |
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.plugin.api.AlertPlugin; |
||||||
|
|
||||||
|
import java.util.Map; |
||||||
|
|
||||||
|
/** |
||||||
|
* PluginManager |
||||||
|
*/ |
||||||
|
public interface PluginManager { |
||||||
|
|
||||||
|
AlertPlugin findOne(String name); |
||||||
|
|
||||||
|
Map<String, AlertPlugin> findAll(); |
||||||
|
|
||||||
|
void addPlugin(AlertPlugin plugin); |
||||||
|
} |
||||||
@ -0,0 +1,72 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package org.apache.dolphinscheduler.common.plugin; |
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.plugin.api.AlertPlugin; |
||||||
|
import org.apache.dolphinscheduler.plugin.model.AlertInfo; |
||||||
|
import org.apache.dolphinscheduler.plugin.model.PluginName; |
||||||
|
import org.junit.Before; |
||||||
|
import org.junit.Test; |
||||||
|
|
||||||
|
import java.util.HashMap; |
||||||
|
import java.util.Map; |
||||||
|
|
||||||
|
import static org.junit.Assert.*; |
||||||
|
|
||||||
|
public class FilePluginManagerTest { |
||||||
|
|
||||||
|
private FilePluginManager filePluginManager; |
||||||
|
private AlertPlugin alertPlugin; |
||||||
|
|
||||||
|
@Before |
||||||
|
public void before() { |
||||||
|
filePluginManager = new FilePluginManager(null, null, null); |
||||||
|
alertPlugin = new AlertPlugin() { |
||||||
|
@Override |
||||||
|
public String getId() { |
||||||
|
return "test"; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public PluginName getName() { |
||||||
|
return new PluginName().setChinese("ch").setEnglish("en"); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public Map<String, Object> process(AlertInfo info) { |
||||||
|
return new HashMap<>(); |
||||||
|
} |
||||||
|
}; |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void findOne() { |
||||||
|
filePluginManager.addPlugin(alertPlugin); |
||||||
|
assertEquals(alertPlugin, filePluginManager.findOne(alertPlugin.getId())); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void findAll() { |
||||||
|
assertNotNull(filePluginManager.findAll()); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void addPlugin() { |
||||||
|
filePluginManager.addPlugin(alertPlugin); |
||||||
|
assertNotNull(filePluginManager.findAll()); |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,61 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package org.apache.dolphinscheduler.common.plugin; |
||||||
|
|
||||||
|
import org.junit.Before; |
||||||
|
import org.junit.Test; |
||||||
|
|
||||||
|
import java.net.URL; |
||||||
|
|
||||||
|
import static org.junit.Assert.*; |
||||||
|
|
||||||
|
public class PluginClassLoaderTest { |
||||||
|
|
||||||
|
private PluginClassLoader pluginClassLoader; |
||||||
|
private ClassLoader parent; |
||||||
|
|
||||||
|
@Before |
||||||
|
public void setUp() { |
||||||
|
parent = Thread.currentThread().getContextClassLoader(); |
||||||
|
pluginClassLoader = new PluginClassLoader( |
||||||
|
new URL[]{}, parent, |
||||||
|
null, null); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void loadClassNull() { |
||||||
|
Class clazz = null; |
||||||
|
try { |
||||||
|
clazz = pluginClassLoader.loadClass("java.lang.Object"); |
||||||
|
} catch (ClassNotFoundException e) { |
||||||
|
e.printStackTrace(); |
||||||
|
} |
||||||
|
assertEquals(null, clazz.getClassLoader()); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void loadClassApp() { |
||||||
|
Class clazz = null; |
||||||
|
try { |
||||||
|
clazz = pluginClassLoader.loadClass("org.apache.dolphinscheduler.common.Constants"); |
||||||
|
} catch (ClassNotFoundException e) { |
||||||
|
e.printStackTrace(); |
||||||
|
} |
||||||
|
assertEquals(parent, clazz.getClassLoader()); |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
@ -1,19 +0,0 @@ |
|||||||
Copyright (C) 2009-2015 The Project Lombok Authors. |
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy |
|
||||||
of this software and associated documentation files (the "Software"), to deal |
|
||||||
in the Software without restriction, including without limitation the rights |
|
||||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
|
||||||
copies of the Software, and to permit persons to whom the Software is |
|
||||||
furnished to do so, subject to the following conditions: |
|
||||||
|
|
||||||
The above copyright notice and this permission notice shall be included in |
|
||||||
all copies or substantial portions of the Software. |
|
||||||
|
|
||||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
|
||||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
|
||||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
|
||||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
|
||||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
|
||||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN |
|
||||||
THE SOFTWARE. |
|
||||||
@ -0,0 +1,52 @@ |
|||||||
|
<?xml version="1.0"?> |
||||||
|
<!-- |
||||||
|
~ Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
~ contributor license agreements. See the NOTICE file distributed with |
||||||
|
~ this work for additional information regarding copyright ownership. |
||||||
|
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
~ (the "License"); you may not use this file except in compliance with |
||||||
|
~ the License. You may obtain a copy of the License at |
||||||
|
~ |
||||||
|
~ http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
~ |
||||||
|
~ Unless required by applicable law or agreed to in writing, software |
||||||
|
~ distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
~ See the License for the specific language governing permissions and |
||||||
|
~ limitations under the License. |
||||||
|
--> |
||||||
|
|
||||||
|
<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" |
||||||
|
xmlns="http://maven.apache.org/POM/4.0.0" |
||||||
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> |
||||||
|
<modelVersion>4.0.0</modelVersion> |
||||||
|
<parent> |
||||||
|
<groupId>org.apache.dolphinscheduler</groupId> |
||||||
|
<artifactId>dolphinscheduler</artifactId> |
||||||
|
<version>1.2.1-SNAPSHOT</version> |
||||||
|
</parent> |
||||||
|
<artifactId>dolphinscheduler-plugin-api</artifactId> |
||||||
|
<name>${project.artifactId}</name> |
||||||
|
<packaging>jar</packaging> |
||||||
|
|
||||||
|
<properties> |
||||||
|
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> |
||||||
|
</properties> |
||||||
|
|
||||||
|
<dependencies> |
||||||
|
<dependency> |
||||||
|
<groupId>org.slf4j</groupId> |
||||||
|
<artifactId>slf4j-api</artifactId> |
||||||
|
</dependency> |
||||||
|
<dependency> |
||||||
|
<groupId>junit</groupId> |
||||||
|
<artifactId>junit</artifactId> |
||||||
|
<scope>test</scope> |
||||||
|
</dependency> |
||||||
|
<dependency> |
||||||
|
<groupId>commons-io</groupId> |
||||||
|
<artifactId>commons-io</artifactId> |
||||||
|
</dependency> |
||||||
|
</dependencies> |
||||||
|
|
||||||
|
</project> |
||||||
@ -0,0 +1,45 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package org.apache.dolphinscheduler.plugin.api; |
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.plugin.model.AlertInfo; |
||||||
|
import org.apache.dolphinscheduler.plugin.model.PluginName; |
||||||
|
|
||||||
|
import java.util.Map; |
||||||
|
|
||||||
|
/** |
||||||
|
* Plugin |
||||||
|
*/ |
||||||
|
public interface AlertPlugin { |
||||||
|
|
||||||
|
/** |
||||||
|
* Get alert plugin id |
||||||
|
* |
||||||
|
* @return alert plugin id, which should be unique |
||||||
|
*/ |
||||||
|
String getId(); |
||||||
|
|
||||||
|
/** |
||||||
|
* Get alert plugin name, which will show in front end portal |
||||||
|
* |
||||||
|
* @return plugin name |
||||||
|
*/ |
||||||
|
PluginName getName(); |
||||||
|
|
||||||
|
Map<String, Object> process(AlertInfo info); |
||||||
|
|
||||||
|
} |
||||||
@ -0,0 +1,129 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package org.apache.dolphinscheduler.plugin.model; |
||||||
|
|
||||||
|
/** |
||||||
|
* AlertData |
||||||
|
*/ |
||||||
|
public class AlertData { |
||||||
|
|
||||||
|
/** |
||||||
|
* alert primary key |
||||||
|
*/ |
||||||
|
private int id; |
||||||
|
/** |
||||||
|
* title |
||||||
|
*/ |
||||||
|
private String title; |
||||||
|
/** |
||||||
|
* content |
||||||
|
*/ |
||||||
|
private String content; |
||||||
|
/** |
||||||
|
* log |
||||||
|
*/ |
||||||
|
private String log; |
||||||
|
/** |
||||||
|
* alertgroup_id |
||||||
|
*/ |
||||||
|
private int alertGroupId; |
||||||
|
/** |
||||||
|
* receivers |
||||||
|
*/ |
||||||
|
private String receivers; |
||||||
|
/** |
||||||
|
* show_type |
||||||
|
*/ |
||||||
|
private String showType; |
||||||
|
/** |
||||||
|
* receivers_cc |
||||||
|
*/ |
||||||
|
private String receiversCc; |
||||||
|
|
||||||
|
public int getId() { |
||||||
|
return id; |
||||||
|
} |
||||||
|
|
||||||
|
public AlertData setId(int id) { |
||||||
|
this.id = id; |
||||||
|
return this; |
||||||
|
} |
||||||
|
|
||||||
|
public String getTitle() { |
||||||
|
return title; |
||||||
|
} |
||||||
|
|
||||||
|
public AlertData setTitle(String title) { |
||||||
|
this.title = title; |
||||||
|
return this; |
||||||
|
} |
||||||
|
|
||||||
|
public String getContent() { |
||||||
|
return content; |
||||||
|
} |
||||||
|
|
||||||
|
public AlertData setContent(String content) { |
||||||
|
this.content = content; |
||||||
|
return this; |
||||||
|
} |
||||||
|
|
||||||
|
public String getLog() { |
||||||
|
return log; |
||||||
|
} |
||||||
|
|
||||||
|
public AlertData setLog(String log) { |
||||||
|
this.log = log; |
||||||
|
return this; |
||||||
|
} |
||||||
|
|
||||||
|
public int getAlertGroupId() { |
||||||
|
return alertGroupId; |
||||||
|
} |
||||||
|
|
||||||
|
public AlertData setAlertGroupId(int alertGroupId) { |
||||||
|
this.alertGroupId = alertGroupId; |
||||||
|
return this; |
||||||
|
} |
||||||
|
|
||||||
|
public String getReceivers() { |
||||||
|
return receivers; |
||||||
|
} |
||||||
|
|
||||||
|
public AlertData setReceivers(String receivers) { |
||||||
|
this.receivers = receivers; |
||||||
|
return this; |
||||||
|
} |
||||||
|
|
||||||
|
public String getReceiversCc() { |
||||||
|
return receiversCc; |
||||||
|
} |
||||||
|
|
||||||
|
public AlertData setReceiversCc(String receiversCc) { |
||||||
|
this.receiversCc = receiversCc; |
||||||
|
return this; |
||||||
|
} |
||||||
|
|
||||||
|
public String getShowType() { |
||||||
|
return showType; |
||||||
|
} |
||||||
|
|
||||||
|
public AlertData setShowType(String showType) { |
||||||
|
this.showType = showType; |
||||||
|
return this; |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
@ -0,0 +1,61 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package org.apache.dolphinscheduler.plugin.model; |
||||||
|
|
||||||
|
import java.util.HashMap; |
||||||
|
import java.util.Map; |
||||||
|
|
||||||
|
/** |
||||||
|
* AlertInfo |
||||||
|
*/ |
||||||
|
public class AlertInfo { |
||||||
|
|
||||||
|
private Map<String, Object> alertProps; |
||||||
|
|
||||||
|
private AlertData alertData; |
||||||
|
|
||||||
|
public AlertInfo() { |
||||||
|
this.alertProps = new HashMap<>(); |
||||||
|
} |
||||||
|
|
||||||
|
public Map<String, Object> getAlertProps() { |
||||||
|
return alertProps; |
||||||
|
} |
||||||
|
|
||||||
|
public AlertInfo setAlertProps(Map<String, Object> alertProps) { |
||||||
|
this.alertProps = alertProps; |
||||||
|
return this; |
||||||
|
} |
||||||
|
|
||||||
|
public AlertInfo addProp(String key, Object value) { |
||||||
|
this.alertProps.put(key, value); |
||||||
|
return this; |
||||||
|
} |
||||||
|
|
||||||
|
public Object getProp(String key) { |
||||||
|
return this.alertProps.get(key); |
||||||
|
} |
||||||
|
|
||||||
|
public AlertData getAlertData() { |
||||||
|
return alertData; |
||||||
|
} |
||||||
|
|
||||||
|
public AlertInfo setAlertData(AlertData alertData) { |
||||||
|
this.alertData = alertData; |
||||||
|
return this; |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,45 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package org.apache.dolphinscheduler.plugin.model; |
||||||
|
|
||||||
|
/** |
||||||
|
* PluginName |
||||||
|
*/ |
||||||
|
public class PluginName { |
||||||
|
|
||||||
|
private String chinese; |
||||||
|
|
||||||
|
private String english; |
||||||
|
|
||||||
|
public String getChinese() { |
||||||
|
return chinese; |
||||||
|
} |
||||||
|
|
||||||
|
public PluginName setChinese(String chinese) { |
||||||
|
this.chinese = chinese; |
||||||
|
return this; |
||||||
|
} |
||||||
|
|
||||||
|
public String getEnglish() { |
||||||
|
return english; |
||||||
|
} |
||||||
|
|
||||||
|
public PluginName setEnglish(String english) { |
||||||
|
this.english = english; |
||||||
|
return this; |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,33 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package org.apache.dolphinscheduler.plugin.spi; |
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.plugin.api.AlertPlugin; |
||||||
|
|
||||||
|
/** |
||||||
|
* PluginProvider |
||||||
|
*/ |
||||||
|
public interface AlertPluginProvider { |
||||||
|
|
||||||
|
/** |
||||||
|
* create an alert plugin |
||||||
|
* |
||||||
|
* @return an alert plugin |
||||||
|
*/ |
||||||
|
AlertPlugin createPlugin(); |
||||||
|
|
||||||
|
} |
||||||
@ -0,0 +1,190 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package org.apache.dolphinscheduler.plugin.utils; |
||||||
|
|
||||||
|
import org.apache.commons.io.IOUtils; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
import java.io.IOException; |
||||||
|
import java.io.InputStream; |
||||||
|
import java.util.Properties; |
||||||
|
|
||||||
|
/** |
||||||
|
* property utils |
||||||
|
* single instance |
||||||
|
*/ |
||||||
|
public class PropertyUtils { |
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(PropertyUtils.class); |
||||||
|
|
||||||
|
private static final Properties properties = new Properties(); |
||||||
|
|
||||||
|
private PropertyUtils() { |
||||||
|
throw new IllegalStateException("PropertyUtils class"); |
||||||
|
} |
||||||
|
|
||||||
|
static { |
||||||
|
String propertyFiles = "/plugin.properties"; |
||||||
|
InputStream fis = null; |
||||||
|
try { |
||||||
|
fis = PropertyUtils.class.getResourceAsStream(propertyFiles); |
||||||
|
properties.load(fis); |
||||||
|
} catch (IOException e) { |
||||||
|
logger.error(e.getMessage(), e); |
||||||
|
if (fis != null) { |
||||||
|
IOUtils.closeQuietly(fis); |
||||||
|
} |
||||||
|
} finally { |
||||||
|
IOUtils.closeQuietly(fis); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get property value |
||||||
|
* |
||||||
|
* @param key property name |
||||||
|
* @param defaultVal default value |
||||||
|
* @return property value |
||||||
|
*/ |
||||||
|
public static String getString(String key, String defaultVal) { |
||||||
|
String val = properties.getProperty(key.trim()); |
||||||
|
return val == null ? defaultVal : val; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get property value |
||||||
|
* |
||||||
|
* @param key property name |
||||||
|
* @return property value |
||||||
|
*/ |
||||||
|
public static String getString(String key) { |
||||||
|
if (key == null) { |
||||||
|
return null; |
||||||
|
} |
||||||
|
return properties.getProperty(key.trim()); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get property value |
||||||
|
* |
||||||
|
* @param key property name |
||||||
|
* @return get property int value , if key == null, then return -1 |
||||||
|
*/ |
||||||
|
public static int getInt(String key) { |
||||||
|
return getInt(key, -1); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get int |
||||||
|
* |
||||||
|
* @param key key |
||||||
|
* @param defaultValue default value |
||||||
|
* @return property value |
||||||
|
*/ |
||||||
|
public static int getInt(String key, int defaultValue) { |
||||||
|
String value = properties.getProperty(key.trim()); |
||||||
|
if (value == null) { |
||||||
|
return defaultValue; |
||||||
|
} |
||||||
|
try { |
||||||
|
return Integer.parseInt(value); |
||||||
|
} catch (NumberFormatException e) { |
||||||
|
logger.info(e.getMessage(), e); |
||||||
|
} |
||||||
|
return defaultValue; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get property value |
||||||
|
* |
||||||
|
* @param key property name |
||||||
|
* @return property value |
||||||
|
*/ |
||||||
|
public static boolean getBoolean(String key) { |
||||||
|
String value = properties.getProperty(key.trim()); |
||||||
|
if (value == null) { |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
return Boolean.parseBoolean(value); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get property value |
||||||
|
* |
||||||
|
* @param key property name |
||||||
|
* @param defaultValue default value |
||||||
|
* @return property value |
||||||
|
*/ |
||||||
|
public static Boolean getBoolean(String key, boolean defaultValue) { |
||||||
|
String value = properties.getProperty(key.trim()); |
||||||
|
if (value == null) { |
||||||
|
return defaultValue; |
||||||
|
} |
||||||
|
|
||||||
|
return Boolean.parseBoolean(value); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get property long value |
||||||
|
* |
||||||
|
* @param key key |
||||||
|
* @param defaultVal default value |
||||||
|
* @return property value |
||||||
|
*/ |
||||||
|
public static long getLong(String key, long defaultVal) { |
||||||
|
String val = getString(key); |
||||||
|
return val == null ? defaultVal : Long.parseLong(val); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get long |
||||||
|
* |
||||||
|
* @param key key |
||||||
|
* @return property value |
||||||
|
*/ |
||||||
|
public static long getLong(String key) { |
||||||
|
return getLong(key, -1); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get double |
||||||
|
* |
||||||
|
* @param key key |
||||||
|
* @param defaultVal default value |
||||||
|
* @return property value |
||||||
|
*/ |
||||||
|
public static double getDouble(String key, double defaultVal) { |
||||||
|
String val = properties.getProperty(key.trim()); |
||||||
|
return val == null ? defaultVal : Double.parseDouble(val); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* @param key key |
||||||
|
* @param type type |
||||||
|
* @param defaultValue default value |
||||||
|
* @param <T> T |
||||||
|
* @return get enum value |
||||||
|
*/ |
||||||
|
public <T extends Enum<T>> T getEnum(String key, Class<T> type, |
||||||
|
T defaultValue) { |
||||||
|
String val = properties.getProperty(key.trim()); |
||||||
|
return val == null ? defaultValue : Enum.valueOf(type, val); |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
@ -0,0 +1,80 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package org.apache.dolphinscheduler.plugin.model; |
||||||
|
|
||||||
|
import org.junit.Before; |
||||||
|
import org.junit.Test; |
||||||
|
|
||||||
|
import static org.junit.Assert.*; |
||||||
|
|
||||||
|
public class AlertDataTest { |
||||||
|
|
||||||
|
private AlertData alertData; |
||||||
|
|
||||||
|
@Before |
||||||
|
public void before() { |
||||||
|
alertData = new AlertData(); |
||||||
|
alertData.setId(1) |
||||||
|
.setContent("content") |
||||||
|
.setShowType("email") |
||||||
|
.setTitle("title") |
||||||
|
.setReceivers("receivers") |
||||||
|
.setReceiversCc("cc") |
||||||
|
.setLog("log") |
||||||
|
.setAlertGroupId(1); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void getId() { |
||||||
|
assertEquals(1, alertData.getId()); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void getTitle() { |
||||||
|
assertEquals("title", alertData.getTitle()); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void getContent() { |
||||||
|
assertEquals("content", alertData.getContent()); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void getLog() { |
||||||
|
assertEquals("log", alertData.getLog()); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void getAlertGroupId() { |
||||||
|
assertEquals(1, alertData.getAlertGroupId()); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void getReceivers() { |
||||||
|
assertEquals("receivers", alertData.getReceivers()); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void getReceiversCc() { |
||||||
|
assertEquals("cc", alertData.getReceiversCc()); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void getShowType() { |
||||||
|
assertEquals("email", alertData.getShowType()); |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,54 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package org.apache.dolphinscheduler.plugin.model; |
||||||
|
|
||||||
|
import org.junit.Before; |
||||||
|
import org.junit.Test; |
||||||
|
|
||||||
|
import java.util.HashMap; |
||||||
|
import java.util.Map; |
||||||
|
|
||||||
|
import static org.junit.Assert.*; |
||||||
|
|
||||||
|
public class AlertInfoTest { |
||||||
|
|
||||||
|
private AlertInfo alertInfo; |
||||||
|
|
||||||
|
@Before |
||||||
|
public void before() { |
||||||
|
alertInfo = new AlertInfo(); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void getAlertProps() { |
||||||
|
Map<String, Object> map = new HashMap<>(); |
||||||
|
alertInfo.setAlertProps(map); |
||||||
|
assertNotNull(alertInfo.getAlertProps()); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void getProp() { |
||||||
|
alertInfo.addProp("k", "v"); |
||||||
|
assertEquals("v", alertInfo.getProp("k")); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void getAlertData() { |
||||||
|
alertInfo.setAlertData(new AlertData()); |
||||||
|
assertNotNull(alertInfo.getAlertData()); |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,83 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package org.apache.dolphinscheduler.plugin.utils; |
||||||
|
|
||||||
|
import org.junit.Test; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
import static org.junit.Assert.*; |
||||||
|
|
||||||
|
public class PropertyUtilsTest { |
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(PropertyUtilsTest.class); |
||||||
|
|
||||||
|
/** |
||||||
|
* Test getString |
||||||
|
*/ |
||||||
|
@Test |
||||||
|
public void testGetString() { |
||||||
|
|
||||||
|
String result = PropertyUtils.getString("test.string"); |
||||||
|
logger.info(result); |
||||||
|
assertEquals("teststring", result); |
||||||
|
|
||||||
|
//If key is null, then return null
|
||||||
|
result = PropertyUtils.getString(null); |
||||||
|
assertNull(result); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* Test getBoolean |
||||||
|
*/ |
||||||
|
@Test |
||||||
|
public void testGetBoolean() { |
||||||
|
|
||||||
|
//Expected true
|
||||||
|
Boolean result = PropertyUtils.getBoolean("test.true"); |
||||||
|
assertTrue(result); |
||||||
|
|
||||||
|
//Expected false
|
||||||
|
result = PropertyUtils.getBoolean("test.false"); |
||||||
|
assertFalse(result); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Test getLong |
||||||
|
*/ |
||||||
|
@Test |
||||||
|
public void testGetLong() { |
||||||
|
long result = PropertyUtils.getLong("test.long"); |
||||||
|
assertSame(100L, result); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Test getDouble |
||||||
|
*/ |
||||||
|
@Test |
||||||
|
public void testGetDouble() { |
||||||
|
|
||||||
|
//If key is undefine in alert.properties, and there is a defaultval, then return defaultval
|
||||||
|
double result = PropertyUtils.getDouble("abc", 5.0); |
||||||
|
assertEquals(5.0, result, 0); |
||||||
|
|
||||||
|
result = PropertyUtils.getDouble("cba", 5.0); |
||||||
|
assertEquals(3.1, result, 0.01); |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
@ -0,0 +1,262 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.server.master.consumer; |
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.common.enums.CommandType; |
||||||
|
import org.apache.dolphinscheduler.common.enums.DbType; |
||||||
|
import org.apache.dolphinscheduler.common.enums.ExecutionStatus; |
||||||
|
import org.apache.dolphinscheduler.common.enums.Priority; |
||||||
|
import org.apache.dolphinscheduler.dao.entity.*; |
||||||
|
import org.apache.dolphinscheduler.server.master.dispatch.ExecutorDispatcher; |
||||||
|
import org.apache.dolphinscheduler.server.master.dispatch.executor.NettyExecutorManager; |
||||||
|
import org.apache.dolphinscheduler.server.registry.DependencyConfig; |
||||||
|
import org.apache.dolphinscheduler.server.registry.ZookeeperNodeManager; |
||||||
|
import org.apache.dolphinscheduler.server.registry.ZookeeperRegistryCenter; |
||||||
|
import org.apache.dolphinscheduler.server.zk.SpringZKServer; |
||||||
|
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; |
||||||
|
import org.apache.dolphinscheduler.service.process.ProcessService; |
||||||
|
import org.apache.dolphinscheduler.service.queue.TaskPriorityQueue; |
||||||
|
import org.apache.dolphinscheduler.service.queue.TaskPriorityQueueImpl; |
||||||
|
import org.apache.dolphinscheduler.service.zk.ZookeeperCachedOperator; |
||||||
|
import org.apache.dolphinscheduler.service.zk.ZookeeperConfig; |
||||||
|
import org.junit.Before; |
||||||
|
import org.junit.Test; |
||||||
|
import org.junit.runner.RunWith; |
||||||
|
import org.mockito.Mockito; |
||||||
|
import org.springframework.beans.factory.annotation.Autowired; |
||||||
|
import org.springframework.test.context.ContextConfiguration; |
||||||
|
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; |
||||||
|
|
||||||
|
import java.util.Date; |
||||||
|
|
||||||
|
|
||||||
|
@RunWith(SpringJUnit4ClassRunner.class) |
||||||
|
@ContextConfiguration(classes={DependencyConfig.class, SpringApplicationContext.class, SpringZKServer.class, |
||||||
|
NettyExecutorManager.class, ExecutorDispatcher.class, ZookeeperRegistryCenter.class, TaskPriorityQueueConsumer.class, |
||||||
|
ZookeeperNodeManager.class, ZookeeperCachedOperator.class, ZookeeperConfig.class}) |
||||||
|
public class TaskPriorityQueueConsumerTest { |
||||||
|
|
||||||
|
|
||||||
|
@Autowired |
||||||
|
private TaskPriorityQueue taskPriorityQueue; |
||||||
|
|
||||||
|
@Autowired |
||||||
|
private TaskPriorityQueueConsumer taskPriorityQueueConsumer; |
||||||
|
|
||||||
|
@Autowired |
||||||
|
private ProcessService processService; |
||||||
|
|
||||||
|
@Autowired |
||||||
|
private ExecutorDispatcher dispatcher; |
||||||
|
|
||||||
|
@Before |
||||||
|
public void init(){ |
||||||
|
|
||||||
|
Tenant tenant = new Tenant(); |
||||||
|
tenant.setId(1); |
||||||
|
tenant.setTenantCode("journey"); |
||||||
|
tenant.setTenantName("journey"); |
||||||
|
tenant.setDescription("journey"); |
||||||
|
tenant.setQueueId(1); |
||||||
|
tenant.setCreateTime(new Date()); |
||||||
|
tenant.setUpdateTime(new Date()); |
||||||
|
|
||||||
|
Mockito.when(processService.getTenantForProcess(1,2)).thenReturn(tenant); |
||||||
|
|
||||||
|
Mockito.when(processService.queryUserQueueByProcessInstanceId(1)).thenReturn("default"); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
@Test |
||||||
|
public void testSHELLTask() throws Exception { |
||||||
|
TaskInstance taskInstance = new TaskInstance(); |
||||||
|
taskInstance.setId(1); |
||||||
|
taskInstance.setTaskType("SHELL"); |
||||||
|
taskInstance.setProcessDefinitionId(1); |
||||||
|
taskInstance.setProcessInstanceId(1); |
||||||
|
taskInstance.setState(ExecutionStatus.KILL); |
||||||
|
taskInstance.setTaskJson("{\"conditionResult\":\"{\\\"successNode\\\":[\\\"\\\"],\\\"failedNode\\\":[\\\"\\\"]}\",\"conditionsTask\":false,\"depList\":[],\"dependence\":\"{}\",\"forbidden\":false,\"id\":\"tasks-55201\",\"maxRetryTimes\":0,\"name\":\"测试任务\",\"params\":\"{\\\"rawScript\\\":\\\"echo \\\\\\\"测试任务\\\\\\\"\\\",\\\"localParams\\\":[],\\\"resourceList\\\":[]}\",\"preTasks\":\"[]\",\"retryInterval\":1,\"runFlag\":\"NORMAL\",\"taskInstancePriority\":\"MEDIUM\",\"taskTimeoutParameter\":{\"enable\":false,\"interval\":0},\"timeout\":\"{\\\"enable\\\":false,\\\"strategy\\\":\\\"\\\"}\",\"type\":\"SHELL\",\"workerGroup\":\"default\"}"); |
||||||
|
taskInstance.setProcessInstancePriority(Priority.MEDIUM); |
||||||
|
taskInstance.setWorkerGroup("default"); |
||||||
|
taskInstance.setExecutorId(2); |
||||||
|
|
||||||
|
ProcessInstance processInstance = new ProcessInstance(); |
||||||
|
processInstance.setTenantId(1); |
||||||
|
processInstance.setCommandType(CommandType.START_PROCESS); |
||||||
|
taskInstance.setProcessInstance(processInstance); |
||||||
|
|
||||||
|
ProcessDefinition processDefinition = new ProcessDefinition(); |
||||||
|
processDefinition.setUserId(2); |
||||||
|
processDefinition.setProjectId(1); |
||||||
|
taskInstance.setProcessDefine(processDefinition); |
||||||
|
|
||||||
|
Mockito.when(processService.getTaskInstanceDetailByTaskId(1)).thenReturn(taskInstance); |
||||||
|
taskPriorityQueue.put("2_1_2_1_default"); |
||||||
|
|
||||||
|
Thread.sleep(10000); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
@Test |
||||||
|
public void testSQLTask() throws Exception { |
||||||
|
TaskInstance taskInstance = new TaskInstance(); |
||||||
|
taskInstance.setId(1); |
||||||
|
taskInstance.setTaskType("SQL"); |
||||||
|
taskInstance.setProcessDefinitionId(1); |
||||||
|
taskInstance.setProcessInstanceId(1); |
||||||
|
taskInstance.setState(ExecutionStatus.KILL); |
||||||
|
taskInstance.setTaskJson("{\"conditionsTask\":false,\"depList\":[],\"dependence\":\"{}\",\"forbidden\":false,\"id\":\"tasks-3655\",\"maxRetryTimes\":0,\"name\":\"UDF测试\",\"params\":\"{\\\"postStatements\\\":[],\\\"connParams\\\":\\\"\\\",\\\"receiversCc\\\":\\\"\\\",\\\"udfs\\\":\\\"1\\\",\\\"type\\\":\\\"HIVE\\\",\\\"title\\\":\\\"test\\\",\\\"sql\\\":\\\"select id,name,ds,zodia(ds) from t_journey_user\\\",\\\"preStatements\\\":[],\\\"sqlType\\\":0,\\\"receivers\\\":\\\"825193156@qq.com\\\",\\\"datasource\\\":3,\\\"showType\\\":\\\"TABLE\\\",\\\"localParams\\\":[]}\",\"preTasks\":\"[]\",\"retryInterval\":1,\"runFlag\":\"NORMAL\",\"taskInstancePriority\":\"MEDIUM\",\"taskTimeoutParameter\":{\"enable\":false,\"interval\":0},\"timeout\":\"{\\\"enable\\\":false,\\\"strategy\\\":\\\"\\\"}\",\"type\":\"SQL\"}"); |
||||||
|
taskInstance.setProcessInstancePriority(Priority.MEDIUM); |
||||||
|
taskInstance.setWorkerGroup("default"); |
||||||
|
taskInstance.setExecutorId(2); |
||||||
|
|
||||||
|
ProcessInstance processInstance = new ProcessInstance(); |
||||||
|
processInstance.setTenantId(1); |
||||||
|
processInstance.setCommandType(CommandType.START_PROCESS); |
||||||
|
taskInstance.setProcessInstance(processInstance); |
||||||
|
|
||||||
|
ProcessDefinition processDefinition = new ProcessDefinition(); |
||||||
|
processDefinition.setUserId(2); |
||||||
|
processDefinition.setProjectId(1); |
||||||
|
taskInstance.setProcessDefine(processDefinition); |
||||||
|
|
||||||
|
Mockito.when(processService.getTaskInstanceDetailByTaskId(1)).thenReturn(taskInstance); |
||||||
|
taskPriorityQueue.put("2_1_2_1_default"); |
||||||
|
|
||||||
|
DataSource dataSource = new DataSource(); |
||||||
|
dataSource.setId(1); |
||||||
|
dataSource.setName("sqlDatasource"); |
||||||
|
dataSource.setType(DbType.MYSQL); |
||||||
|
dataSource.setUserId(2); |
||||||
|
dataSource.setConnectionParams("{\"address\":\"jdbc:mysql://192.168.221.185:3306\",\"database\":\"dolphinscheduler_qiaozhanwei\",\"jdbcUrl\":\"jdbc:mysql://192.168.221.185:3306/dolphinscheduler_qiaozhanwei\",\"user\":\"root\",\"password\":\"root@123\"}"); |
||||||
|
dataSource.setCreateTime(new Date()); |
||||||
|
dataSource.setUpdateTime(new Date()); |
||||||
|
|
||||||
|
Mockito.when(processService.findDataSourceById(1)).thenReturn(dataSource); |
||||||
|
|
||||||
|
Thread.sleep(10000); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
@Test |
||||||
|
public void testDataxTask() throws Exception { |
||||||
|
TaskInstance taskInstance = new TaskInstance(); |
||||||
|
taskInstance.setId(1); |
||||||
|
taskInstance.setTaskType("DATAX"); |
||||||
|
taskInstance.setProcessDefinitionId(1); |
||||||
|
taskInstance.setProcessInstanceId(1); |
||||||
|
taskInstance.setState(ExecutionStatus.KILL); |
||||||
|
taskInstance.setTaskJson("{\"conditionResult\":\"{\\\"successNode\\\":[\\\"\\\"],\\\"failedNode\\\":[\\\"\\\"]}\",\"conditionsTask\":false,\"depList\":[],\"dependence\":\"{}\",\"forbidden\":false,\"id\":\"tasks-97625\",\"maxRetryTimes\":0,\"name\":\"MySQL数据相互导入\",\"params\":\"{\\\"targetTable\\\":\\\"pv2\\\",\\\"postStatements\\\":[],\\\"jobSpeedRecord\\\":1000,\\\"customConfig\\\":0,\\\"dtType\\\":\\\"MYSQL\\\",\\\"dsType\\\":\\\"MYSQL\\\",\\\"jobSpeedByte\\\":0,\\\"dataSource\\\":80,\\\"dataTarget\\\":80,\\\"sql\\\":\\\"SELECT dt,count FROM pv\\\",\\\"preStatements\\\":[]}\",\"preTasks\":\"[]\",\"retryInterval\":1,\"runFlag\":\"NORMAL\",\"taskInstancePriority\":\"MEDIUM\",\"taskTimeoutParameter\":{\"enable\":false,\"interval\":0},\"timeout\":\"{\\\"enable\\\":false,\\\"strategy\\\":\\\"\\\"}\",\"type\":\"DATAX\",\"workerGroup\":\"default\"}"); |
||||||
|
taskInstance.setProcessInstancePriority(Priority.MEDIUM); |
||||||
|
taskInstance.setWorkerGroup("default"); |
||||||
|
taskInstance.setExecutorId(2); |
||||||
|
|
||||||
|
ProcessInstance processInstance = new ProcessInstance(); |
||||||
|
processInstance.setTenantId(1); |
||||||
|
processInstance.setCommandType(CommandType.START_PROCESS); |
||||||
|
taskInstance.setProcessInstance(processInstance); |
||||||
|
|
||||||
|
ProcessDefinition processDefinition = new ProcessDefinition(); |
||||||
|
processDefinition.setUserId(2); |
||||||
|
processDefinition.setProjectId(1); |
||||||
|
taskInstance.setProcessDefine(processDefinition); |
||||||
|
|
||||||
|
Mockito.when(processService.getTaskInstanceDetailByTaskId(1)).thenReturn(taskInstance); |
||||||
|
taskPriorityQueue.put("2_1_2_1_default"); |
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
DataSource dataSource = new DataSource(); |
||||||
|
dataSource.setId(80); |
||||||
|
dataSource.setName("datax"); |
||||||
|
dataSource.setType(DbType.MYSQL); |
||||||
|
dataSource.setUserId(2); |
||||||
|
dataSource.setConnectionParams("{\"address\":\"jdbc:mysql://192.168.221.185:3306\",\"database\":\"dolphinscheduler_qiaozhanwei\",\"jdbcUrl\":\"jdbc:mysql://192.168.221.185:3306/dolphinscheduler_qiaozhanwei\",\"user\":\"root\",\"password\":\"root@123\"}"); |
||||||
|
dataSource.setCreateTime(new Date()); |
||||||
|
dataSource.setUpdateTime(new Date()); |
||||||
|
|
||||||
|
Mockito.when(processService.findDataSourceById(80)).thenReturn(dataSource); |
||||||
|
|
||||||
|
Thread.sleep(10000); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
@Test |
||||||
|
public void testSqoopTask() throws Exception { |
||||||
|
TaskInstance taskInstance = new TaskInstance(); |
||||||
|
taskInstance.setId(1); |
||||||
|
taskInstance.setTaskType("SQOOP"); |
||||||
|
taskInstance.setProcessDefinitionId(1); |
||||||
|
taskInstance.setProcessInstanceId(1); |
||||||
|
taskInstance.setState(ExecutionStatus.KILL); |
||||||
|
taskInstance.setTaskJson("{\"conditionResult\":\"{\\\"successNode\\\":[\\\"\\\"],\\\"failedNode\\\":[\\\"\\\"]}\",\"conditionsTask\":false,\"depList\":[],\"dependence\":\"{}\",\"forbidden\":false,\"id\":\"tasks-63634\",\"maxRetryTimes\":0,\"name\":\"MySQL数据导入HDSF\",\"params\":\"{\\\"sourceType\\\":\\\"MYSQL\\\",\\\"targetType\\\":\\\"HDFS\\\",\\\"targetParams\\\":\\\"{\\\\\\\"targetPath\\\\\\\":\\\\\\\"/test/datatest\\\\\\\",\\\\\\\"deleteTargetDir\\\\\\\":true,\\\\\\\"fileType\\\\\\\":\\\\\\\"--as-textfile\\\\\\\",\\\\\\\"compressionCodec\\\\\\\":\\\\\\\"\\\\\\\",\\\\\\\"fieldsTerminated\\\\\\\":\\\\\\\",\\\\\\\",\\\\\\\"linesTerminated\\\\\\\":\\\\\\\"\\\\\\\\\\\\\\\\n\\\\\\\"}\\\",\\\"modelType\\\":\\\"import\\\",\\\"sourceParams\\\":\\\"{\\\\\\\"srcType\\\\\\\":\\\\\\\"MYSQL\\\\\\\",\\\\\\\"srcDatasource\\\\\\\":1,\\\\\\\"srcTable\\\\\\\":\\\\\\\"t_ds_user\\\\\\\",\\\\\\\"srcQueryType\\\\\\\":\\\\\\\"0\\\\\\\",\\\\\\\"srcQuerySql\\\\\\\":\\\\\\\"\\\\\\\",\\\\\\\"srcColumnType\\\\\\\":\\\\\\\"0\\\\\\\",\\\\\\\"srcColumns\\\\\\\":\\\\\\\"\\\\\\\",\\\\\\\"srcConditionList\\\\\\\":[],\\\\\\\"mapColumnHive\\\\\\\":[],\\\\\\\"mapColumnJava\\\\\\\":[]}\\\",\\\"localParams\\\":[],\\\"concurrency\\\":1}\",\"preTasks\":\"[]\",\"retryInterval\":1,\"runFlag\":\"NORMAL\",\"taskInstancePriority\":\"MEDIUM\",\"taskTimeoutParameter\":{\"enable\":false,\"interval\":0},\"timeout\":\"{\\\"enable\\\":false,\\\"strategy\\\":\\\"\\\"}\",\"type\":\"SQOOP\",\"workerGroup\":\"default\"}"); |
||||||
|
taskInstance.setProcessInstancePriority(Priority.MEDIUM); |
||||||
|
taskInstance.setWorkerGroup("default"); |
||||||
|
taskInstance.setExecutorId(2); |
||||||
|
|
||||||
|
ProcessInstance processInstance = new ProcessInstance(); |
||||||
|
processInstance.setTenantId(1); |
||||||
|
processInstance.setCommandType(CommandType.START_PROCESS); |
||||||
|
taskInstance.setProcessInstance(processInstance); |
||||||
|
|
||||||
|
ProcessDefinition processDefinition = new ProcessDefinition(); |
||||||
|
processDefinition.setUserId(2); |
||||||
|
processDefinition.setProjectId(1); |
||||||
|
taskInstance.setProcessDefine(processDefinition); |
||||||
|
|
||||||
|
Mockito.when(processService.getTaskInstanceDetailByTaskId(1)).thenReturn(taskInstance); |
||||||
|
taskPriorityQueue.put("2_1_2_1_default"); |
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
DataSource dataSource = new DataSource(); |
||||||
|
dataSource.setId(1); |
||||||
|
dataSource.setName("datax"); |
||||||
|
dataSource.setType(DbType.MYSQL); |
||||||
|
dataSource.setUserId(2); |
||||||
|
dataSource.setConnectionParams("{\"address\":\"jdbc:mysql://192.168.221.185:3306\",\"database\":\"dolphinscheduler_qiaozhanwei\",\"jdbcUrl\":\"jdbc:mysql://192.168.221.185:3306/dolphinscheduler_qiaozhanwei\",\"user\":\"root\",\"password\":\"root@123\"}"); |
||||||
|
dataSource.setCreateTime(new Date()); |
||||||
|
dataSource.setUpdateTime(new Date()); |
||||||
|
|
||||||
|
Mockito.when(processService.findDataSourceById(1)).thenReturn(dataSource); |
||||||
|
|
||||||
|
Thread.sleep(10000); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
@Test |
||||||
|
public void testTaskInstanceIsFinalState(){ |
||||||
|
TaskInstance taskInstance = new TaskInstance(); |
||||||
|
taskInstance.setId(1); |
||||||
|
taskInstance.setTaskType("SHELL"); |
||||||
|
taskInstance.setProcessDefinitionId(1); |
||||||
|
taskInstance.setProcessInstanceId(1); |
||||||
|
taskInstance.setState(ExecutionStatus.KILL); |
||||||
|
taskInstance.setTaskJson("{\"conditionResult\":\"{\\\"successNode\\\":[\\\"\\\"],\\\"failedNode\\\":[\\\"\\\"]}\",\"conditionsTask\":false,\"depList\":[],\"dependence\":\"{}\",\"forbidden\":false,\"id\":\"tasks-55201\",\"maxRetryTimes\":0,\"name\":\"测试任务\",\"params\":\"{\\\"rawScript\\\":\\\"echo \\\\\\\"测试任务\\\\\\\"\\\",\\\"localParams\\\":[],\\\"resourceList\\\":[]}\",\"preTasks\":\"[]\",\"retryInterval\":1,\"runFlag\":\"NORMAL\",\"taskInstancePriority\":\"MEDIUM\",\"taskTimeoutParameter\":{\"enable\":false,\"interval\":0},\"timeout\":\"{\\\"enable\\\":false,\\\"strategy\\\":\\\"\\\"}\",\"type\":\"SHELL\",\"workerGroup\":\"default\"}"); |
||||||
|
taskInstance.setProcessInstancePriority(Priority.MEDIUM); |
||||||
|
taskInstance.setWorkerGroup("default"); |
||||||
|
taskInstance.setExecutorId(2); |
||||||
|
|
||||||
|
|
||||||
|
Mockito.when( processService.findTaskInstanceById(1)).thenReturn(taskInstance); |
||||||
|
|
||||||
|
taskPriorityQueueConsumer.taskInstanceIsFinalState(1); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
} |
||||||
@ -0,0 +1,80 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.server.master.runner; |
||||||
|
|
||||||
|
import junit.framework.Assert; |
||||||
|
import org.apache.dolphinscheduler.server.master.config.MasterConfig; |
||||||
|
import org.apache.dolphinscheduler.server.master.consumer.TaskPriorityQueueConsumer; |
||||||
|
import org.apache.dolphinscheduler.server.master.dispatch.ExecutorDispatcher; |
||||||
|
import org.apache.dolphinscheduler.server.master.dispatch.executor.NettyExecutorManager; |
||||||
|
import org.apache.dolphinscheduler.server.registry.DependencyConfig; |
||||||
|
import org.apache.dolphinscheduler.server.registry.ZookeeperNodeManager; |
||||||
|
import org.apache.dolphinscheduler.server.registry.ZookeeperRegistryCenter; |
||||||
|
import org.apache.dolphinscheduler.server.zk.SpringZKServer; |
||||||
|
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; |
||||||
|
import org.apache.dolphinscheduler.service.queue.TaskPriorityQueueImpl; |
||||||
|
import org.apache.dolphinscheduler.service.zk.ZookeeperCachedOperator; |
||||||
|
import org.apache.dolphinscheduler.service.zk.ZookeeperConfig; |
||||||
|
import org.junit.Test; |
||||||
|
import org.junit.runner.RunWith; |
||||||
|
import org.mockito.Mockito; |
||||||
|
import org.springframework.test.context.ContextConfiguration; |
||||||
|
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; |
||||||
|
|
||||||
|
import java.util.HashSet; |
||||||
|
import java.util.Set; |
||||||
|
|
||||||
|
@RunWith(SpringJUnit4ClassRunner.class) |
||||||
|
@ContextConfiguration(classes={DependencyConfig.class, SpringApplicationContext.class, SpringZKServer.class, |
||||||
|
NettyExecutorManager.class, ExecutorDispatcher.class, ZookeeperRegistryCenter.class, TaskPriorityQueueConsumer.class, |
||||||
|
ZookeeperNodeManager.class, ZookeeperCachedOperator.class, ZookeeperConfig.class, MasterConfig.class}) |
||||||
|
public class MasterTaskExecThreadTest { |
||||||
|
|
||||||
|
@Test |
||||||
|
public void testExistsValidWorkerGroup1(){ |
||||||
|
ZookeeperRegistryCenter zookeeperRegistryCenter = Mockito.mock(ZookeeperRegistryCenter.class); |
||||||
|
Mockito.when(zookeeperRegistryCenter.getWorkerGroupDirectly()).thenReturn(null); |
||||||
|
MasterTaskExecThread masterTaskExecThread = new MasterTaskExecThread(null); |
||||||
|
masterTaskExecThread.existsValidWorkerGroup("default"); |
||||||
|
} |
||||||
|
@Test |
||||||
|
public void testExistsValidWorkerGroup2(){ |
||||||
|
ZookeeperRegistryCenter zookeeperRegistryCenter = Mockito.mock(ZookeeperRegistryCenter.class); |
||||||
|
Set<String> workerGorups = new HashSet<>(); |
||||||
|
workerGorups.add("test1"); |
||||||
|
workerGorups.add("test2"); |
||||||
|
|
||||||
|
Mockito.when(zookeeperRegistryCenter.getWorkerGroupDirectly()).thenReturn(workerGorups); |
||||||
|
MasterTaskExecThread masterTaskExecThread = new MasterTaskExecThread(null); |
||||||
|
masterTaskExecThread.existsValidWorkerGroup("default"); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void testExistsValidWorkerGroup3(){ |
||||||
|
ZookeeperRegistryCenter zookeeperRegistryCenter = Mockito.mock(ZookeeperRegistryCenter.class); |
||||||
|
Set<String> workerGorups = new HashSet<>(); |
||||||
|
workerGorups.add("test1"); |
||||||
|
|
||||||
|
Mockito.when(zookeeperRegistryCenter.getWorkerGroupDirectly()).thenReturn(workerGorups); |
||||||
|
Mockito.when(zookeeperRegistryCenter.getWorkerGroupNodesDirectly("test1")).thenReturn(workerGorups); |
||||||
|
MasterTaskExecThread masterTaskExecThread = new MasterTaskExecThread(null); |
||||||
|
masterTaskExecThread.existsValidWorkerGroup("test1"); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
} |
||||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue