dailidong
4 years ago
committed by
GitHub
111 changed files with 2776 additions and 3817 deletions
@ -0,0 +1,68 @@
|
||||
### Dolphin Scheduler的Ambari插件使用说明 |
||||
|
||||
##### 备注 |
||||
|
||||
1. 本文档适用于对Ambari中基本了解的用户 |
||||
2. 本文档是对已安装Ambari服务添加Dolphin Scheduler(1.3.0版本)服务的说明 |
||||
|
||||
##### 一 安装准备 |
||||
|
||||
1. 准备RPM包 |
||||
|
||||
- 在源码dolphinscheduler-dist目录下执行命令```mvn -U clean install rpm:attached-rpm -Prpmbuild -Dmaven.test.skip=true -X```即可生成(在目录 dolphinscheduler-dist/target/rpm/apache-dolphinscheduler-incubating/RPMS/noarch 下) |
||||
|
||||
2. 创建DS的安装用户--权限 |
||||
|
||||
3. 初始化数据库信息 |
||||
|
||||
``` |
||||
-- 创建Dolphin Scheduler的数据库：dolphinscheduler |
||||
CREATE DATABASE dolphinscheduler DEFAULT CHARACTER SET utf8 DEFAULT COLLATE |
||||
utf8_general_ci; |
||||
|
||||
-- 初始化dolphinscheduler数据库的用户和密码，并分配权限 |
||||
-- 替换下面sql语句中的{user}为dolphinscheduler数据库的用户 |
||||
GRANT ALL PRIVILEGES ON dolphinscheduler.* TO '{user}'@'%' IDENTIFIED BY '{password}'; |
||||
GRANT ALL PRIVILEGES ON dolphinscheduler.* TO '{user}'@'localhost' IDENTIFIED BY |
||||
'{password}'; |
||||
flush privileges; |
||||
``` |
||||
|
||||
|
||||
|
||||
##### 二 Ambari安装Dolphin Scheduler |
||||
|
||||
1. Ambari界面安装Dolphin Scheduler |
||||
|
||||
 |
||||
|
||||
2. 选择Dolphin Scheduler的Master安装的节点 |
||||
|
||||
 |
||||
|
||||
3. 配置Dolphin Scheduler的Worker、Api、Logger、Alert安装的节点 |
||||
|
||||
 |
||||
|
||||
4. 设置Dolphin Scheduler服务的安装用户（**步骤一中创建的**）及所属的用户组 |
||||
|
||||
 |
||||
|
||||
5. 配置数据库的信息（和步骤一中初始化数据库中一致） |
||||
|
||||
 |
||||
|
||||
6. 配置其它的信息--如果需要的话 |
||||
|
||||
 |
||||
|
||||
 |
||||
|
||||
7. 正常执行接下来的步骤 |
||||
|
||||
 |
||||
|
||||
8. 安装成功后的界面 |
||||
|
||||
 |
||||
|
||||
@ -1,164 +0,0 @@
|
||||
{ |
||||
"DOLPHIN": { |
||||
"service": [], |
||||
"DOLPHIN_API": [ |
||||
{ |
||||
"name": "dolphin_api_port_check", |
||||
"label": "dolphin_api_port_check", |
||||
"description": "dolphin_api_port_check.", |
||||
"interval": 10, |
||||
"scope": "ANY", |
||||
"source": { |
||||
"type": "PORT", |
||||
"uri": "{{dolphin-application-api/server.port}}", |
||||
"default_port": 12345, |
||||
"reporting": { |
||||
"ok": { |
||||
"text": "TCP OK - {0:.3f}s response on port {1}" |
||||
}, |
||||
"warning": { |
||||
"text": "TCP OK - {0:.3f}s response on port {1}", |
||||
"value": 1.5 |
||||
}, |
||||
"critical": { |
||||
"text": "Connection failed: {0} to {1}:{2}", |
||||
"value": 5.0 |
||||
} |
||||
} |
||||
} |
||||
} |
||||
], |
||||
"DOLPHIN_MASTER": [ |
||||
{ |
||||
"name": "DOLPHIN_MASTER_CHECK", |
||||
"label": "check dolphin scheduler master status", |
||||
"description": "", |
||||
"interval":10, |
||||
"scope": "HOST", |
||||
"enabled": true, |
||||
"source": { |
||||
"type": "SCRIPT", |
||||
"path": "DOLPHIN/2.0.0/package/alerts/alert_dolphin_scheduler_status.py", |
||||
"parameters": [ |
||||
|
||||
{ |
||||
"name": "connection.timeout", |
||||
"display_name": "Connection Timeout", |
||||
"value": 5.0, |
||||
"type": "NUMERIC", |
||||
"description": "The maximum time before this alert is considered to be CRITICAL", |
||||
"units": "seconds", |
||||
"threshold": "CRITICAL" |
||||
}, |
||||
{ |
||||
"name": "alertName", |
||||
"display_name": "alertName", |
||||
"value": "DOLPHIN_MASTER", |
||||
"type": "STRING", |
||||
"description": "alert name" |
||||
} |
||||
] |
||||
} |
||||
} |
||||
], |
||||
"DOLPHIN_WORKER": [ |
||||
{ |
||||
"name": "DOLPHIN_WORKER_CHECK", |
||||
"label": "check dolphin scheduler worker status", |
||||
"description": "", |
||||
"interval":10, |
||||
"scope": "HOST", |
||||
"enabled": true, |
||||
"source": { |
||||
"type": "SCRIPT", |
||||
"path": "DOLPHIN/2.0.0/package/alerts/alert_dolphin_scheduler_status.py", |
||||
"parameters": [ |
||||
|
||||
{ |
||||
"name": "connection.timeout", |
||||
"display_name": "Connection Timeout", |
||||
"value": 5.0, |
||||
"type": "NUMERIC", |
||||
"description": "The maximum time before this alert is considered to be CRITICAL", |
||||
"units": "seconds", |
||||
"threshold": "CRITICAL" |
||||
}, |
||||
{ |
||||
"name": "alertName", |
||||
"display_name": "alertName", |
||||
"value": "DOLPHIN_WORKER", |
||||
"type": "STRING", |
||||
"description": "alert name" |
||||
} |
||||
] |
||||
} |
||||
} |
||||
], |
||||
"DOLPHIN_ALERT": [ |
||||
{ |
||||
"name": "DOLPHIN_DOLPHIN_ALERT_CHECK", |
||||
"label": "check dolphin scheduler alert status", |
||||
"description": "", |
||||
"interval":10, |
||||
"scope": "HOST", |
||||
"enabled": true, |
||||
"source": { |
||||
"type": "SCRIPT", |
||||
"path": "DOLPHIN/2.0.0/package/alerts/alert_dolphin_scheduler_status.py", |
||||
"parameters": [ |
||||
|
||||
{ |
||||
"name": "connection.timeout", |
||||
"display_name": "Connection Timeout", |
||||
"value": 5.0, |
||||
"type": "NUMERIC", |
||||
"description": "The maximum time before this alert is considered to be CRITICAL", |
||||
"units": "seconds", |
||||
"threshold": "CRITICAL" |
||||
}, |
||||
{ |
||||
"name": "alertName", |
||||
"display_name": "alertName", |
||||
"value": "DOLPHIN_ALERT", |
||||
"type": "STRING", |
||||
"description": "alert name" |
||||
} |
||||
] |
||||
} |
||||
} |
||||
], |
||||
"DOLPHIN_ALERT": [ |
||||
{ |
||||
"name": "DOLPHIN_DOLPHIN_LOGGER_CHECK", |
||||
"label": "check dolphin scheduler alert status", |
||||
"description": "", |
||||
"interval":10, |
||||
"scope": "HOST", |
||||
"enabled": true, |
||||
"source": { |
||||
"type": "SCRIPT", |
||||
"path": "DOLPHIN/2.0.0/package/alerts/alert_dolphin_scheduler_status.py", |
||||
"parameters": [ |
||||
|
||||
{ |
||||
"name": "connection.timeout", |
||||
"display_name": "Connection Timeout", |
||||
"value": 5.0, |
||||
"type": "NUMERIC", |
||||
"description": "The maximum time before this alert is considered to be CRITICAL", |
||||
"units": "seconds", |
||||
"threshold": "CRITICAL" |
||||
}, |
||||
{ |
||||
"name": "alertName", |
||||
"display_name": "alertName", |
||||
"value": "DOLPHIN_LOGGER", |
||||
"type": "STRING", |
||||
"description": "alert name" |
||||
} |
||||
] |
||||
} |
||||
} |
||||
] |
||||
} |
||||
} |
||||
@ -1,143 +0,0 @@
|
||||
<!-- |
||||
~ Licensed to the Apache Software Foundation (ASF) under one or more |
||||
~ contributor license agreements. See the NOTICE file distributed with |
||||
~ this work for additional information regarding copyright ownership. |
||||
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
~ (the "License"); you may not use this file except in compliance with |
||||
~ the License. You may obtain a copy of the License at |
||||
~ |
||||
~ http://www.apache.org/licenses/LICENSE-2.0 |
||||
~ |
||||
~ Unless required by applicable law or agreed to in writing, software |
||||
~ distributed under the License is distributed on an "AS IS" BASIS, |
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
~ See the License for the specific language governing permissions and |
||||
~ limitations under the License. |
||||
--> |
||||
<configuration> |
||||
<property> |
||||
<name>alert.type</name> |
||||
<value>EMAIL</value> |
||||
<description>alert type is EMAIL/SMS</description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>alert.template</name> |
||||
<value>html</value> |
||||
<description>alter msg template, default is html template</description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>mail.protocol</name> |
||||
<value>SMTP</value> |
||||
<description></description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>mail.server.host</name> |
||||
<value>xxx.xxx.com</value> |
||||
<description></description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>mail.server.port</name> |
||||
<value>25</value> |
||||
<value-attributes> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
<description></description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>mail.sender</name> |
||||
<value>admin</value> |
||||
<description></description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>mail.user</name> |
||||
<value>admin</value> |
||||
<description></description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>mail.passwd</name> |
||||
<value>000000</value> |
||||
<description></description> |
||||
<property-type>PASSWORD</property-type> |
||||
<value-attributes> |
||||
<type>password</type> |
||||
</value-attributes> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
|
||||
<property> |
||||
<name>mail.smtp.starttls.enable</name> |
||||
<value>true</value> |
||||
<value-attributes> |
||||
<type>boolean</type> |
||||
</value-attributes> |
||||
<description></description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>mail.smtp.ssl.enable</name> |
||||
<value>true</value> |
||||
<value-attributes> |
||||
<type>boolean</type> |
||||
</value-attributes> |
||||
<description></description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>mail.smtp.ssl.trust</name> |
||||
<value>xxx.xxx.com</value> |
||||
<description></description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
|
||||
<property> |
||||
<name>enterprise.wechat.enable</name> |
||||
<value>false</value> |
||||
<description></description> |
||||
<value-attributes> |
||||
<type>value-list</type> |
||||
<entries> |
||||
<entry> |
||||
<value>true</value> |
||||
<label>Enabled</label> |
||||
</entry> |
||||
<entry> |
||||
<value>false</value> |
||||
<label>Disabled</label> |
||||
</entry> |
||||
</entries> |
||||
<selection-cardinality>1</selection-cardinality> |
||||
</value-attributes> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>enterprise.wechat.corp.id</name> |
||||
<value>wechatId</value> |
||||
<description></description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>enterprise.wechat.secret</name> |
||||
<value>secret</value> |
||||
<description></description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>enterprise.wechat.agent.id</name> |
||||
<value>agentId</value> |
||||
<description></description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>enterprise.wechat.users</name> |
||||
<value>wechatUsers</value> |
||||
<description></description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
</configuration> |
||||
@ -1,87 +0,0 @@
|
||||
<!-- |
||||
~ Licensed to the Apache Software Foundation (ASF) under one or more |
||||
~ contributor license agreements. See the NOTICE file distributed with |
||||
~ this work for additional information regarding copyright ownership. |
||||
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
~ (the "License"); you may not use this file except in compliance with |
||||
~ the License. You may obtain a copy of the License at |
||||
~ |
||||
~ http://www.apache.org/licenses/LICENSE-2.0 |
||||
~ |
||||
~ Unless required by applicable law or agreed to in writing, software |
||||
~ distributed under the License is distributed on an "AS IS" BASIS, |
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
~ See the License for the specific language governing permissions and |
||||
~ limitations under the License. |
||||
--> |
||||
<configuration> |
||||
<property> |
||||
<name>server.port</name> |
||||
<value>12345</value> |
||||
<description> |
||||
server port |
||||
</description> |
||||
<value-attributes> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
</property> |
||||
<property> |
||||
<name>server.servlet.session.timeout</name> |
||||
<value>7200</value> |
||||
<value-attributes> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
<description> |
||||
</description> |
||||
</property> |
||||
<property> |
||||
<name>server.servlet.context-path</name> |
||||
<value>/dolphinscheduler/</value> |
||||
<description> |
||||
</description> |
||||
</property> |
||||
<property> |
||||
<name>spring.servlet.multipart.max-file-size</name> |
||||
<value>1024</value> |
||||
<value-attributes> |
||||
<unit>MB</unit> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
<description> |
||||
</description> |
||||
</property> |
||||
<property> |
||||
<name>spring.servlet.multipart.max-request-size</name> |
||||
<value>1024</value> |
||||
<value-attributes> |
||||
<unit>MB</unit> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
<description> |
||||
</description> |
||||
</property> |
||||
<property> |
||||
<name>server.jetty.max-http-post-size</name> |
||||
<value>5000000</value> |
||||
<value-attributes> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
<description> |
||||
</description> |
||||
</property> |
||||
<property> |
||||
<name>spring.messages.encoding</name> |
||||
<value>UTF-8</value> |
||||
<description></description> |
||||
</property> |
||||
<property> |
||||
<name>spring.messages.basename</name> |
||||
<value>i18n/messages</value> |
||||
<description></description> |
||||
</property> |
||||
<property> |
||||
<name>security.authentication.type</name> |
||||
<value>PASSWORD</value> |
||||
<description></description> |
||||
</property> |
||||
</configuration> |
||||
@ -1,158 +0,0 @@
|
||||
<!-- |
||||
~ Licensed to the Apache Software Foundation (ASF) under one or more |
||||
~ contributor license agreements. See the NOTICE file distributed with |
||||
~ this work for additional information regarding copyright ownership. |
||||
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
~ (the "License"); you may not use this file except in compliance with |
||||
~ the License. You may obtain a copy of the License at |
||||
~ |
||||
~ http://www.apache.org/licenses/LICENSE-2.0 |
||||
~ |
||||
~ Unless required by applicable law or agreed to in writing, software |
||||
~ distributed under the License is distributed on an "AS IS" BASIS, |
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
~ See the License for the specific language governing permissions and |
||||
~ limitations under the License. |
||||
--> |
||||
<configuration> |
||||
<property> |
||||
<name>resource.storage.type</name> |
||||
<display-name>Choose Resource Upload Startup Type</display-name> |
||||
<description> |
||||
Resource upload startup type : HDFS,S3,NONE |
||||
</description> |
||||
<value>NONE</value> |
||||
<value-attributes> |
||||
<type>value-list</type> |
||||
<entries> |
||||
<entry> |
||||
<value>HDFS</value> |
||||
<label>HDFS</label> |
||||
</entry> |
||||
<entry> |
||||
<value>S3</value> |
||||
<label>S3</label> |
||||
</entry> |
||||
<entry> |
||||
<value>NONE</value> |
||||
<label>NONE</label> |
||||
</entry> |
||||
</entries> |
||||
<selection-cardinality>1</selection-cardinality> |
||||
</value-attributes> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>resource.upload.path</name> |
||||
<value>/dolphinscheduler</value> |
||||
<description> |
||||
resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended |
||||
</description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>data.basedir.path</name> |
||||
<value>/tmp/dolphinscheduler</value> |
||||
<description> |
||||
user data local directory path, please make sure the directory exists and have read write permissions |
||||
</description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
|
||||
<property> |
||||
<name>hadoop.security.authentication.startup.state</name> |
||||
<value>false</value> |
||||
<value-attributes> |
||||
<type>value-list</type> |
||||
<entries> |
||||
<entry> |
||||
<value>true</value> |
||||
<label>Enabled</label> |
||||
</entry> |
||||
<entry> |
||||
<value>false</value> |
||||
<label>Disabled</label> |
||||
</entry> |
||||
</entries> |
||||
<selection-cardinality>1</selection-cardinality> |
||||
</value-attributes> |
||||
<description>whether kerberos starts</description> |
||||
</property> |
||||
<property> |
||||
<name>java.security.krb5.conf.path</name> |
||||
<value>/opt/krb5.conf</value> |
||||
<description> |
||||
java.security.krb5.conf path |
||||
</description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>login.user.keytab.username</name> |
||||
<value>hdfs-mycluster@ESZ.COM</value> |
||||
<description> |
||||
LoginUserFromKeytab user |
||||
</description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>login.user.keytab.path</name> |
||||
<value>/opt/hdfs.headless.keytab</value> |
||||
<description> |
||||
LoginUserFromKeytab path |
||||
</description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>resource.view.suffixs</name> |
||||
<value>txt,log,sh,conf,cfg,py,java,sql,hql,xml,properties</value> |
||||
<description></description> |
||||
</property> |
||||
<property> |
||||
<name>hdfs.root.user</name> |
||||
<value>hdfs</value> |
||||
<description> |
||||
Users who have permission to create directories under the HDFS root path |
||||
</description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>fs.defaultFS</name> |
||||
<value>hdfs://mycluster:8020</value> |
||||
<description> |
||||
HA or single namenode, |
||||
If namenode ha needs to copy core-site.xml and hdfs-site.xml to the conf directory， |
||||
support s3，for example : s3a://dolphinscheduler |
||||
</description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>fs.s3a.endpoint</name> |
||||
<value>http://host:9010</value> |
||||
<description> |
||||
s3 need，s3 endpoint |
||||
</description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>fs.s3a.access.key</name> |
||||
<value>A3DXS30FO22544RE</value> |
||||
<description> |
||||
s3 need，s3 access key |
||||
</description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>fs.s3a.secret.key</name> |
||||
<value>OloCLq3n+8+sdPHUhJ21XrSxTC+JK</value> |
||||
<description> |
||||
s3 need，s3 secret key |
||||
</description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>kerberos.expire.time</name> |
||||
<value>7</value> |
||||
<description></description> |
||||
</property> |
||||
|
||||
</configuration> |
||||
@ -1,467 +0,0 @@
|
||||
<!-- |
||||
~ Licensed to the Apache Software Foundation (ASF) under one or more |
||||
~ contributor license agreements. See the NOTICE file distributed with |
||||
~ this work for additional information regarding copyright ownership. |
||||
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
~ (the "License"); you may not use this file except in compliance with |
||||
~ the License. You may obtain a copy of the License at |
||||
~ |
||||
~ http://www.apache.org/licenses/LICENSE-2.0 |
||||
~ |
||||
~ Unless required by applicable law or agreed to in writing, software |
||||
~ distributed under the License is distributed on an "AS IS" BASIS, |
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
~ See the License for the specific language governing permissions and |
||||
~ limitations under the License. |
||||
--> |
||||
<configuration> |
||||
<property> |
||||
<name>spring.datasource.initialSize</name> |
||||
<value>5</value> |
||||
<description> |
||||
Init connection number |
||||
</description> |
||||
<value-attributes> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>spring.datasource.minIdle</name> |
||||
<value>5</value> |
||||
<description> |
||||
Min connection number |
||||
</description> |
||||
<value-attributes> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>spring.datasource.maxActive</name> |
||||
<value>50</value> |
||||
<description> |
||||
Max connection number |
||||
</description> |
||||
<value-attributes> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>spring.datasource.maxWait</name> |
||||
<value>60000</value> |
||||
<description> |
||||
Max wait time for get a connection in milliseconds. |
||||
If configuring maxWait, fair locks are enabled by default and concurrency efficiency decreases. |
||||
If necessary, unfair locks can be used by configuring the useUnfairLock attribute to true. |
||||
</description> |
||||
<value-attributes> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>spring.datasource.timeBetweenEvictionRunsMillis</name> |
||||
<value>60000</value> |
||||
<description> |
||||
Milliseconds for check to close free connections |
||||
</description> |
||||
<value-attributes> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>spring.datasource.timeBetweenConnectErrorMillis</name> |
||||
<value>60000</value> |
||||
<description> |
||||
The Destroy thread detects the connection interval and closes the physical connection in milliseconds |
||||
if the connection idle time is greater than or equal to minEvictableIdleTimeMillis. |
||||
</description> |
||||
<value-attributes> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>spring.datasource.minEvictableIdleTimeMillis</name> |
||||
<value>300000</value> |
||||
<description> |
||||
The longest time a connection remains idle without being evicted, in milliseconds |
||||
</description> |
||||
<value-attributes> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>spring.datasource.validationQuery</name> |
||||
<value>SELECT 1</value> |
||||
<description> |
||||
The SQL used to check whether the connection is valid requires a query statement. |
||||
If validation Query is null, testOnBorrow, testOnReturn, and testWhileIdle will not work. |
||||
</description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>spring.datasource.validationQueryTimeout</name> |
||||
<value>3</value> |
||||
<value-attributes> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
<description> |
||||
Check whether the connection is valid for timeout, in seconds |
||||
</description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>spring.datasource.testWhileIdle</name> |
||||
<value>true</value> |
||||
<value-attributes> |
||||
<type>boolean</type> |
||||
</value-attributes> |
||||
<description> |
||||
When applying for a connection, |
||||
if it is detected that the connection is idle longer than time Between Eviction Runs Millis, |
||||
validation Query is performed to check whether the connection is valid |
||||
</description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>spring.datasource.testOnBorrow</name> |
||||
<value>true</value> |
||||
<value-attributes> |
||||
<type>boolean</type> |
||||
</value-attributes> |
||||
<description> |
||||
Execute validation to check if the connection is valid when applying for a connection |
||||
</description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>spring.datasource.testOnReturn</name> |
||||
<value>false</value> |
||||
<value-attributes> |
||||
<type>boolean</type> |
||||
</value-attributes> |
||||
<description> |
||||
Execute validation to check if the connection is valid when the connection is returned |
||||
</description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>spring.datasource.defaultAutoCommit</name> |
||||
<value>true</value> |
||||
<value-attributes> |
||||
<type>boolean</type> |
||||
</value-attributes> |
||||
<description> |
||||
</description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>spring.datasource.keepAlive</name> |
||||
<value>false</value> |
||||
<value-attributes> |
||||
<type>boolean</type> |
||||
</value-attributes> |
||||
<description> |
||||
</description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
|
||||
<property> |
||||
<name>spring.datasource.poolPreparedStatements</name> |
||||
<value>true</value> |
||||
<value-attributes> |
||||
<type>boolean</type> |
||||
</value-attributes> |
||||
<description> |
||||
Open PSCache, specify count PSCache for every connection |
||||
</description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>spring.datasource.maxPoolPreparedStatementPerConnectionSize</name> |
||||
<value>20</value> |
||||
<value-attributes> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
<description></description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>spring.datasource.spring.datasource.filters</name> |
||||
<value>stat,wall,log4j</value> |
||||
<description></description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>spring.datasource.connectionProperties</name> |
||||
<value>druid.stat.mergeSql=true;druid.stat.slowSqlMillis=5000</value> |
||||
<description></description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
|
||||
<property> |
||||
<name>mybatis-plus.mapper-locations</name> |
||||
<value>classpath*:/org.apache.dolphinscheduler.dao.mapper/*.xml</value> |
||||
<description></description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>mybatis-plus.typeEnumsPackage</name> |
||||
<value>org.apache.dolphinscheduler.*.enums</value> |
||||
<description></description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>mybatis-plus.typeAliasesPackage</name> |
||||
<value>org.apache.dolphinscheduler.dao.entity</value> |
||||
<description> |
||||
Entity scan, where multiple packages are separated by a comma or semicolon |
||||
</description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>mybatis-plus.global-config.db-config.id-type</name> |
||||
<value>AUTO</value> |
||||
<value-attributes> |
||||
<type>value-list</type> |
||||
<entries> |
||||
<entry> |
||||
<value>AUTO</value> |
||||
<label>AUTO</label> |
||||
</entry> |
||||
<entry> |
||||
<value>INPUT</value> |
||||
<label>INPUT</label> |
||||
</entry> |
||||
<entry> |
||||
<value>ID_WORKER</value> |
||||
<label>ID_WORKER</label> |
||||
</entry> |
||||
<entry> |
||||
<value>UUID</value> |
||||
<label>UUID</label> |
||||
</entry> |
||||
</entries> |
||||
<selection-cardinality>1</selection-cardinality> |
||||
</value-attributes> |
||||
<description> |
||||
Primary key type AUTO:" database ID AUTO ", |
||||
INPUT:" user INPUT ID", |
||||
ID_WORKER:" global unique ID (numeric type unique ID)", |
||||
UUID:" global unique ID UUID"; |
||||
</description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>mybatis-plus.global-config.db-config.field-strategy</name> |
||||
<value>NOT_NULL</value> |
||||
<value-attributes> |
||||
<type>value-list</type> |
||||
<entries> |
||||
<entry> |
||||
<value>IGNORED</value> |
||||
<label>IGNORED</label> |
||||
</entry> |
||||
<entry> |
||||
<value>NOT_NULL</value> |
||||
<label>NOT_NULL</label> |
||||
</entry> |
||||
<entry> |
||||
<value>NOT_EMPTY</value> |
||||
<label>NOT_EMPTY</label> |
||||
</entry> |
||||
</entries> |
||||
<selection-cardinality>1</selection-cardinality> |
||||
</value-attributes> |
||||
<description> |
||||
Field policy IGNORED:" ignore judgment ", |
||||
NOT_NULL:" not NULL judgment "), |
||||
NOT_EMPTY:" not NULL judgment" |
||||
</description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>mybatis-plus.global-config.db-config.column-underline</name> |
||||
<value>true</value> |
||||
<value-attributes> |
||||
<type>boolean</type> |
||||
</value-attributes> |
||||
<description></description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>mybatis-plus.global-config.db-config.logic-delete-value</name> |
||||
<value>1</value> |
||||
<value-attributes> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
<description></description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>mybatis-plus.global-config.db-config.logic-not-delete-value</name> |
||||
<value>0</value> |
||||
<value-attributes> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
<description></description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>mybatis-plus.global-config.db-config.banner</name> |
||||
<value>true</value> |
||||
<value-attributes> |
||||
<type>boolean</type> |
||||
</value-attributes> |
||||
<description></description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
|
||||
<property> |
||||
<name>mybatis-plus.configuration.map-underscore-to-camel-case</name> |
||||
<value>true</value> |
||||
<value-attributes> |
||||
<type>boolean</type> |
||||
</value-attributes> |
||||
<description></description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>mybatis-plus.configuration.cache-enabled</name> |
||||
<value>false</value> |
||||
<value-attributes> |
||||
<type>boolean</type> |
||||
</value-attributes> |
||||
<description></description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>mybatis-plus.configuration.call-setters-on-nulls</name> |
||||
<value>true</value> |
||||
<value-attributes> |
||||
<type>boolean</type> |
||||
</value-attributes> |
||||
<description></description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>mybatis-plus.configuration.jdbc-type-for-null</name> |
||||
<value>null</value> |
||||
<description></description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>master.exec.threads</name> |
||||
<value>100</value> |
||||
<value-attributes> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
<description></description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>master.exec.task.num</name> |
||||
<value>20</value> |
||||
<value-attributes> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
<description></description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>master.heartbeat.interval</name> |
||||
<value>10</value> |
||||
<value-attributes> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
<description></description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>master.task.commit.retryTimes</name> |
||||
<value>5</value> |
||||
<value-attributes> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
<description></description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>master.task.commit.interval</name> |
||||
<value>1000</value> |
||||
<value-attributes> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
<description></description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>master.max.cpuload.avg</name> |
||||
<value>100</value> |
||||
<value-attributes> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
<description></description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>master.reserved.memory</name> |
||||
<value>0.1</value> |
||||
<value-attributes> |
||||
<type>float</type> |
||||
</value-attributes> |
||||
<description></description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>worker.exec.threads</name> |
||||
<value>100</value> |
||||
<value-attributes> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
<description></description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>worker.heartbeat.interval</name> |
||||
<value>10</value> |
||||
<value-attributes> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
<description></description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>worker.fetch.task.num</name> |
||||
<value>3</value> |
||||
<value-attributes> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
<description></description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>worker.max.cpuload.avg</name> |
||||
<value>100</value> |
||||
<value-attributes> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
<description></description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>worker.reserved.memory</name> |
||||
<value>0.1</value> |
||||
<value-attributes> |
||||
<type>float</type> |
||||
</value-attributes> |
||||
<description></description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
|
||||
</configuration> |
||||
@ -1,123 +0,0 @@
|
||||
<!-- |
||||
~ Licensed to the Apache Software Foundation (ASF) under one or more |
||||
~ contributor license agreements. See the NOTICE file distributed with |
||||
~ this work for additional information regarding copyright ownership. |
||||
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
~ (the "License"); you may not use this file except in compliance with |
||||
~ the License. You may obtain a copy of the License at |
||||
~ |
||||
~ http://www.apache.org/licenses/LICENSE-2.0 |
||||
~ |
||||
~ Unless required by applicable law or agreed to in writing, software |
||||
~ distributed under the License is distributed on an "AS IS" BASIS, |
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
~ See the License for the specific language governing permissions and |
||||
~ limitations under the License. |
||||
--> |
||||
<configuration> |
||||
<property> |
||||
<name>dolphin.database.type</name> |
||||
<value>mysql</value> |
||||
<description>Dolphin Scheduler DataBase Type Which Is Select</description> |
||||
<display-name>Dolphin Database Type</display-name> |
||||
<value-attributes> |
||||
<type>value-list</type> |
||||
<entries> |
||||
<entry> |
||||
<value>mysql</value> |
||||
<label>Mysql</label> |
||||
</entry> |
||||
<entry> |
||||
<value>postgresql</value> |
||||
<label>Postgresql</label> |
||||
</entry> |
||||
</entries> |
||||
<selection-cardinality>1</selection-cardinality> |
||||
</value-attributes> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
|
||||
<property> |
||||
<name>dolphin.database.host</name> |
||||
<value></value> |
||||
<display-name>Dolphin Database Host</display-name> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
|
||||
<property> |
||||
<name>dolphin.database.port</name> |
||||
<value></value> |
||||
<display-name>Dolphin Database Port</display-name> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
|
||||
<property> |
||||
<name>dolphin.database.username</name> |
||||
<value></value> |
||||
<display-name>Dolphin Database Username</display-name> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
|
||||
<property> |
||||
<name>dolphin.database.password</name> |
||||
<value></value> |
||||
<display-name>Dolphin Database Password</display-name> |
||||
<property-type>PASSWORD</property-type> |
||||
<value-attributes> |
||||
<type>password</type> |
||||
</value-attributes> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
|
||||
<property> |
||||
<name>dolphin.user</name> |
||||
<value></value> |
||||
<description>Which user to install and admin dolphin scheduler</description> |
||||
<display-name>Deploy User</display-name> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>dolphin.group</name> |
||||
<value></value> |
||||
<description>Which user to install and admin dolphin scheduler</description> |
||||
<display-name>Deploy Group</display-name> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
|
||||
<property> |
||||
<name>dolphinscheduler-env-content</name> |
||||
<display-name>Dolphinscheduler Env template</display-name> |
||||
<description>This is the jinja template for dolphinscheduler.env.sh file</description> |
||||
<value># |
||||
# Licensed to the Apache Software Foundation (ASF) under one or more |
||||
# contributor license agreements. See the NOTICE file distributed with |
||||
# this work for additional information regarding copyright ownership. |
||||
# The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
# (the "License"); you may not use this file except in compliance with |
||||
# the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
# |
||||
|
||||
export HADOOP_HOME=/opt/soft/hadoop |
||||
export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop |
||||
export SPARK_HOME1=/opt/soft/spark1 |
||||
export SPARK_HOME2=/opt/soft/spark2 |
||||
export PYTHON_HOME=/opt/soft/python |
||||
export JAVA_HOME=/opt/soft/java |
||||
export HIVE_HOME=/opt/soft/hive |
||||
export FLINK_HOME=/opt/soft/flink</value> |
||||
<value-attributes> |
||||
<type>content</type> |
||||
<empty-value-valid>false</empty-value-valid> |
||||
<show-property-name>false</show-property-name> |
||||
</value-attributes> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
</configuration> |
||||
@ -1,88 +0,0 @@
|
||||
<!-- |
||||
~ Licensed to the Apache Software Foundation (ASF) under one or more |
||||
~ contributor license agreements. See the NOTICE file distributed with |
||||
~ this work for additional information regarding copyright ownership. |
||||
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
~ (the "License"); you may not use this file except in compliance with |
||||
~ the License. You may obtain a copy of the License at |
||||
~ |
||||
~ http://www.apache.org/licenses/LICENSE-2.0 |
||||
~ |
||||
~ Unless required by applicable law or agreed to in writing, software |
||||
~ distributed under the License is distributed on an "AS IS" BASIS, |
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
~ See the License for the specific language governing permissions and |
||||
~ limitations under the License. |
||||
--> |
||||
<configuration> |
||||
<property> |
||||
<name>master.exec.threads</name> |
||||
<value>100</value> |
||||
<value-attributes> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
<description>master execute thread num</description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>master.exec.task.num</name> |
||||
<value>20</value> |
||||
<value-attributes> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
<description>master execute task number in parallel</description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>master.heartbeat.interval</name> |
||||
<value>10</value> |
||||
<value-attributes> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
<description>master heartbeat interval</description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>master.task.commit.retryTimes</name> |
||||
<value>5</value> |
||||
<value-attributes> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
<description>master commit task retry times</description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>master.task.commit.interval</name> |
||||
<value>1000</value> |
||||
<value-attributes> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
<description>master commit task interval</description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>master.max.cpuload.avg</name> |
||||
<value>100</value> |
||||
<value-attributes> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
<description>only less than cpu avg load, master server can work. default value : the number of cpu cores * 2</description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>master.reserved.memory</name> |
||||
<value>0.3</value> |
||||
<description>only larger than reserved memory, master server can work. default value : physical memory * 1/10, unit is G.</description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
|
||||
<property> |
||||
<name>master.listen.port</name> |
||||
<value>5678</value> |
||||
<value-attributes> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
<description>master listen port</description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
</configuration> |
||||
@ -1,126 +0,0 @@
|
||||
<!-- |
||||
~ Licensed to the Apache Software Foundation (ASF) under one or more |
||||
~ contributor license agreements. See the NOTICE file distributed with |
||||
~ this work for additional information regarding copyright ownership. |
||||
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
~ (the "License"); you may not use this file except in compliance with |
||||
~ the License. You may obtain a copy of the License at |
||||
~ |
||||
~ http://www.apache.org/licenses/LICENSE-2.0 |
||||
~ |
||||
~ Unless required by applicable law or agreed to in writing, software |
||||
~ distributed under the License is distributed on an "AS IS" BASIS, |
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
~ See the License for the specific language governing permissions and |
||||
~ limitations under the License. |
||||
--> |
||||
<configuration> |
||||
<property> |
||||
<name>org.quartz.scheduler.instanceName</name> |
||||
<value>DolphinScheduler</value> |
||||
<description></description> |
||||
</property> |
||||
<property> |
||||
<!-- 列举枚举值 --> |
||||
<name>org.quartz.scheduler.instanceId</name> |
||||
<value>AUTO</value> |
||||
<description></description> |
||||
</property> |
||||
<property> |
||||
<name>org.quartz.scheduler.makeSchedulerThreadDaemon</name> |
||||
<value>true</value> |
||||
<value-attributes> |
||||
<type>boolean</type> |
||||
</value-attributes> |
||||
<description></description> |
||||
</property> |
||||
<property> |
||||
<name>org.quartz.jobStore.useProperties</name> |
||||
<value>false</value> |
||||
<value-attributes> |
||||
<type>boolean</type> |
||||
</value-attributes> |
||||
<description></description> |
||||
</property> |
||||
<property> |
||||
<name>org.quartz.threadPool.class</name> |
||||
<value>org.quartz.simpl.SimpleThreadPool</value> |
||||
<description></description> |
||||
</property> |
||||
<property> |
||||
<name>org.quartz.threadPool.makeThreadsDaemons</name> |
||||
<value>true</value> |
||||
<value-attributes> |
||||
<type>boolean</type> |
||||
</value-attributes> |
||||
<description></description> |
||||
</property> |
||||
<property> |
||||
<name>org.quartz.threadPool.threadCount</name> |
||||
<value>25</value> |
||||
<value-attributes> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
<description></description> |
||||
</property> |
||||
<property> |
||||
<name>org.quartz.threadPool.threadPriority</name> |
||||
<value>5</value> |
||||
<value-attributes> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
<description></description> |
||||
</property> |
||||
<property> |
||||
<name>org.quartz.jobStore.class</name> |
||||
<value>org.quartz.impl.jdbcjobstore.JobStoreTX</value> |
||||
<description></description> |
||||
</property> |
||||
<property> |
||||
<name>org.quartz.jobStore.tablePrefix</name> |
||||
<value>QRTZ_</value> |
||||
<description></description> |
||||
</property> |
||||
<property> |
||||
<name>org.quartz.jobStore.isClustered</name> |
||||
<value>true</value> |
||||
<value-attributes> |
||||
<type>boolean</type> |
||||
</value-attributes> |
||||
<description></description> |
||||
</property> |
||||
<property> |
||||
<name>org.quartz.jobStore.misfireThreshold</name> |
||||
<value>60000</value> |
||||
<value-attributes> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
<description></description> |
||||
</property> |
||||
<property> |
||||
<name>org.quartz.jobStore.clusterCheckinInterval</name> |
||||
<value>5000</value> |
||||
<value-attributes> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
<description></description> |
||||
</property> |
||||
<property> |
||||
<name>org.quartz.jobStore.acquireTriggersWithinLock</name> |
||||
<value>true</value> |
||||
<value-attributes> |
||||
<type>boolean</type> |
||||
</value-attributes> |
||||
<description></description> |
||||
</property> |
||||
<property> |
||||
<name>org.quartz.jobStore.dataSource</name> |
||||
<value>myDs</value> |
||||
<description></description> |
||||
</property> |
||||
<property> |
||||
<name>org.quartz.dataSource.myDs.connectionProvider.class</name> |
||||
<value>org.apache.dolphinscheduler.service.quartz.DruidConnectionProvider</value> |
||||
<description></description> |
||||
</property> |
||||
</configuration> |
||||
@ -1,76 +0,0 @@
|
||||
<!-- |
||||
~ Licensed to the Apache Software Foundation (ASF) under one or more |
||||
~ contributor license agreements. See the NOTICE file distributed with |
||||
~ this work for additional information regarding copyright ownership. |
||||
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
~ (the "License"); you may not use this file except in compliance with |
||||
~ the License. You may obtain a copy of the License at |
||||
~ |
||||
~ http://www.apache.org/licenses/LICENSE-2.0 |
||||
~ |
||||
~ Unless required by applicable law or agreed to in writing, software |
||||
~ distributed under the License is distributed on an "AS IS" BASIS, |
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
~ See the License for the specific language governing permissions and |
||||
~ limitations under the License. |
||||
--> |
||||
<configuration> |
||||
<property> |
||||
<name>worker.exec.threads</name> |
||||
<value>100</value> |
||||
<value-attributes> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
<description>worker execute thread num</description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>worker.heartbeat.interval</name> |
||||
<value>10</value> |
||||
<value-attributes> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
<description>worker heartbeat interval</description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>worker.fetch.task.num</name> |
||||
<value>3</value> |
||||
<value-attributes> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
<description>submit the number of tasks at a time</description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>worker.max.cpuload.avg</name> |
||||
<value>100</value> |
||||
<value-attributes> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
<description>only less than cpu avg load, worker server can work. default value : the number of cpu cores * 2</description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>worker.reserved.memory</name> |
||||
<value>0.3</value> |
||||
<description>only larger than reserved memory, worker server can work. default value : physical memory * 1/10, unit is G.</description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
|
||||
<property> |
||||
<name>worker.listen.port</name> |
||||
<value>1234</value> |
||||
<value-attributes> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
<description>worker listen port</description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>worker.group</name> |
||||
<value>default</value> |
||||
<description>default worker group</description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
</configuration> |
||||
@ -1,84 +0,0 @@
|
||||
<!-- |
||||
~ Licensed to the Apache Software Foundation (ASF) under one or more |
||||
~ contributor license agreements. See the NOTICE file distributed with |
||||
~ this work for additional information regarding copyright ownership. |
||||
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
~ (the "License"); you may not use this file except in compliance with |
||||
~ the License. You may obtain a copy of the License at |
||||
~ |
||||
~ http://www.apache.org/licenses/LICENSE-2.0 |
||||
~ |
||||
~ Unless required by applicable law or agreed to in writing, software |
||||
~ distributed under the License is distributed on an "AS IS" BASIS, |
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
~ See the License for the specific language governing permissions and |
||||
~ limitations under the License. |
||||
--> |
||||
<configuration> |
||||
<property> |
||||
<name>dolphinscheduler.queue.impl</name> |
||||
<value>zookeeper</value> |
||||
<description> |
||||
Task queue implementation, default "zookeeper" |
||||
</description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>zookeeper.dolphinscheduler.root</name> |
||||
<value>/dolphinscheduler</value> |
||||
<description> |
||||
dolphinscheduler root directory |
||||
</description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>zookeeper.session.timeout</name> |
||||
<value>300</value> |
||||
<value-attributes> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
<description> |
||||
</description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>zookeeper.connection.timeout</name> |
||||
<value>300</value> |
||||
<value-attributes> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
<description> |
||||
</description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>zookeeper.retry.base.sleep</name> |
||||
<value>100</value> |
||||
<value-attributes> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
<description> |
||||
</description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>zookeeper.retry.max.sleep</name> |
||||
<value>30000</value> |
||||
<value-attributes> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
<description> |
||||
</description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
<property> |
||||
<name>zookeeper.retry.maxtime</name> |
||||
<value>5</value> |
||||
<value-attributes> |
||||
<type>int</type> |
||||
</value-attributes> |
||||
<description> |
||||
</description> |
||||
<on-ambari-upgrade add="true"/> |
||||
</property> |
||||
</configuration> |
||||
@ -1,137 +0,0 @@
|
||||
<?xml version="1.0"?> |
||||
<!-- |
||||
Licensed to the Apache Software Foundation (ASF) under one or more |
||||
contributor license agreements. See the NOTICE file distributed with |
||||
this work for additional information regarding copyright ownership. |
||||
The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
(the "License"); you may not use this file except in compliance with |
||||
the License. You may obtain a copy of the License at |
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
Unless required by applicable law or agreed to in writing, software |
||||
distributed under the License is distributed on an "AS IS" BASIS, |
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
See the License for the specific language governing permissions and |
||||
limitations under the License. |
||||
--> |
||||
<metainfo> |
||||
<schemaVersion>2.0</schemaVersion> |
||||
<services> |
||||
<service> |
||||
<name>DOLPHIN</name> |
||||
<displayName>Dolphin Scheduler</displayName> |
||||
<comment>分布式易扩展的可视化DAG工作流任务调度系统</comment> |
||||
<version>2.0.0</version> |
||||
<components> |
||||
<component> |
||||
<name>DOLPHIN_MASTER</name> |
||||
<displayName>DS Master</displayName> |
||||
<category>MASTER</category> |
||||
<cardinality>1+</cardinality> |
||||
<commandScript> |
||||
<script>scripts/dolphin_master_service.py</script> |
||||
<scriptType>PYTHON</scriptType> |
||||
<timeout>600</timeout> |
||||
</commandScript> |
||||
</component> |
||||
|
||||
<component> |
||||
<name>DOLPHIN_LOGGER</name> |
||||
<displayName>DS Logger</displayName> |
||||
<category>SLAVE</category> |
||||
<cardinality>1+</cardinality> |
||||
<commandScript> |
||||
<script>scripts/dolphin_logger_service.py</script> |
||||
<scriptType>PYTHON</scriptType> |
||||
<timeout>600</timeout> |
||||
</commandScript> |
||||
</component> |
||||
|
||||
<component> |
||||
<name>DOLPHIN_WORKER</name> |
||||
<displayName>DS Worker</displayName> |
||||
<category>SLAVE</category> |
||||
<cardinality>1+</cardinality> |
||||
<dependencies> |
||||
<dependency> |
||||
<name>DOLPHIN/DOLPHIN_LOGGER</name> |
||||
<scope>host</scope> |
||||
<auto-deploy> |
||||
<enabled>true</enabled> |
||||
</auto-deploy> |
||||
</dependency> |
||||
</dependencies> |
||||
<commandScript> |
||||
<script>scripts/dolphin_worker_service.py</script> |
||||
<scriptType>PYTHON</scriptType> |
||||
<timeout>600</timeout> |
||||
</commandScript> |
||||
</component> |
||||
|
||||
<component> |
||||
<name>DOLPHIN_ALERT</name> |
||||
<displayName>DS Alert</displayName> |
||||
<category>SLAVE</category> |
||||
<cardinality>1</cardinality> |
||||
<commandScript> |
||||
<script>scripts/dolphin_alert_service.py</script> |
||||
<scriptType>PYTHON</scriptType> |
||||
<timeout>600</timeout> |
||||
</commandScript> |
||||
</component> |
||||
|
||||
<component> |
||||
<name>DOLPHIN_API</name> |
||||
<displayName>DS_Api</displayName> |
||||
<category>SLAVE</category> |
||||
<cardinality>1</cardinality> |
||||
<commandScript> |
||||
<script>scripts/dolphin_api_service.py</script> |
||||
<scriptType>PYTHON</scriptType> |
||||
<timeout>600</timeout> |
||||
</commandScript> |
||||
</component> |
||||
</components> |
||||
|
||||
<requiredServices> |
||||
<service>ZOOKEEPER</service> |
||||
</requiredServices> |
||||
|
||||
<osSpecifics> |
||||
<osSpecific> |
||||
<osFamily>any</osFamily> |
||||
<packages> |
||||
<package> |
||||
<name>apache-dolphinscheduler-incubating-1.2.1*</name> |
||||
</package> |
||||
</packages> |
||||
</osSpecific> |
||||
</osSpecifics> |
||||
|
||||
<configuration-dependencies> |
||||
<config-type>dolphin-alert</config-type> |
||||
<config-type>dolphin-app-api</config-type> |
||||
<config-type>dolphin-app-dao</config-type> |
||||
<config-type>dolphin-common</config-type> |
||||
<config-type>dolphin-env</config-type> |
||||
<config-type>dolphin-quartz</config-type> |
||||
</configuration-dependencies> |
||||
|
||||
<themes> |
||||
<theme> |
||||
<fileName>theme.json</fileName> |
||||
<default>true</default> |
||||
</theme> |
||||
</themes> |
||||
|
||||
<quickLinksConfigurations-dir>quicklinks</quickLinksConfigurations-dir> |
||||
<quickLinksConfigurations> |
||||
<quickLinksConfiguration> |
||||
<fileName>quicklinks.json</fileName> |
||||
<default>true</default> |
||||
</quickLinksConfiguration> |
||||
</quickLinksConfigurations> |
||||
</service> |
||||
</services> |
||||
</metainfo> |
||||
@ -1,124 +0,0 @@
|
||||
""" |
||||
Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
Unless required by applicable law or agreed to in writing, software |
||||
distributed under the License is distributed on an "AS IS" BASIS, |
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
See the License for the specific language governing permissions and |
||||
limitations under the License. |
||||
""" |
||||
|
||||
import socket |
||||
import urllib2 |
||||
import os |
||||
import logging |
||||
import ambari_simplejson as json |
||||
from resource_management.libraries.script.script import Script |
||||
import sys |
||||
reload(sys) |
||||
sys.setdefaultencoding('utf-8') |
||||
|
||||
logger = logging.getLogger('ambari_alerts') |
||||
|
||||
config = Script.get_config() |
||||
|
||||
|
||||
def get_tokens(): |
||||
""" |
||||
Returns a tuple of tokens in the format {{site/property}} that will be used |
||||
to build the dictionary passed into execute |
||||
|
||||
:rtype tuple |
||||
""" |
||||
|
||||
def get_info(url, connection_timeout): |
||||
response = None |
||||
|
||||
try: |
||||
response = urllib2.urlopen(url, timeout=connection_timeout) |
||||
json_data = response.read() |
||||
return json_data |
||||
finally: |
||||
if response is not None: |
||||
try: |
||||
response.close() |
||||
except: |
||||
pass |
||||
|
||||
|
||||
def execute(configurations={}, parameters={}, host_name=None): |
||||
""" |
||||
Returns a tuple containing the result code and a pre-formatted result label |
||||
|
||||
Keyword arguments: |
||||
configurations : a mapping of configuration key to value |
||||
parameters : a mapping of script parameter key to value |
||||
host_name : the name of this host where the alert is running |
||||
|
||||
:type configurations dict |
||||
:type parameters dict |
||||
:type host_name str |
||||
""" |
||||
|
||||
alert_name = parameters['alertName'] |
||||
|
||||
dolphin_pidfile_dir = "/opt/soft/run/dolphinscheduler" |
||||
|
||||
pid = "0" |
||||
|
||||
|
||||
from resource_management.core import sudo |
||||
|
||||
is_running = True |
||||
pid_file_path = "" |
||||
if alert_name == 'DOLPHIN_MASTER': |
||||
pid_file_path = dolphin_pidfile_dir + "/master-server.pid" |
||||
elif alert_name == 'DOLPHIN_WORKER': |
||||
pid_file_path = dolphin_pidfile_dir + "/worker-server.pid" |
||||
elif alert_name == 'DOLPHIN_ALERT': |
||||
pid_file_path = dolphin_pidfile_dir + "/alert-server.pid" |
||||
elif alert_name == 'DOLPHIN_LOGGER': |
||||
pid_file_path = dolphin_pidfile_dir + "/logger-server.pid" |
||||
elif alert_name == 'DOLPHIN_API': |
||||
pid_file_path = dolphin_pidfile_dir + "/api-server.pid" |
||||
|
||||
if not pid_file_path or not os.path.isfile(pid_file_path): |
||||
is_running = False |
||||
|
||||
try: |
||||
pid = int(sudo.read_file(pid_file_path)) |
||||
except: |
||||
is_running = False |
||||
|
||||
try: |
||||
# Kill will not actually kill the process |
||||
# From the doc: |
||||
# If sig is 0, then no signal is sent, but error checking is still |
||||
# performed; this can be used to check for the existence of a |
||||
# process ID or process group ID. |
||||
sudo.kill(pid, 0) |
||||
except OSError: |
||||
is_running = False |
||||
|
||||
if host_name is None: |
||||
host_name = socket.getfqdn() |
||||
|
||||
if not is_running: |
||||
result_code = "CRITICAL" |
||||
else: |
||||
result_code = "OK" |
||||
|
||||
label = "The comment {0} of DOLPHIN_SCHEDULER on {1} is {2}".format(alert_name, host_name, result_code) |
||||
|
||||
return ((result_code, [label])) |
||||
|
||||
if __name__ == "__main__": |
||||
pass |
||||
@ -1,61 +0,0 @@
|
||||
""" |
||||
Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
Unless required by applicable law or agreed to in writing, software |
||||
distributed under the License is distributed on an "AS IS" BASIS, |
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
See the License for the specific language governing permissions and |
||||
limitations under the License. |
||||
""" |
||||
import time |
||||
from resource_management import * |
||||
|
||||
from dolphin_env import dolphin_env |
||||
|
||||
|
||||
class DolphinAlertService(Script): |
||||
def install(self, env): |
||||
import params |
||||
env.set_params(params) |
||||
self.install_packages(env) |
||||
Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True) |
||||
|
||||
def configure(self, env): |
||||
import params |
||||
params.pika_slave = True |
||||
env.set_params(params) |
||||
|
||||
dolphin_env() |
||||
|
||||
def start(self, env): |
||||
import params |
||||
env.set_params(params) |
||||
self.configure(env) |
||||
no_op_test = format("ls {dolphin_pidfile_dir}/alert-server.pid >/dev/null 2>&1 && ps `cat {dolphin_pidfile_dir}/alert-server.pid` | grep `cat {dolphin_pidfile_dir}/alert-server.pid` >/dev/null 2>&1") |
||||
|
||||
start_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh start alert-server") |
||||
Execute(start_cmd, user=params.dolphin_user, not_if=no_op_test) |
||||
|
||||
def stop(self, env): |
||||
import params |
||||
env.set_params(params) |
||||
stop_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh stop alert-server") |
||||
Execute(stop_cmd, user=params.dolphin_user) |
||||
time.sleep(5) |
||||
|
||||
def status(self, env): |
||||
import status_params |
||||
env.set_params(status_params) |
||||
check_process_status(status_params.dolphin_run_dir + "alert-server.pid") |
||||
|
||||
|
||||
if __name__ == "__main__": |
||||
DolphinAlertService().execute() |
||||
@ -1,70 +0,0 @@
|
||||
""" |
||||
Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
Unless required by applicable law or agreed to in writing, software |
||||
distributed under the License is distributed on an "AS IS" BASIS, |
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
See the License for the specific language governing permissions and |
||||
limitations under the License. |
||||
""" |
||||
import time |
||||
from resource_management import * |
||||
|
||||
from dolphin_env import dolphin_env |
||||
|
||||
|
||||
class DolphinApiService(Script): |
||||
def install(self, env): |
||||
import params |
||||
env.set_params(params) |
||||
self.install_packages(env) |
||||
Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True) |
||||
|
||||
def configure(self, env): |
||||
import params |
||||
params.pika_slave = True |
||||
env.set_params(params) |
||||
|
||||
dolphin_env() |
||||
|
||||
def start(self, env): |
||||
import params |
||||
env.set_params(params) |
||||
self.configure(env) |
||||
|
||||
#init |
||||
init_cmd=format("sh " + params.dolphin_home + "/script/create-dolphinscheduler.sh") |
||||
Execute(init_cmd, user=params.dolphin_user) |
||||
|
||||
#upgrade |
||||
upgrade_cmd=format("sh " + params.dolphin_home + "/script/upgrade-dolphinscheduler.sh") |
||||
Execute(upgrade_cmd, user=params.dolphin_user) |
||||
|
||||
no_op_test = format("ls {dolphin_pidfile_dir}/api-server.pid >/dev/null 2>&1 && ps `cat {dolphin_pidfile_dir}/api-server.pid` | grep `cat {dolphin_pidfile_dir}/api-server.pid` >/dev/null 2>&1") |
||||
|
||||
start_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh start api-server") |
||||
Execute(start_cmd, user=params.dolphin_user, not_if=no_op_test) |
||||
|
||||
def stop(self, env): |
||||
import params |
||||
env.set_params(params) |
||||
stop_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh stop api-server") |
||||
Execute(stop_cmd, user=params.dolphin_user) |
||||
time.sleep(5) |
||||
|
||||
def status(self, env): |
||||
import status_params |
||||
env.set_params(status_params) |
||||
check_process_status(status_params.dolphin_run_dir + "api-server.pid") |
||||
|
||||
|
||||
if __name__ == "__main__": |
||||
DolphinApiService().execute() |
||||
@ -1,123 +0,0 @@
|
||||
""" |
||||
Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
Unless required by applicable law or agreed to in writing, software |
||||
distributed under the License is distributed on an "AS IS" BASIS, |
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
See the License for the specific language governing permissions and |
||||
limitations under the License. |
||||
|
||||
""" |
||||
|
||||
from resource_management import * |
||||
|
||||
|
||||
def dolphin_env(): |
||||
import params |
||||
|
||||
Directory(params.dolphin_pidfile_dir, |
||||
mode=0777, |
||||
owner=params.dolphin_user, |
||||
group=params.dolphin_group, |
||||
create_parents=True |
||||
) |
||||
Directory(params.dolphin_log_dir, |
||||
mode=0777, |
||||
owner=params.dolphin_user, |
||||
group=params.dolphin_group, |
||||
create_parents=True |
||||
) |
||||
Directory(params.dolphin_conf_dir, |
||||
mode=0777, |
||||
owner=params.dolphin_user, |
||||
group=params.dolphin_group, |
||||
create_parents=True |
||||
) |
||||
|
||||
Directory(params.dolphin_common_map['data.basedir.path'], |
||||
mode=0777, |
||||
owner=params.dolphin_user, |
||||
group=params.dolphin_group, |
||||
create_parents=True |
||||
) |
||||
|
||||
|
||||
File(format(params.dolphin_env_path), |
||||
mode=0777, |
||||
content=InlineTemplate(params.dolphin_env_content), |
||||
owner=params.dolphin_user, |
||||
group=params.dolphin_group |
||||
) |
||||
|
||||
|
||||
File(format(params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh"), |
||||
mode=0755, |
||||
content=Template("dolphin-daemon.sh.j2"), |
||||
owner=params.dolphin_user, |
||||
group=params.dolphin_group |
||||
) |
||||
|
||||
File(format(params.dolphin_conf_dir + "/master.properties"), |
||||
mode=0755, |
||||
content=Template("master.properties.j2"), |
||||
owner=params.dolphin_user, |
||||
group=params.dolphin_group |
||||
) |
||||
|
||||
File(format(params.dolphin_conf_dir + "/worker.properties"), |
||||
mode=0755, |
||||
content=Template("worker.properties.j2"), |
||||
owner=params.dolphin_user, |
||||
group=params.dolphin_group |
||||
) |
||||
|
||||
|
||||
File(format(params.dolphin_conf_dir + "/alert.properties"), |
||||
mode=0755, |
||||
content=Template("alert.properties.j2"), |
||||
owner=params.dolphin_user, |
||||
group=params.dolphin_group |
||||
) |
||||
|
||||
File(format(params.dolphin_conf_dir + "/datasource.properties"), |
||||
mode=0755, |
||||
content=Template("datasource.properties.j2"), |
||||
owner=params.dolphin_user, |
||||
group=params.dolphin_group |
||||
) |
||||
|
||||
File(format(params.dolphin_conf_dir + "/application-api.properties"), |
||||
mode=0755, |
||||
content=Template("application-api.properties.j2"), |
||||
owner=params.dolphin_user, |
||||
group=params.dolphin_group |
||||
) |
||||
|
||||
File(format(params.dolphin_conf_dir + "/common.properties"), |
||||
mode=0755, |
||||
content=Template("common.properties.j2"), |
||||
owner=params.dolphin_user, |
||||
group=params.dolphin_group |
||||
) |
||||
|
||||
File(format(params.dolphin_conf_dir + "/quartz.properties"), |
||||
mode=0755, |
||||
content=Template("quartz.properties.j2"), |
||||
owner=params.dolphin_user, |
||||
group=params.dolphin_group |
||||
) |
||||
|
||||
File(format(params.dolphin_conf_dir + "/zookeeper.properties"), |
||||
mode=0755, |
||||
content=Template("zookeeper.properties.j2"), |
||||
owner=params.dolphin_user, |
||||
group=params.dolphin_group |
||||
) |
||||
@ -1,61 +0,0 @@
|
||||
""" |
||||
Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
Unless required by applicable law or agreed to in writing, software |
||||
distributed under the License is distributed on an "AS IS" BASIS, |
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
See the License for the specific language governing permissions and |
||||
limitations under the License. |
||||
""" |
||||
import time |
||||
from resource_management import * |
||||
|
||||
from dolphin_env import dolphin_env |
||||
|
||||
|
||||
class DolphinLoggerService(Script): |
||||
def install(self, env): |
||||
import params |
||||
env.set_params(params) |
||||
self.install_packages(env) |
||||
Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True) |
||||
|
||||
def configure(self, env): |
||||
import params |
||||
params.pika_slave = True |
||||
env.set_params(params) |
||||
|
||||
dolphin_env() |
||||
|
||||
def start(self, env): |
||||
import params |
||||
env.set_params(params) |
||||
self.configure(env) |
||||
no_op_test = format("ls {dolphin_pidfile_dir}/logger-server.pid >/dev/null 2>&1 && ps `cat {dolphin_pidfile_dir}/logger-server.pid` | grep `cat {dolphin_pidfile_dir}/logger-server.pid` >/dev/null 2>&1") |
||||
|
||||
start_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh start logger-server") |
||||
Execute(start_cmd, user=params.dolphin_user, not_if=no_op_test) |
||||
|
||||
def stop(self, env): |
||||
import params |
||||
env.set_params(params) |
||||
stop_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh stop logger-server") |
||||
Execute(stop_cmd, user=params.dolphin_user) |
||||
time.sleep(5) |
||||
|
||||
def status(self, env): |
||||
import status_params |
||||
env.set_params(status_params) |
||||
check_process_status(status_params.dolphin_run_dir + "logger-server.pid") |
||||
|
||||
|
||||
if __name__ == "__main__": |
||||
DolphinLoggerService().execute() |
||||
@ -1,61 +0,0 @@
|
||||
# -*- coding: utf-8 -*- |
||||
""" |
||||
Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
Unless required by applicable law or agreed to in writing, software |
||||
distributed under the License is distributed on an "AS IS" BASIS, |
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
See the License for the specific language governing permissions and |
||||
limitations under the License. |
||||
""" |
||||
import time |
||||
from resource_management import * |
||||
|
||||
from dolphin_env import dolphin_env |
||||
|
||||
|
||||
class DolphinMasterService(Script): |
||||
def install(self, env): |
||||
import params |
||||
env.set_params(params) |
||||
self.install_packages(env) |
||||
Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True) |
||||
|
||||
def configure(self, env): |
||||
import params |
||||
params.pika_slave = True |
||||
env.set_params(params) |
||||
|
||||
dolphin_env() |
||||
|
||||
def start(self, env): |
||||
import params |
||||
env.set_params(params) |
||||
self.configure(env) |
||||
no_op_test = format("ls {dolphin_pidfile_dir}/master-server.pid >/dev/null 2>&1 && ps `cat {dolphin_pidfile_dir}/master-server.pid` | grep `cat {dolphin_pidfile_dir}/master-server.pid` >/dev/null 2>&1") |
||||
start_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh start master-server") |
||||
Execute(start_cmd, user=params.dolphin_user, not_if=no_op_test) |
||||
|
||||
def stop(self, env): |
||||
import params |
||||
env.set_params(params) |
||||
stop_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh stop master-server") |
||||
Execute(stop_cmd, user=params.dolphin_user) |
||||
time.sleep(5) |
||||
|
||||
def status(self, env): |
||||
import status_params |
||||
env.set_params(status_params) |
||||
check_process_status(status_params.dolphin_run_dir + "master-server.pid") |
||||
|
||||
|
||||
if __name__ == "__main__": |
||||
DolphinMasterService().execute() |
||||
@ -1,60 +0,0 @@
|
||||
""" |
||||
Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
Unless required by applicable law or agreed to in writing, software |
||||
distributed under the License is distributed on an "AS IS" BASIS, |
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
See the License for the specific language governing permissions and |
||||
limitations under the License. |
||||
""" |
||||
import time |
||||
from resource_management import * |
||||
|
||||
from dolphin_env import dolphin_env |
||||
|
||||
|
||||
class DolphinWorkerService(Script): |
||||
def install(self, env): |
||||
import params |
||||
env.set_params(params) |
||||
self.install_packages(env) |
||||
Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True) |
||||
|
||||
def configure(self, env): |
||||
import params |
||||
params.pika_slave = True |
||||
env.set_params(params) |
||||
|
||||
dolphin_env() |
||||
|
||||
def start(self, env): |
||||
import params |
||||
env.set_params(params) |
||||
self.configure(env) |
||||
no_op_test = format("ls {dolphin_pidfile_dir}/worker-server.pid >/dev/null 2>&1 && ps `cat {dolphin_pidfile_dir}/worker-server.pid` | grep `cat {dolphin_pidfile_dir}/worker-server.pid` >/dev/null 2>&1") |
||||
start_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh start worker-server") |
||||
Execute(start_cmd, user=params.dolphin_user, not_if=no_op_test) |
||||
|
||||
def stop(self, env): |
||||
import params |
||||
env.set_params(params) |
||||
stop_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh stop worker-server") |
||||
Execute(stop_cmd, user=params.dolphin_user) |
||||
time.sleep(5) |
||||
|
||||
def status(self, env): |
||||
import status_params |
||||
env.set_params(status_params) |
||||
check_process_status(status_params.dolphin_run_dir + "worker-server.pid") |
||||
|
||||
|
||||
if __name__ == "__main__": |
||||
DolphinWorkerService().execute() |
||||
@ -1,154 +0,0 @@
|
||||
""" |
||||
Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
Unless required by applicable law or agreed to in writing, software |
||||
distributed under the License is distributed on an "AS IS" BASIS, |
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
See the License for the specific language governing permissions and |
||||
limitations under the License. |
||||
""" |
||||
|
||||
|
||||
import sys |
||||
from resource_management import * |
||||
from resource_management.core.logger import Logger |
||||
from resource_management.libraries.functions import default |
||||
|
||||
Logger.initialize_logger() |
||||
reload(sys) |
||||
sys.setdefaultencoding('utf-8') |
||||
|
||||
# server configurations |
||||
config = Script.get_config() |
||||
|
||||
# conf_dir = "/etc/" |
||||
dolphin_home = "/opt/soft/dolphinscheduler" |
||||
dolphin_conf_dir = dolphin_home + "/conf" |
||||
dolphin_log_dir = dolphin_home + "/logs" |
||||
dolphin_bin_dir = dolphin_home + "/bin" |
||||
dolphin_lib_jars = dolphin_home + "/lib/*" |
||||
dolphin_pidfile_dir = "/opt/soft/run/dolphinscheduler" |
||||
|
||||
rmHosts = default("/clusterHostInfo/rm_host", []) |
||||
|
||||
# dolphin-env |
||||
dolphin_env_map = {} |
||||
dolphin_env_map.update(config['configurations']['dolphin-env']) |
||||
|
||||
# which user to install and admin dolphin scheduler |
||||
dolphin_user = dolphin_env_map['dolphin.user'] |
||||
dolphin_group = dolphin_env_map['dolphin.group'] |
||||
|
||||
# .dolphinscheduler_env.sh |
||||
dolphin_env_path = dolphin_conf_dir + '/env/dolphinscheduler_env.sh' |
||||
dolphin_env_content = dolphin_env_map['dolphinscheduler-env-content'] |
||||
|
||||
# database config |
||||
dolphin_database_config = {} |
||||
dolphin_database_config['dolphin_database_type'] = dolphin_env_map['dolphin.database.type'] |
||||
dolphin_database_config['dolphin_database_username'] = dolphin_env_map['dolphin.database.username'] |
||||
dolphin_database_config['dolphin_database_password'] = dolphin_env_map['dolphin.database.password'] |
||||
if 'mysql' == dolphin_database_config['dolphin_database_type']: |
||||
dolphin_database_config['dolphin_database_driver'] = 'com.mysql.jdbc.Driver' |
||||
dolphin_database_config['driverDelegateClass'] = 'org.quartz.impl.jdbcjobstore.StdJDBCDelegate' |
||||
dolphin_database_config['dolphin_database_url'] = 'jdbc:mysql://' + dolphin_env_map['dolphin.database.host'] \ |
||||
+ ':' + dolphin_env_map['dolphin.database.port'] \ |
||||
+ '/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8' |
||||
else: |
||||
dolphin_database_config['dolphin_database_driver'] = 'org.postgresql.Driver' |
||||
dolphin_database_config['driverDelegateClass'] = 'org.quartz.impl.jdbcjobstore.PostgreSQLDelegate' |
||||
dolphin_database_config['dolphin_database_url'] = 'jdbc:postgresql://' + dolphin_env_map['dolphin.database.host'] \ |
||||
+ ':' + dolphin_env_map['dolphin.database.port'] \ |
||||
+ '/dolphinscheduler' |
||||
|
||||
|
||||
|
||||
|
||||
|
||||
# application-alert.properties |
||||
dolphin_alert_map = {} |
||||
wechat_push_url = 'https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=$token' |
||||
wechat_token_url = 'https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid=$corpId&corpsecret=$secret' |
||||
wechat_team_send_msg = '{\"toparty\":\"$toParty\",\"agentid\":\"$agentId\",\"msgtype\":\"text\",\"text\":{\"content\":\"$msg\"},\"safe\":\"0\"}' |
||||
wechat_user_send_msg = '{\"touser\":\"$toUser\",\"agentid\":\"$agentId\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"$msg\"}}' |
||||
|
||||
dolphin_alert_config_map = config['configurations']['dolphin-alert'] |
||||
|
||||
if dolphin_alert_config_map['enterprise.wechat.enable']: |
||||
dolphin_alert_map['enterprise.wechat.push.ur'] = wechat_push_url |
||||
dolphin_alert_map['enterprise.wechat.token.url'] = wechat_token_url |
||||
dolphin_alert_map['enterprise.wechat.team.send.msg'] = wechat_team_send_msg |
||||
dolphin_alert_map['enterprise.wechat.user.send.msg'] = wechat_user_send_msg |
||||
|
||||
dolphin_alert_map.update(dolphin_alert_config_map) |
||||
|
||||
|
||||
|
||||
# application-api.properties |
||||
dolphin_app_api_map = {} |
||||
dolphin_app_api_map.update(config['configurations']['dolphin-application-api']) |
||||
|
||||
|
||||
# common.properties |
||||
dolphin_common_map = {} |
||||
|
||||
if 'yarn-site' in config['configurations'] and \ |
||||
'yarn.resourcemanager.webapp.address' in config['configurations']['yarn-site']: |
||||
yarn_resourcemanager_webapp_address = config['configurations']['yarn-site']['yarn.resourcemanager.webapp.address'] |
||||
yarn_application_status_address = 'http://' + yarn_resourcemanager_webapp_address + '/ws/v1/cluster/apps/%s' |
||||
dolphin_common_map['yarn.application.status.address'] = yarn_application_status_address |
||||
|
||||
rmHosts = default("/clusterHostInfo/rm_host", []) |
||||
if len(rmHosts) > 1: |
||||
dolphin_common_map['yarn.resourcemanager.ha.rm.ids'] = ','.join(rmHosts) |
||||
else: |
||||
dolphin_common_map['yarn.resourcemanager.ha.rm.ids'] = '' |
||||
|
||||
dolphin_common_map_tmp = config['configurations']['dolphin-common'] |
||||
data_basedir_path = dolphin_common_map_tmp['data.basedir.path'] |
||||
process_exec_basepath = data_basedir_path + '/exec' |
||||
data_download_basedir_path = data_basedir_path + '/download' |
||||
dolphin_common_map['process.exec.basepath'] = process_exec_basepath |
||||
dolphin_common_map['data.download.basedir.path'] = data_download_basedir_path |
||||
dolphin_common_map['dolphinscheduler.env.path'] = dolphin_env_path |
||||
dolphin_common_map.update(config['configurations']['dolphin-common']) |
||||
|
||||
# datasource.properties |
||||
dolphin_datasource_map = {} |
||||
dolphin_datasource_map['spring.datasource.type'] = 'com.alibaba.druid.pool.DruidDataSource' |
||||
dolphin_datasource_map['spring.datasource.driver-class-name'] = dolphin_database_config['dolphin_database_driver'] |
||||
dolphin_datasource_map['spring.datasource.url'] = dolphin_database_config['dolphin_database_url'] |
||||
dolphin_datasource_map['spring.datasource.username'] = dolphin_database_config['dolphin_database_username'] |
||||
dolphin_datasource_map['spring.datasource.password'] = dolphin_database_config['dolphin_database_password'] |
||||
dolphin_datasource_map.update(config['configurations']['dolphin-datasource']) |
||||
|
||||
# master.properties |
||||
dolphin_master_map = config['configurations']['dolphin-master'] |
||||
|
||||
# quartz.properties |
||||
dolphin_quartz_map = {} |
||||
dolphin_quartz_map['org.quartz.jobStore.driverDelegateClass'] = dolphin_database_config['driverDelegateClass'] |
||||
dolphin_quartz_map.update(config['configurations']['dolphin-quartz']) |
||||
|
||||
# worker.properties |
||||
dolphin_worker_map = config['configurations']['dolphin-worker'] |
||||
|
||||
# zookeeper.properties |
||||
dolphin_zookeeper_map={} |
||||
zookeeperHosts = default("/clusterHostInfo/zookeeper_hosts", []) |
||||
if len(zookeeperHosts) > 0 and "clientPort" in config['configurations']['zoo.cfg']: |
||||
clientPort = config['configurations']['zoo.cfg']['clientPort'] |
||||
zookeeperPort = ":" + clientPort + "," |
||||
dolphin_zookeeper_map['zookeeper.quorum'] = zookeeperPort.join(zookeeperHosts) + ":" + clientPort |
||||
dolphin_zookeeper_map.update(config['configurations']['dolphin-zookeeper']) |
||||
|
||||
|
||||
|
||||
@ -1,31 +0,0 @@
|
||||
""" |
||||
Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
Unless required by applicable law or agreed to in writing, software |
||||
distributed under the License is distributed on an "AS IS" BASIS, |
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
See the License for the specific language governing permissions and |
||||
limitations under the License. |
||||
|
||||
""" |
||||
|
||||
from resource_management import * |
||||
from resource_management.libraries.functions import get_unique_id_and_date |
||||
|
||||
class ServiceCheck(Script): |
||||
def service_check(self, env): |
||||
import params |
||||
#env.set_params(params) |
||||
|
||||
# Execute(format("which pika_server")) |
||||
|
||||
if __name__ == "__main__": |
||||
ServiceCheck().execute() |
||||
@ -1,23 +0,0 @@
|
||||
""" |
||||
Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
Unless required by applicable law or agreed to in writing, software |
||||
distributed under the License is distributed on an "AS IS" BASIS, |
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
See the License for the specific language governing permissions and |
||||
limitations under the License. |
||||
""" |
||||
|
||||
from resource_management import * |
||||
|
||||
config = Script.get_config() |
||||
|
||||
dolphin_run_dir = "/opt/soft/run/dolphinscheduler/" |
||||
@ -1,20 +0,0 @@
|
||||
# |
||||
# Licensed to the Apache Software Foundation (ASF) under one or more |
||||
# contributor license agreements. See the NOTICE file distributed with |
||||
# this work for additional information regarding copyright ownership. |
||||
# The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
# (the "License"); you may not use this file except in compliance with |
||||
# the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
# |
||||
|
||||
{% for key, value in dolphin_app_api_map.iteritems() -%} |
||||
{{key}}={{value}} |
||||
{% endfor %} |
||||
@ -1,20 +0,0 @@
|
||||
# |
||||
# Licensed to the Apache Software Foundation (ASF) under one or more |
||||
# contributor license agreements. See the NOTICE file distributed with |
||||
# this work for additional information regarding copyright ownership. |
||||
# The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
# (the "License"); you may not use this file except in compliance with |
||||
# the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
# |
||||
|
||||
{% for key, value in dolphin_common_map.iteritems() -%} |
||||
{{key}}={{value}} |
||||
{% endfor %} |
||||
@ -1,20 +0,0 @@
|
||||
# |
||||
# Licensed to the Apache Software Foundation (ASF) under one or more |
||||
# contributor license agreements. See the NOTICE file distributed with |
||||
# this work for additional information regarding copyright ownership. |
||||
# The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
# (the "License"); you may not use this file except in compliance with |
||||
# the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
# |
||||
|
||||
{% for key, value in dolphin_datasource_map.iteritems() -%} |
||||
{{key}}={{value}} |
||||
{% endfor %} |
||||
@ -1,116 +0,0 @@
|
||||
#!/bin/sh |
||||
# |
||||
# Licensed to the Apache Software Foundation (ASF) under one or more |
||||
# contributor license agreements. See the NOTICE file distributed with |
||||
# this work for additional information regarding copyright ownership. |
||||
# The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
# (the "License"); you may not use this file except in compliance with |
||||
# the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
# |
||||
|
||||
usage="Usage: dolphinscheduler-daemon.sh (start|stop) <command> " |
||||
|
||||
# if no args specified, show usage |
||||
if [ $# -le 1 ]; then |
||||
echo $usage |
||||
exit 1 |
||||
fi |
||||
|
||||
startStop=$1 |
||||
shift |
||||
command=$1 |
||||
shift |
||||
|
||||
echo "Begin $startStop $command......" |
||||
|
||||
BIN_DIR=`dirname $0` |
||||
BIN_DIR=`cd "$BIN_DIR"; pwd` |
||||
DOLPHINSCHEDULER_HOME=$BIN_DIR/.. |
||||
|
||||
export HOSTNAME=`hostname` |
||||
|
||||
DOLPHINSCHEDULER_LIB_JARS={{dolphin_lib_jars}} |
||||
|
||||
DOLPHINSCHEDULER_OPTS="-server -Xmx16g -Xms4g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70" |
||||
STOP_TIMEOUT=5 |
||||
|
||||
log={{dolphin_log_dir}}/dolphinscheduler-$command-$HOSTNAME.out |
||||
pid={{dolphin_pidfile_dir}}/$command.pid |
||||
|
||||
cd $DOLPHINSCHEDULER_HOME |
||||
|
||||
if [ "$command" = "api-server" ]; then |
||||
LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/logback-api.xml -Dspring.profiles.active=api" |
||||
CLASS=org.apache.dolphinscheduler.api.ApiApplicationServer |
||||
elif [ "$command" = "master-server" ]; then |
||||
LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/logback-master.xml -Ddruid.mysql.usePingMethod=false" |
||||
CLASS=org.apache.dolphinscheduler.server.master.MasterServer |
||||
elif [ "$command" = "worker-server" ]; then |
||||
LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/logback-worker.xml -Ddruid.mysql.usePingMethod=false" |
||||
CLASS=org.apache.dolphinscheduler.server.worker.WorkerServer |
||||
elif [ "$command" = "alert-server" ]; then |
||||
LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/logback-alert.xml" |
||||
CLASS=org.apache.dolphinscheduler.alert.AlertServer |
||||
elif [ "$command" = "logger-server" ]; then |
||||
CLASS=org.apache.dolphinscheduler.server.log.LoggerServer |
||||
else |
||||
echo "Error: No command named \`$command' was found." |
||||
exit 1 |
||||
fi |
||||
|
||||
case $startStop in |
||||
(start) |
||||
|
||||
if [ -f $pid ]; then |
||||
if kill -0 `cat $pid` > /dev/null 2>&1; then |
||||
echo $command running as process `cat $pid`. Stop it first. |
||||
exit 1 |
||||
fi |
||||
fi |
||||
|
||||
echo starting $command, logging to $log |
||||
|
||||
exec_command="$LOG_FILE $DOLPHINSCHEDULER_OPTS -classpath {{dolphin_conf_dir}}:{{dolphin_lib_jars}} $CLASS" |
||||
|
||||
echo "nohup java $exec_command > $log 2>&1 < /dev/null &" |
||||
nohup java $exec_command > $log 2>&1 < /dev/null & |
||||
echo $! > $pid |
||||
;; |
||||
|
||||
(stop) |
||||
|
||||
if [ -f $pid ]; then |
||||
TARGET_PID=`cat $pid` |
||||
if kill -0 $TARGET_PID > /dev/null 2>&1; then |
||||
echo stopping $command |
||||
kill $TARGET_PID |
||||
sleep $STOP_TIMEOUT |
||||
if kill -0 $TARGET_PID > /dev/null 2>&1; then |
||||
echo "$command did not stop gracefully after $STOP_TIMEOUT seconds: killing with kill -9" |
||||
kill -9 $TARGET_PID |
||||
fi |
||||
else |
||||
echo no $command to stop |
||||
fi |
||||
rm -f $pid |
||||
else |
||||
echo no $command to stop |
||||
fi |
||||
;; |
||||
|
||||
(*) |
||||
echo $usage |
||||
exit 1 |
||||
;; |
||||
|
||||
esac |
||||
|
||||
echo "End $startStop $command." |
||||
@ -1,20 +0,0 @@
|
||||
# |
||||
# Licensed to the Apache Software Foundation (ASF) under one or more |
||||
# contributor license agreements. See the NOTICE file distributed with |
||||
# this work for additional information regarding copyright ownership. |
||||
# The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
# (the "License"); you may not use this file except in compliance with |
||||
# the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
# |
||||
|
||||
{% for key, value in dolphin_master_map.iteritems() -%} |
||||
{{key}}={{value}} |
||||
{% endfor %} |
||||
@ -1,20 +0,0 @@
|
||||
# |
||||
# Licensed to the Apache Software Foundation (ASF) under one or more |
||||
# contributor license agreements. See the NOTICE file distributed with |
||||
# this work for additional information regarding copyright ownership. |
||||
# The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
# (the "License"); you may not use this file except in compliance with |
||||
# the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
# |
||||
|
||||
{% for key, value in dolphin_quartz_map.iteritems() -%} |
||||
{{key}}={{value}} |
||||
{% endfor %} |
||||
@ -1,20 +0,0 @@
|
||||
# |
||||
# Licensed to the Apache Software Foundation (ASF) under one or more |
||||
# contributor license agreements. See the NOTICE file distributed with |
||||
# this work for additional information regarding copyright ownership. |
||||
# The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
# (the "License"); you may not use this file except in compliance with |
||||
# the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
# |
||||
|
||||
{% for key, value in dolphin_worker_map.iteritems() -%} |
||||
{{key}}={{value}} |
||||
{% endfor %} |
||||
@ -1,20 +0,0 @@
|
||||
# |
||||
# Licensed to the Apache Software Foundation (ASF) under one or more |
||||
# contributor license agreements. See the NOTICE file distributed with |
||||
# this work for additional information regarding copyright ownership. |
||||
# The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
# (the "License"); you may not use this file except in compliance with |
||||
# the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
# |
||||
|
||||
{% for key, value in dolphin_zookeeper_map.iteritems() -%} |
||||
{{key}}={{value}} |
||||
{% endfor %} |
||||
@ -1,26 +0,0 @@
|
||||
{ |
||||
"name": "default", |
||||
"description": "default quick links configuration", |
||||
"configuration": { |
||||
"protocol": |
||||
{ |
||||
"type":"http" |
||||
}, |
||||
|
||||
"links": [ |
||||
{ |
||||
"name": "dolphin-application-ui", |
||||
"label": "DolphinApplication UI", |
||||
"requires_user_name": "false", |
||||
"component_name": "DOLPHIN_API", |
||||
"url": "%@://%@:%@/dolphinscheduler/ui/view/login/index.html", |
||||
"port":{ |
||||
"http_property": "server.port", |
||||
"http_default_port": "12345", |
||||
"regex": "^(\\d+)$", |
||||
"site": "dolphin-application-api" |
||||
} |
||||
} |
||||
] |
||||
} |
||||
} |
||||
@ -1,661 +0,0 @@
|
||||
{ |
||||
"name": "default", |
||||
"description": "Default theme for Dolphin Scheduler service", |
||||
"configuration": { |
||||
"layouts": [ |
||||
{ |
||||
"name": "default", |
||||
"tabs": [ |
||||
{ |
||||
"name": "settings", |
||||
"display-name": "Settings", |
||||
"layout": { |
||||
"tab-rows": "3", |
||||
"tab-columns": "3", |
||||
"sections": [ |
||||
{ |
||||
"name": "dolphin-env-config", |
||||
"display-name": "Dolphin Env Config", |
||||
"row-index": "0", |
||||
"column-index": "0", |
||||
"row-span": "1", |
||||
"column-span": "2", |
||||
"section-rows": "1", |
||||
"section-columns": "2", |
||||
"subsections": [ |
||||
{ |
||||
"name": "env-row1-col1", |
||||
"display-name": "Deploy User Info", |
||||
"row-index": "0", |
||||
"column-index": "0", |
||||
"row-span": "1", |
||||
"column-span": "1" |
||||
}, |
||||
{ |
||||
"name": "env-row1-col2", |
||||
"display-name": "System Env Optimization", |
||||
"row-index": "0", |
||||
"column-index": "1", |
||||
"row-span": "1", |
||||
"column-span": "1" |
||||
} |
||||
] |
||||
}, |
||||
{ |
||||
"name": "dolphin-database-config", |
||||
"display-name": "Database Config", |
||||
"row-index": "1", |
||||
"column-index": "0", |
||||
"row-span": "1", |
||||
"column-span": "2", |
||||
"section-rows": "1", |
||||
"section-columns": "3", |
||||
"subsections": [ |
||||
{ |
||||
"name": "database-row1-col1", |
||||
"row-index": "0", |
||||
"column-index": "0", |
||||
"row-span": "1", |
||||
"column-span": "1" |
||||
}, |
||||
{ |
||||
"name": "database-row1-col2", |
||||
"row-index": "0", |
||||
"column-index": "1", |
||||
"row-span": "1", |
||||
"column-span": "1" |
||||
}, |
||||
{ |
||||
"name": "database-row1-col3", |
||||
"row-index": "0", |
||||
"column-index": "2", |
||||
"row-span": "1", |
||||
"column-span": "1" |
||||
} |
||||
] |
||||
}, |
||||
{ |
||||
"name": "dynamic-config", |
||||
"row-index": "2", |
||||
"column-index": "0", |
||||
"row-span": "1", |
||||
"column-span": "2", |
||||
"section-rows": "1", |
||||
"section-columns": "3", |
||||
"subsections": [ |
||||
{ |
||||
"name": "dynamic-row1-col1", |
||||
"display-name": "Resource FS Config", |
||||
"row-index": "0", |
||||
"column-index": "0", |
||||
"row-span": "1", |
||||
"column-span": "1" |
||||
}, |
||||
{ |
||||
"name": "dynamic-row1-col2", |
||||
"display-name": "Kerberos Info", |
||||
"row-index": "0", |
||||
"column-index": "1", |
||||
"row-span": "1", |
||||
"column-span": "1" |
||||
}, |
||||
{ |
||||
"name": "dynamic-row1-col3", |
||||
"display-name": "Wechat Info", |
||||
"row-index": "0", |
||||
"column-index": "1", |
||||
"row-span": "1", |
||||
"column-span": "1" |
||||
} |
||||
] |
||||
} |
||||
] |
||||
} |
||||
} |
||||
] |
||||
} |
||||
], |
||||
"placement": { |
||||
"configuration-layout": "default", |
||||
"configs": [ |
||||
{ |
||||
"config": "dolphin-env/dolphin.database.type", |
||||
"subsection-name": "database-row1-col1" |
||||
}, |
||||
{ |
||||
"config": "dolphin-env/dolphin.database.host", |
||||
"subsection-name": "database-row1-col2" |
||||
}, |
||||
{ |
||||
"config": "dolphin-env/dolphin.database.port", |
||||
"subsection-name": "database-row1-col2" |
||||
}, |
||||
{ |
||||
"config": "dolphin-env/dolphin.database.username", |
||||
"subsection-name": "database-row1-col3" |
||||
}, |
||||
{ |
||||
"config": "dolphin-env/dolphin.database.password", |
||||
"subsection-name": "database-row1-col3" |
||||
}, |
||||
{ |
||||
"config": "dolphin-env/dolphin.user", |
||||
"subsection-name": "env-row1-col1" |
||||
}, |
||||
{ |
||||
"config": "dolphin-env/dolphin.group", |
||||
"subsection-name": "env-row1-col1" |
||||
}, |
||||
{ |
||||
"config": "dolphin-env/dolphinscheduler-env-content", |
||||
"subsection-name": "env-row1-col2" |
||||
}, |
||||
{ |
||||
"config": "dolphin-common/resource.storage.type", |
||||
"subsection-name": "dynamic-row1-col1" |
||||
}, |
||||
{ |
||||
"config": "dolphin-common/resource.upload.path", |
||||
"subsection-name": "dynamic-row1-col1", |
||||
"depends-on": [ |
||||
{ |
||||
"configs":[ |
||||
"dolphin-common/resource.storage.type" |
||||
], |
||||
"if": "${dolphin-common/resource.storage.type} === HDFS || ${dolphin-common/resource.storage.type} === S3", |
||||
"then": { |
||||
"property_value_attributes": { |
||||
"visible": true |
||||
} |
||||
}, |
||||
"else": { |
||||
"property_value_attributes": { |
||||
"visible": false |
||||
} |
||||
} |
||||
} |
||||
] |
||||
}, |
||||
{ |
||||
"config": "dolphin-common/hdfs.root.user", |
||||
"subsection-name": "dynamic-row1-col1", |
||||
"depends-on": [ |
||||
{ |
||||
"configs":[ |
||||
"dolphin-common/resource.storage.type" |
||||
], |
||||
"if": "${dolphin-common/resource.storage.type} === HDFS", |
||||
"then": { |
||||
"property_value_attributes": { |
||||
"visible": true |
||||
} |
||||
}, |
||||
"else": { |
||||
"property_value_attributes": { |
||||
"visible": false |
||||
} |
||||
} |
||||
} |
||||
] |
||||
}, |
||||
{ |
||||
"config": "dolphin-common/data.store2hdfs.basepath", |
||||
"subsection-name": "dynamic-row1-col1", |
||||
"depends-on": [ |
||||
{ |
||||
"configs":[ |
||||
"dolphin-common/resource.storage.type" |
||||
], |
||||
"if": "${dolphin-common/resource.storage.type} === HDFS", |
||||
"then": { |
||||
"property_value_attributes": { |
||||
"visible": true |
||||
} |
||||
}, |
||||
"else": { |
||||
"property_value_attributes": { |
||||
"visible": false |
||||
} |
||||
} |
||||
} |
||||
] |
||||
}, |
||||
{ |
||||
"config": "dolphin-common/fs.defaultFS", |
||||
"subsection-name": "dynamic-row1-col1", |
||||
"depends-on": [ |
||||
{ |
||||
"configs":[ |
||||
"dolphin-common/resource.storage.type" |
||||
], |
||||
"if": "${dolphin-common/resource.storage.type} === HDFS", |
||||
"then": { |
||||
"property_value_attributes": { |
||||
"visible": true |
||||
} |
||||
}, |
||||
"else": { |
||||
"property_value_attributes": { |
||||
"visible": false |
||||
} |
||||
} |
||||
} |
||||
] |
||||
}, |
||||
{ |
||||
"config": "dolphin-common/fs.s3a.endpoint", |
||||
"subsection-name": "dynamic-row1-col1", |
||||
"depends-on": [ |
||||
{ |
||||
"configs":[ |
||||
"dolphin-common/resource.storage.type" |
||||
], |
||||
"if": "${dolphin-common/resource.storage.type} === S3", |
||||
"then": { |
||||
"property_value_attributes": { |
||||
"visible": true |
||||
} |
||||
}, |
||||
"else": { |
||||
"property_value_attributes": { |
||||
"visible": false |
||||
} |
||||
} |
||||
} |
||||
] |
||||
}, |
||||
{ |
||||
"config": "dolphin-common/fs.s3a.access.key", |
||||
"subsection-name": "dynamic-row1-col1", |
||||
"depends-on": [ |
||||
{ |
||||
"configs":[ |
||||
"dolphin-common/resource.storage.type" |
||||
], |
||||
"if": "${dolphin-common/resource.storage.type} === S3", |
||||
"then": { |
||||
"property_value_attributes": { |
||||
"visible": true |
||||
} |
||||
}, |
||||
"else": { |
||||
"property_value_attributes": { |
||||
"visible": false |
||||
} |
||||
} |
||||
} |
||||
] |
||||
}, |
||||
{ |
||||
"config": "dolphin-common/fs.s3a.secret.key", |
||||
"subsection-name": "dynamic-row1-col1", |
||||
"depends-on": [ |
||||
{ |
||||
"configs":[ |
||||
"dolphin-common/resource.storage.type" |
||||
], |
||||
"if": "${dolphin-common/resource.storage.type} === S3", |
||||
"then": { |
||||
"property_value_attributes": { |
||||
"visible": true |
||||
} |
||||
}, |
||||
"else": { |
||||
"property_value_attributes": { |
||||
"visible": false |
||||
} |
||||
} |
||||
} |
||||
] |
||||
}, |
||||
{ |
||||
"config": "dolphin-common/hadoop.security.authentication.startup.state", |
||||
"subsection-name": "dynamic-row1-col2" |
||||
}, |
||||
{ |
||||
"config": "dolphin-common/java.security.krb5.conf.path", |
||||
"subsection-name": "dynamic-row1-col2", |
||||
"depends-on": [ |
||||
{ |
||||
"configs":[ |
||||
"dolphin-common/hadoop.security.authentication.startup.state" |
||||
], |
||||
"if": "${dolphin-common/hadoop.security.authentication.startup.state}", |
||||
"then": { |
||||
"property_value_attributes": { |
||||
"visible": true |
||||
} |
||||
}, |
||||
"else": { |
||||
"property_value_attributes": { |
||||
"visible": false |
||||
} |
||||
} |
||||
} |
||||
] |
||||
}, |
||||
{ |
||||
"config": "dolphin-common/login.user.keytab.username", |
||||
"subsection-name": "dynamic-row1-col2", |
||||
"depends-on": [ |
||||
{ |
||||
"configs":[ |
||||
"dolphin-common/hadoop.security.authentication.startup.state" |
||||
], |
||||
"if": "${dolphin-common/hadoop.security.authentication.startup.state}", |
||||
"then": { |
||||
"property_value_attributes": { |
||||
"visible": true |
||||
} |
||||
}, |
||||
"else": { |
||||
"property_value_attributes": { |
||||
"visible": false |
||||
} |
||||
} |
||||
} |
||||
] |
||||
}, |
||||
{ |
||||
"config": "dolphin-common/login.user.keytab.path", |
||||
"subsection-name": "dynamic-row1-col2", |
||||
"depends-on": [ |
||||
{ |
||||
"configs":[ |
||||
"dolphin-common/hadoop.security.authentication.startup.state" |
||||
], |
||||
"if": "${dolphin-common/hadoop.security.authentication.startup.state}", |
||||
"then": { |
||||
"property_value_attributes": { |
||||
"visible": true |
||||
} |
||||
}, |
||||
"else": { |
||||
"property_value_attributes": { |
||||
"visible": false |
||||
} |
||||
} |
||||
} |
||||
] |
||||
}, |
||||
{ |
||||
"config": "dolphin-common/kerberos.expire.time", |
||||
"subsection-name": "dynamic-row1-col2", |
||||
"depends-on": [ |
||||
{ |
||||
"configs":[ |
||||
"dolphin-common/hadoop.security.authentication.startup.state" |
||||
], |
||||
"if": "${dolphin-common/hadoop.security.authentication.startup.state}", |
||||
"then": { |
||||
"property_value_attributes": { |
||||
"visible": true |
||||
} |
||||
}, |
||||
"else": { |
||||
"property_value_attributes": { |
||||
"visible": false |
||||
} |
||||
} |
||||
} |
||||
] |
||||
}, |
||||
{ |
||||
"config": "dolphin-alert/enterprise.wechat.enable", |
||||
"subsection-name": "dynamic-row1-col3" |
||||
}, |
||||
{ |
||||
"config": "dolphin-alert/enterprise.wechat.corp.id", |
||||
"subsection-name": "dynamic-row1-col3", |
||||
"depends-on": [ |
||||
{ |
||||
"configs":[ |
||||
"dolphin-alert/enterprise.wechat.enable" |
||||
], |
||||
"if": "${dolphin-alert/enterprise.wechat.enable}", |
||||
"then": { |
||||
"property_value_attributes": { |
||||
"visible": true |
||||
} |
||||
}, |
||||
"else": { |
||||
"property_value_attributes": { |
||||
"visible": false |
||||
} |
||||
} |
||||
} |
||||
] |
||||
}, |
||||
{ |
||||
"config": "dolphin-alert/enterprise.wechat.secret", |
||||
"subsection-name": "dynamic-row1-col3", |
||||
"depends-on": [ |
||||
{ |
||||
"configs":[ |
||||
"dolphin-alert/enterprise.wechat.enable" |
||||
], |
||||
"if": "${dolphin-alert/enterprise.wechat.enable}", |
||||
"then": { |
||||
"property_value_attributes": { |
||||
"visible": true |
||||
} |
||||
}, |
||||
"else": { |
||||
"property_value_attributes": { |
||||
"visible": false |
||||
} |
||||
} |
||||
} |
||||
] |
||||
}, |
||||
{ |
||||
"config": "dolphin-alert/enterprise.wechat.agent.id", |
||||
"subsection-name": "dynamic-row1-col3", |
||||
"depends-on": [ |
||||
{ |
||||
"configs":[ |
||||
"dolphin-alert/enterprise.wechat.enable" |
||||
], |
||||
"if": "${dolphin-alert/enterprise.wechat.enable}", |
||||
"then": { |
||||
"property_value_attributes": { |
||||
"visible": true |
||||
} |
||||
}, |
||||
"else": { |
||||
"property_value_attributes": { |
||||
"visible": false |
||||
} |
||||
} |
||||
} |
||||
] |
||||
}, |
||||
{ |
||||
"config": "dolphin-alert/enterprise.wechat.users", |
||||
"subsection-name": "dynamic-row1-col3", |
||||
"depends-on": [ |
||||
{ |
||||
"configs":[ |
||||
"dolphin-alert/enterprise.wechat.enable" |
||||
], |
||||
"if": "${dolphin-alert/enterprise.wechat.enable}", |
||||
"then": { |
||||
"property_value_attributes": { |
||||
"visible": true |
||||
} |
||||
}, |
||||
"else": { |
||||
"property_value_attributes": { |
||||
"visible": false |
||||
} |
||||
} |
||||
} |
||||
] |
||||
} |
||||
] |
||||
}, |
||||
"widgets": [ |
||||
{ |
||||
"config": "dolphin-env/dolphin.database.type", |
||||
"widget": { |
||||
"type": "combo" |
||||
} |
||||
}, |
||||
{ |
||||
"config": "dolphin-env/dolphin.database.host", |
||||
"widget": { |
||||
"type": "text-field" |
||||
} |
||||
}, |
||||
{ |
||||
"config": "dolphin-env/dolphin.database.port", |
||||
"widget": { |
||||
"type": "text-field", |
||||
"units": [ |
||||
{ |
||||
"unit-name": "int" |
||||
} |
||||
] |
||||
} |
||||
}, |
||||
{ |
||||
"config": "dolphin-env/dolphin.database.username", |
||||
"widget": { |
||||
"type": "text-field" |
||||
} |
||||
}, |
||||
{ |
||||
"config": "dolphin-env/dolphin.database.password", |
||||
"widget": { |
||||
"type": "password" |
||||
} |
||||
}, |
||||
{ |
||||
"config": "dolphin-env/dolphin.user", |
||||
"widget": { |
||||
"type": "text-field" |
||||
} |
||||
}, |
||||
{ |
||||
"config": "dolphin-env/dolphin.group", |
||||
"widget": { |
||||
"type": "text-field" |
||||
} |
||||
}, |
||||
{ |
||||
"config": "dolphin-env/dolphinscheduler-env-content", |
||||
"widget": { |
||||
"type": "text-area" |
||||
} |
||||
}, |
||||
{ |
||||
"config": "dolphin-common/resource.storage.type", |
||||
"widget": { |
||||
"type": "combo" |
||||
} |
||||
}, |
||||
{ |
||||
"config": "dolphin-common/resource.upload.path", |
||||
"widget": { |
||||
"type": "text-field" |
||||
} |
||||
}, |
||||
{ |
||||
"config": "dolphin-common/hdfs.root.user", |
||||
"widget": { |
||||
"type": "text-field" |
||||
} |
||||
}, |
||||
{ |
||||
"config": "dolphin-common/data.store2hdfs.basepath", |
||||
"widget": { |
||||
"type": "text-field" |
||||
} |
||||
}, |
||||
{ |
||||
"config": "dolphin-common/fs.defaultFS", |
||||
"widget": { |
||||
"type": "text-field" |
||||
} |
||||
}, |
||||
{ |
||||
"config": "dolphin-common/fs.s3a.endpoint", |
||||
"widget": { |
||||
"type": "text-field" |
||||
} |
||||
}, |
||||
{ |
||||
"config": "dolphin-common/fs.s3a.access.key", |
||||
"widget": { |
||||
"type": "text-field" |
||||
} |
||||
}, |
||||
{ |
||||
"config": "dolphin-common/fs.s3a.secret.key", |
||||
"widget": { |
||||
"type": "text-field" |
||||
} |
||||
}, |
||||
{ |
||||
"config": "dolphin-common/hadoop.security.authentication.startup.state", |
||||
"widget": { |
||||
"type": "toggle" |
||||
} |
||||
}, |
||||
{ |
||||
"config": "dolphin-common/java.security.krb5.conf.path", |
||||
"widget": { |
||||
"type": "text-field" |
||||
} |
||||
}, |
||||
{ |
||||
"config": "dolphin-common/login.user.keytab.username", |
||||
"widget": { |
||||
"type": "text-field" |
||||
} |
||||
}, |
||||
{ |
||||
"config": "dolphin-common/login.user.keytab.path", |
||||
"widget": { |
||||
"type": "text-field" |
||||
} |
||||
}, |
||||
{ |
||||
"config": "dolphin-common/kerberos.expire.time", |
||||
"widget": { |
||||
"type": "text-field" |
||||
} |
||||
}, |
||||
{ |
||||
"config": "dolphin-alert/enterprise.wechat.enable", |
||||
"widget": { |
||||
"type": "toggle" |
||||
} |
||||
}, |
||||
{ |
||||
"config": "dolphin-alert/enterprise.wechat.corp.id", |
||||
"widget": { |
||||
"type": "text-field" |
||||
} |
||||
}, |
||||
{ |
||||
"config": "dolphin-alert/enterprise.wechat.secret", |
||||
"widget": { |
||||
"type": "text-field" |
||||
} |
||||
}, |
||||
{ |
||||
"config": "dolphin-alert/enterprise.wechat.agent.id", |
||||
"widget": { |
||||
"type": "text-field" |
||||
} |
||||
}, |
||||
{ |
||||
"config": "dolphin-alert/enterprise.wechat.users", |
||||
"widget": { |
||||
"type": "text-field" |
||||
} |
||||
} |
||||
] |
||||
} |
||||
} |
||||
@ -0,0 +1,133 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
package org.apache.dolphinscheduler.alert.plugin; |
||||
|
||||
import org.apache.dolphinscheduler.alert.manager.EmailManager; |
||||
import org.apache.dolphinscheduler.alert.manager.EnterpriseWeChatManager; |
||||
import org.apache.dolphinscheduler.alert.utils.Constants; |
||||
import org.apache.dolphinscheduler.alert.utils.EnterpriseWeChatUtils; |
||||
import org.apache.dolphinscheduler.common.utils.CollectionUtils; |
||||
import org.apache.dolphinscheduler.common.utils.StringUtils; |
||||
import org.apache.dolphinscheduler.plugin.api.AlertPlugin; |
||||
import org.apache.dolphinscheduler.plugin.model.AlertData; |
||||
import org.apache.dolphinscheduler.plugin.model.AlertInfo; |
||||
import org.apache.dolphinscheduler.plugin.model.PluginName; |
||||
import org.slf4j.Logger; |
||||
import org.slf4j.LoggerFactory; |
||||
|
||||
import java.util.*; |
||||
|
||||
/** |
||||
* EmailAlertPlugin |
||||
* |
||||
* This plugin is a default plugin, and mix up email and enterprise wechat, because adapt with former alert behavior |
||||
*/ |
||||
public class EmailAlertPlugin implements AlertPlugin { |
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(EmailAlertPlugin.class); |
||||
|
||||
private PluginName pluginName; |
||||
|
||||
private static final EmailManager emailManager = new EmailManager(); |
||||
private static final EnterpriseWeChatManager weChatManager = new EnterpriseWeChatManager(); |
||||
|
||||
public EmailAlertPlugin() { |
||||
this.pluginName = new PluginName(); |
||||
this.pluginName.setEnglish(Constants.PLUGIN_DEFAULT_EMAIL_EN); |
||||
this.pluginName.setChinese(Constants.PLUGIN_DEFAULT_EMAIL_CH); |
||||
} |
||||
|
||||
@Override |
||||
public String getId() { |
||||
return Constants.PLUGIN_DEFAULT_EMAIL; |
||||
} |
||||
|
||||
@Override |
||||
public PluginName getName() { |
||||
return pluginName; |
||||
} |
||||
|
||||
@Override |
||||
@SuppressWarnings("unchecked") |
||||
public Map<String, Object> process(AlertInfo info) { |
||||
Map<String, Object> retMaps = new HashMap<>(); |
||||
|
||||
AlertData alert = info.getAlertData(); |
||||
|
||||
List<String> receviersList = (List<String>) info.getProp(Constants.PLUGIN_DEFAULT_EMAIL_RECEIVERS); |
||||
|
||||
// receiving group list
|
||||
// custom receiver
|
||||
String receivers = alert.getReceivers(); |
||||
if (StringUtils.isNotEmpty(receivers)) { |
||||
String[] splits = receivers.split(","); |
||||
receviersList.addAll(Arrays.asList(splits)); |
||||
} |
||||
|
||||
List<String> receviersCcList = new ArrayList<>(); |
||||
// Custom Copier
|
||||
String receiversCc = alert.getReceiversCc(); |
||||
if (StringUtils.isNotEmpty(receiversCc)) { |
||||
String[] splits = receiversCc.split(","); |
||||
receviersCcList.addAll(Arrays.asList(splits)); |
||||
} |
||||
|
||||
if (CollectionUtils.isEmpty(receviersList) && CollectionUtils.isEmpty(receviersCcList)) { |
||||
logger.warn("alert send error : At least one receiver address required"); |
||||
retMaps.put(Constants.STATUS, "false"); |
||||
retMaps.put(Constants.MESSAGE, "execution failure,At least one receiver address required."); |
||||
return retMaps; |
||||
} |
||||
|
||||
retMaps = emailManager.send(receviersList, receviersCcList, alert.getTitle(), alert.getContent(), |
||||
alert.getShowType()); |
||||
|
||||
//send flag
|
||||
boolean flag = false; |
||||
|
||||
if (retMaps == null) { |
||||
retMaps = new HashMap<>(); |
||||
retMaps.put(Constants.MESSAGE, "alert send error."); |
||||
retMaps.put(Constants.STATUS, "false"); |
||||
logger.info("alert send error : {}", retMaps.get(Constants.MESSAGE)); |
||||
return retMaps; |
||||
} |
||||
|
||||
flag = Boolean.parseBoolean(String.valueOf(retMaps.get(Constants.STATUS))); |
||||
|
||||
if (flag) { |
||||
logger.info("alert send success"); |
||||
retMaps.put(Constants.MESSAGE, "email send success."); |
||||
if (EnterpriseWeChatUtils.isEnable()) { |
||||
logger.info("Enterprise WeChat is enable!"); |
||||
try { |
||||
String token = EnterpriseWeChatUtils.getToken(); |
||||
weChatManager.send(info, token); |
||||
} catch (Exception e) { |
||||
logger.error(e.getMessage(), e); |
||||
} |
||||
} |
||||
|
||||
} else { |
||||
retMaps.put(Constants.MESSAGE, "alert send error."); |
||||
logger.info("alert send error : {}", retMaps.get(Constants.MESSAGE)); |
||||
} |
||||
|
||||
return retMaps; |
||||
} |
||||
|
||||
} |
||||
@ -0,0 +1,80 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
package org.apache.dolphinscheduler.alert.plugin; |
||||
|
||||
import org.apache.dolphinscheduler.alert.utils.Constants; |
||||
import org.apache.dolphinscheduler.common.enums.ShowType; |
||||
import org.apache.dolphinscheduler.plugin.api.AlertPlugin; |
||||
import org.apache.dolphinscheduler.plugin.model.AlertData; |
||||
import org.apache.dolphinscheduler.plugin.model.AlertInfo; |
||||
import org.apache.dolphinscheduler.plugin.model.PluginName; |
||||
import org.junit.Before; |
||||
import org.junit.Test; |
||||
import org.slf4j.Logger; |
||||
import org.slf4j.LoggerFactory; |
||||
|
||||
import java.util.ArrayList; |
||||
import java.util.List; |
||||
import java.util.Map; |
||||
|
||||
import static org.junit.Assert.*; |
||||
|
||||
public class EmailAlertPluginTest { |
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(EmailAlertPluginTest.class); |
||||
|
||||
private AlertPlugin plugin; |
||||
|
||||
@Before |
||||
public void before() { |
||||
plugin = new EmailAlertPlugin(); |
||||
} |
||||
|
||||
@Test |
||||
public void getId() { |
||||
String id = plugin.getId(); |
||||
assertEquals(Constants.PLUGIN_DEFAULT_EMAIL, id); |
||||
} |
||||
|
||||
@Test |
||||
public void getName() { |
||||
PluginName pluginName = plugin.getName(); |
||||
assertEquals(Constants.PLUGIN_DEFAULT_EMAIL_CH, pluginName.getChinese()); |
||||
assertEquals(Constants.PLUGIN_DEFAULT_EMAIL_EN, pluginName.getEnglish()); |
||||
} |
||||
|
||||
@Test |
||||
public void process() { |
||||
AlertInfo alertInfo = new AlertInfo(); |
||||
AlertData alertData = new AlertData(); |
||||
alertData.setId(1) |
||||
.setAlertGroupId(1) |
||||
.setContent("[\"alarm time：2018-02-05\", \"service name：MYSQL_ALTER\", \"alarm name：MYSQL_ALTER_DUMP\", " + |
||||
"\"get the alarm exception.！，interface error，exception information：timed out\", \"request address：http://blog.csdn.net/dreamInTheWorld/article/details/78539286\"]") |
||||
.setLog("test log") |
||||
.setReceivers("xx@xx.com") |
||||
.setReceiversCc("xx@xx.com") |
||||
.setShowType(ShowType.TEXT.getDescp()) |
||||
.setTitle("test title"); |
||||
|
||||
alertInfo.setAlertData(alertData); |
||||
List<String> list = new ArrayList<String>(){{ add("xx@xx.com"); }}; |
||||
alertInfo.addProp("receivers", list); |
||||
Map<String, Object> ret = plugin.process(alertInfo); |
||||
assertFalse(Boolean.parseBoolean(String.valueOf(ret.get(Constants.STATUS)))); |
||||
} |
||||
} |
||||
@ -0,0 +1,107 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
package org.apache.dolphinscheduler.common.plugin; |
||||
|
||||
import org.apache.dolphinscheduler.common.Constants; |
||||
import org.apache.dolphinscheduler.plugin.api.AlertPlugin; |
||||
import org.apache.dolphinscheduler.plugin.spi.AlertPluginProvider; |
||||
import org.slf4j.Logger; |
||||
import org.slf4j.LoggerFactory; |
||||
|
||||
import java.io.File; |
||||
import java.net.MalformedURLException; |
||||
import java.net.URL; |
||||
import java.util.Map; |
||||
import java.util.ServiceLoader; |
||||
import java.util.concurrent.ConcurrentHashMap; |
||||
|
||||
/** |
||||
* FilePluginManager |
||||
*/ |
||||
public class FilePluginManager implements PluginManager { |
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(FilePluginManager.class); |
||||
|
||||
private Map<String, AlertPlugin> pluginMap = new ConcurrentHashMap<>(); |
||||
|
||||
private Map<String, ServiceLoader<AlertPluginProvider>> pluginLoaderMap = new ConcurrentHashMap<>(); |
||||
|
||||
private Map<String, PluginClassLoader> classLoaderMap = new ConcurrentHashMap<>(); |
||||
|
||||
private String[] whitePrefixes; |
||||
|
||||
private String[] excludePrefixes; |
||||
|
||||
public FilePluginManager(String dirPath, String[] whitePrefixes, String[] excludePrefixes) { |
||||
this.whitePrefixes = whitePrefixes; |
||||
this.excludePrefixes = excludePrefixes; |
||||
try { |
||||
load(dirPath); |
||||
} catch (MalformedURLException e) { |
||||
logger.error("load plugins failed.", e); |
||||
} |
||||
} |
||||
|
||||
private void load(String dirPath) throws MalformedURLException { |
||||
logger.info("start to load jar files in {}", dirPath); |
||||
if (dirPath == null) { |
||||
logger.error("not a valid path - {}", dirPath); |
||||
return; |
||||
} |
||||
File[] files = new File(dirPath).listFiles(); |
||||
if (files == null) { |
||||
logger.error("not a valid path - {}", dirPath); |
||||
return; |
||||
} |
||||
for (File file : files) { |
||||
if (file.isDirectory() && !file.getPath().endsWith(Constants.PLUGIN_JAR_SUFFIX)) { |
||||
continue; |
||||
} |
||||
String pluginName = file.getName() |
||||
.substring(0, file.getName().length() - Constants.PLUGIN_JAR_SUFFIX.length()); |
||||
URL[] urls = new URL[]{ file.toURI().toURL() }; |
||||
PluginClassLoader classLoader = |
||||
new PluginClassLoader(urls, Thread.currentThread().getContextClassLoader(), whitePrefixes, excludePrefixes); |
||||
classLoaderMap.put(pluginName, classLoader); |
||||
|
||||
ServiceLoader<AlertPluginProvider> loader = ServiceLoader.load(AlertPluginProvider.class, classLoader); |
||||
pluginLoaderMap.put(pluginName, loader); |
||||
|
||||
loader.forEach(provider -> { |
||||
AlertPlugin plugin = provider.createPlugin(); |
||||
pluginMap.put(plugin.getId(), plugin); |
||||
logger.info("loaded plugin - {}", plugin.getId()); |
||||
}); |
||||
} |
||||
} |
||||
|
||||
@Override |
||||
public AlertPlugin findOne(String name) { |
||||
return pluginMap.get(name); |
||||
} |
||||
|
||||
@Override |
||||
public Map<String, AlertPlugin> findAll() { |
||||
return pluginMap; |
||||
} |
||||
|
||||
@Override |
||||
public void addPlugin(AlertPlugin plugin) { |
||||
pluginMap.put(plugin.getId(), plugin); |
||||
} |
||||
|
||||
} |
||||
@ -0,0 +1,154 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
package org.apache.dolphinscheduler.common.plugin; |
||||
|
||||
import org.slf4j.Logger; |
||||
import org.slf4j.LoggerFactory; |
||||
|
||||
import java.io.IOException; |
||||
import java.net.URL; |
||||
import java.net.URLClassLoader; |
||||
import java.util.Enumeration; |
||||
import java.util.Iterator; |
||||
import java.util.LinkedList; |
||||
import java.util.List; |
||||
|
||||
/** |
||||
* Plugin Class Loader |
||||
*/ |
||||
public class PluginClassLoader extends URLClassLoader { |
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(PluginClassLoader.class); |
||||
|
||||
private static final String JAVA_PACKAGE_PREFIX = "java."; |
||||
private static final String JAVAX_PACKAGE_PREFIX = "javax."; |
||||
|
||||
private final String[] whitePrefixes; |
||||
|
||||
private final String[] excludePrefixes; |
||||
|
||||
public PluginClassLoader(URL[] urls, ClassLoader parent, String[] whitePrefix, String[] excludePreifx) { |
||||
super(urls, parent); |
||||
this.whitePrefixes = whitePrefix; |
||||
this.excludePrefixes = excludePreifx; |
||||
} |
||||
|
||||
@Override |
||||
public Class<?> loadClass(String name) throws ClassNotFoundException { |
||||
logger.trace("Received request to load class '{}'", name); |
||||
synchronized (getClassLoadingLock(name)) { |
||||
if (name.startsWith(JAVA_PACKAGE_PREFIX) || name.startsWith(JAVAX_PACKAGE_PREFIX)) { |
||||
return findSystemClass(name); |
||||
} |
||||
|
||||
boolean isWhitePrefixes = fromWhitePrefix(name); |
||||
boolean isExcludePrefixed = fromExcludePrefix(name); |
||||
|
||||
// if the class is part of the plugin engine use parent class loader
|
||||
if (!isWhitePrefixes && isExcludePrefixed) { |
||||
return getParent().loadClass(name); |
||||
} |
||||
|
||||
// check whether it's already been loaded
|
||||
Class<?> loadedClass = findLoadedClass(name); |
||||
if (loadedClass != null) { |
||||
logger.debug("Found loaded class '{}'", name); |
||||
return loadedClass; |
||||
} |
||||
|
||||
// nope, try to load locally
|
||||
try { |
||||
loadedClass = findClass(name); |
||||
logger.debug("Found class '{}' in plugin classpath", name); |
||||
return loadedClass; |
||||
} catch (ClassNotFoundException e) { |
||||
// try next step
|
||||
} |
||||
|
||||
// use the standard ClassLoader (which follows normal parent delegation)
|
||||
return super.loadClass(name); |
||||
} |
||||
} |
||||
|
||||
private boolean fromWhitePrefix(String name) { |
||||
if (this.whitePrefixes == null) { |
||||
return false; |
||||
} |
||||
for (String whitePrefix : this.whitePrefixes) { |
||||
if (name.startsWith(whitePrefix)) { |
||||
return true; |
||||
} |
||||
} |
||||
return false; |
||||
} |
||||
|
||||
private boolean fromExcludePrefix(String name) { |
||||
if (this.excludePrefixes == null) { |
||||
return false; |
||||
} |
||||
for (String excludePrefix : this.excludePrefixes) { |
||||
if (name.startsWith(excludePrefix)) { |
||||
return true; |
||||
} |
||||
} |
||||
return false; |
||||
} |
||||
|
||||
@Override |
||||
public Enumeration<URL> getResources(String name) throws IOException { |
||||
List<URL> allRes = new LinkedList<>(); |
||||
|
||||
Enumeration<URL> thisRes = findResources(name); |
||||
if (thisRes != null) { |
||||
while (thisRes.hasMoreElements()) { |
||||
allRes.add(thisRes.nextElement()); |
||||
} |
||||
} |
||||
|
||||
Enumeration<URL> parentRes = super.findResources(name); |
||||
if (parentRes != null) { |
||||
while (parentRes.hasMoreElements()) { |
||||
allRes.add(parentRes.nextElement()); |
||||
} |
||||
} |
||||
|
||||
return new Enumeration<URL>() { |
||||
Iterator<URL> it = allRes.iterator(); |
||||
|
||||
@Override |
||||
public boolean hasMoreElements() { |
||||
return it.hasNext(); |
||||
} |
||||
|
||||
@Override |
||||
public URL nextElement() { |
||||
return it.next(); |
||||
} |
||||
}; |
||||
} |
||||
|
||||
@Override |
||||
public URL getResource(String name) { |
||||
URL res = null; |
||||
|
||||
res = findResource(name); |
||||
if (res == null) { |
||||
res = super.getResource(name); |
||||
} |
||||
return res; |
||||
} |
||||
} |
||||
@ -0,0 +1,33 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
package org.apache.dolphinscheduler.common.plugin; |
||||
|
||||
import org.apache.dolphinscheduler.plugin.api.AlertPlugin; |
||||
|
||||
import java.util.Map; |
||||
|
||||
/** |
||||
* PluginManager |
||||
*/ |
||||
public interface PluginManager { |
||||
|
||||
AlertPlugin findOne(String name); |
||||
|
||||
Map<String, AlertPlugin> findAll(); |
||||
|
||||
void addPlugin(AlertPlugin plugin); |
||||
} |
||||
@ -0,0 +1,72 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
package org.apache.dolphinscheduler.common.plugin; |
||||
|
||||
import org.apache.dolphinscheduler.plugin.api.AlertPlugin; |
||||
import org.apache.dolphinscheduler.plugin.model.AlertInfo; |
||||
import org.apache.dolphinscheduler.plugin.model.PluginName; |
||||
import org.junit.Before; |
||||
import org.junit.Test; |
||||
|
||||
import java.util.HashMap; |
||||
import java.util.Map; |
||||
|
||||
import static org.junit.Assert.*; |
||||
|
||||
public class FilePluginManagerTest { |
||||
|
||||
private FilePluginManager filePluginManager; |
||||
private AlertPlugin alertPlugin; |
||||
|
||||
@Before |
||||
public void before() { |
||||
filePluginManager = new FilePluginManager(null, null, null); |
||||
alertPlugin = new AlertPlugin() { |
||||
@Override |
||||
public String getId() { |
||||
return "test"; |
||||
} |
||||
|
||||
@Override |
||||
public PluginName getName() { |
||||
return new PluginName().setChinese("ch").setEnglish("en"); |
||||
} |
||||
|
||||
@Override |
||||
public Map<String, Object> process(AlertInfo info) { |
||||
return new HashMap<>(); |
||||
} |
||||
}; |
||||
} |
||||
|
||||
@Test |
||||
public void findOne() { |
||||
filePluginManager.addPlugin(alertPlugin); |
||||
assertEquals(alertPlugin, filePluginManager.findOne(alertPlugin.getId())); |
||||
} |
||||
|
||||
@Test |
||||
public void findAll() { |
||||
assertNotNull(filePluginManager.findAll()); |
||||
} |
||||
|
||||
@Test |
||||
public void addPlugin() { |
||||
filePluginManager.addPlugin(alertPlugin); |
||||
assertNotNull(filePluginManager.findAll()); |
||||
} |
||||
} |
||||
@ -0,0 +1,61 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
package org.apache.dolphinscheduler.common.plugin; |
||||
|
||||
import org.junit.Before; |
||||
import org.junit.Test; |
||||
|
||||
import java.net.URL; |
||||
|
||||
import static org.junit.Assert.*; |
||||
|
||||
public class PluginClassLoaderTest { |
||||
|
||||
private PluginClassLoader pluginClassLoader; |
||||
private ClassLoader parent; |
||||
|
||||
@Before |
||||
public void setUp() { |
||||
parent = Thread.currentThread().getContextClassLoader(); |
||||
pluginClassLoader = new PluginClassLoader( |
||||
new URL[]{}, parent, |
||||
null, null); |
||||
} |
||||
|
||||
@Test |
||||
public void loadClassNull() { |
||||
Class clazz = null; |
||||
try { |
||||
clazz = pluginClassLoader.loadClass("java.lang.Object"); |
||||
} catch (ClassNotFoundException e) { |
||||
e.printStackTrace(); |
||||
} |
||||
assertEquals(null, clazz.getClassLoader()); |
||||
} |
||||
|
||||
@Test |
||||
public void loadClassApp() { |
||||
Class clazz = null; |
||||
try { |
||||
clazz = pluginClassLoader.loadClass("org.apache.dolphinscheduler.common.Constants"); |
||||
} catch (ClassNotFoundException e) { |
||||
e.printStackTrace(); |
||||
} |
||||
assertEquals(parent, clazz.getClassLoader()); |
||||
} |
||||
|
||||
} |
||||
@ -1,19 +0,0 @@
|
||||
Copyright (C) 2009-2015 The Project Lombok Authors. |
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy |
||||
of this software and associated documentation files (the "Software"), to deal |
||||
in the Software without restriction, including without limitation the rights |
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
||||
copies of the Software, and to permit persons to whom the Software is |
||||
furnished to do so, subject to the following conditions: |
||||
|
||||
The above copyright notice and this permission notice shall be included in |
||||
all copies or substantial portions of the Software. |
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN |
||||
THE SOFTWARE. |
||||
@ -0,0 +1,52 @@
|
||||
<?xml version="1.0"?> |
||||
<!-- |
||||
~ Licensed to the Apache Software Foundation (ASF) under one or more |
||||
~ contributor license agreements. See the NOTICE file distributed with |
||||
~ this work for additional information regarding copyright ownership. |
||||
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
~ (the "License"); you may not use this file except in compliance with |
||||
~ the License. You may obtain a copy of the License at |
||||
~ |
||||
~ http://www.apache.org/licenses/LICENSE-2.0 |
||||
~ |
||||
~ Unless required by applicable law or agreed to in writing, software |
||||
~ distributed under the License is distributed on an "AS IS" BASIS, |
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
~ See the License for the specific language governing permissions and |
||||
~ limitations under the License. |
||||
--> |
||||
|
||||
<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" |
||||
xmlns="http://maven.apache.org/POM/4.0.0" |
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> |
||||
<modelVersion>4.0.0</modelVersion> |
||||
<parent> |
||||
<groupId>org.apache.dolphinscheduler</groupId> |
||||
<artifactId>dolphinscheduler</artifactId> |
||||
<version>1.2.1-SNAPSHOT</version> |
||||
</parent> |
||||
<artifactId>dolphinscheduler-plugin-api</artifactId> |
||||
<name>${project.artifactId}</name> |
||||
<packaging>jar</packaging> |
||||
|
||||
<properties> |
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> |
||||
</properties> |
||||
|
||||
<dependencies> |
||||
<dependency> |
||||
<groupId>org.slf4j</groupId> |
||||
<artifactId>slf4j-api</artifactId> |
||||
</dependency> |
||||
<dependency> |
||||
<groupId>junit</groupId> |
||||
<artifactId>junit</artifactId> |
||||
<scope>test</scope> |
||||
</dependency> |
||||
<dependency> |
||||
<groupId>commons-io</groupId> |
||||
<artifactId>commons-io</artifactId> |
||||
</dependency> |
||||
</dependencies> |
||||
|
||||
</project> |
||||
@ -0,0 +1,45 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
package org.apache.dolphinscheduler.plugin.api; |
||||
|
||||
import org.apache.dolphinscheduler.plugin.model.AlertInfo; |
||||
import org.apache.dolphinscheduler.plugin.model.PluginName; |
||||
|
||||
import java.util.Map; |
||||
|
||||
/** |
||||
* Plugin |
||||
*/ |
||||
public interface AlertPlugin { |
||||
|
||||
/** |
||||
* Get alert plugin id |
||||
* |
||||
* @return alert plugin id, which should be unique |
||||
*/ |
||||
String getId(); |
||||
|
||||
/** |
||||
* Get alert plugin name, which will show in front end portal |
||||
* |
||||
* @return plugin name |
||||
*/ |
||||
PluginName getName(); |
||||
|
||||
Map<String, Object> process(AlertInfo info); |
||||
|
||||
} |
||||
@ -0,0 +1,129 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
package org.apache.dolphinscheduler.plugin.model; |
||||
|
||||
/** |
||||
* AlertData |
||||
*/ |
||||
public class AlertData { |
||||
|
||||
/** |
||||
* alert primary key |
||||
*/ |
||||
private int id; |
||||
/** |
||||
* title |
||||
*/ |
||||
private String title; |
||||
/** |
||||
* content |
||||
*/ |
||||
private String content; |
||||
/** |
||||
* log |
||||
*/ |
||||
private String log; |
||||
/** |
||||
* alertgroup_id |
||||
*/ |
||||
private int alertGroupId; |
||||
/** |
||||
* receivers |
||||
*/ |
||||
private String receivers; |
||||
/** |
||||
* show_type |
||||
*/ |
||||
private String showType; |
||||
/** |
||||
* receivers_cc |
||||
*/ |
||||
private String receiversCc; |
||||
|
||||
public int getId() { |
||||
return id; |
||||
} |
||||
|
||||
public AlertData setId(int id) { |
||||
this.id = id; |
||||
return this; |
||||
} |
||||
|
||||
public String getTitle() { |
||||
return title; |
||||
} |
||||
|
||||
public AlertData setTitle(String title) { |
||||
this.title = title; |
||||
return this; |
||||
} |
||||
|
||||
public String getContent() { |
||||
return content; |
||||
} |
||||
|
||||
public AlertData setContent(String content) { |
||||
this.content = content; |
||||
return this; |
||||
} |
||||
|
||||
public String getLog() { |
||||
return log; |
||||
} |
||||
|
||||
public AlertData setLog(String log) { |
||||
this.log = log; |
||||
return this; |
||||
} |
||||
|
||||
public int getAlertGroupId() { |
||||
return alertGroupId; |
||||
} |
||||
|
||||
public AlertData setAlertGroupId(int alertGroupId) { |
||||
this.alertGroupId = alertGroupId; |
||||
return this; |
||||
} |
||||
|
||||
public String getReceivers() { |
||||
return receivers; |
||||
} |
||||
|
||||
public AlertData setReceivers(String receivers) { |
||||
this.receivers = receivers; |
||||
return this; |
||||
} |
||||
|
||||
public String getReceiversCc() { |
||||
return receiversCc; |
||||
} |
||||
|
||||
public AlertData setReceiversCc(String receiversCc) { |
||||
this.receiversCc = receiversCc; |
||||
return this; |
||||
} |
||||
|
||||
public String getShowType() { |
||||
return showType; |
||||
} |
||||
|
||||
public AlertData setShowType(String showType) { |
||||
this.showType = showType; |
||||
return this; |
||||
} |
||||
|
||||
} |
||||
@ -0,0 +1,61 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
package org.apache.dolphinscheduler.plugin.model; |
||||
|
||||
import java.util.HashMap; |
||||
import java.util.Map; |
||||
|
||||
/** |
||||
* AlertInfo |
||||
*/ |
||||
public class AlertInfo { |
||||
|
||||
private Map<String, Object> alertProps; |
||||
|
||||
private AlertData alertData; |
||||
|
||||
public AlertInfo() { |
||||
this.alertProps = new HashMap<>(); |
||||
} |
||||
|
||||
public Map<String, Object> getAlertProps() { |
||||
return alertProps; |
||||
} |
||||
|
||||
public AlertInfo setAlertProps(Map<String, Object> alertProps) { |
||||
this.alertProps = alertProps; |
||||
return this; |
||||
} |
||||
|
||||
public AlertInfo addProp(String key, Object value) { |
||||
this.alertProps.put(key, value); |
||||
return this; |
||||
} |
||||
|
||||
public Object getProp(String key) { |
||||
return this.alertProps.get(key); |
||||
} |
||||
|
||||
public AlertData getAlertData() { |
||||
return alertData; |
||||
} |
||||
|
||||
public AlertInfo setAlertData(AlertData alertData) { |
||||
this.alertData = alertData; |
||||
return this; |
||||
} |
||||
} |
||||
@ -0,0 +1,45 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
package org.apache.dolphinscheduler.plugin.model; |
||||
|
||||
/** |
||||
* PluginName |
||||
*/ |
||||
public class PluginName { |
||||
|
||||
private String chinese; |
||||
|
||||
private String english; |
||||
|
||||
public String getChinese() { |
||||
return chinese; |
||||
} |
||||
|
||||
public PluginName setChinese(String chinese) { |
||||
this.chinese = chinese; |
||||
return this; |
||||
} |
||||
|
||||
public String getEnglish() { |
||||
return english; |
||||
} |
||||
|
||||
public PluginName setEnglish(String english) { |
||||
this.english = english; |
||||
return this; |
||||
} |
||||
} |
||||
@ -0,0 +1,33 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
package org.apache.dolphinscheduler.plugin.spi; |
||||
|
||||
import org.apache.dolphinscheduler.plugin.api.AlertPlugin; |
||||
|
||||
/** |
||||
* PluginProvider |
||||
*/ |
||||
public interface AlertPluginProvider { |
||||
|
||||
/** |
||||
* create an alert plugin |
||||
* |
||||
* @return an alert plugin |
||||
*/ |
||||
AlertPlugin createPlugin(); |
||||
|
||||
} |
||||
@ -0,0 +1,190 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
package org.apache.dolphinscheduler.plugin.utils; |
||||
|
||||
import org.apache.commons.io.IOUtils; |
||||
import org.slf4j.Logger; |
||||
import org.slf4j.LoggerFactory; |
||||
|
||||
import java.io.IOException; |
||||
import java.io.InputStream; |
||||
import java.util.Properties; |
||||
|
||||
/** |
||||
* property utils |
||||
* single instance |
||||
*/ |
||||
public class PropertyUtils { |
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(PropertyUtils.class); |
||||
|
||||
private static final Properties properties = new Properties(); |
||||
|
||||
private PropertyUtils() { |
||||
throw new IllegalStateException("PropertyUtils class"); |
||||
} |
||||
|
||||
static { |
||||
String propertyFiles = "/plugin.properties"; |
||||
InputStream fis = null; |
||||
try { |
||||
fis = PropertyUtils.class.getResourceAsStream(propertyFiles); |
||||
properties.load(fis); |
||||
} catch (IOException e) { |
||||
logger.error(e.getMessage(), e); |
||||
if (fis != null) { |
||||
IOUtils.closeQuietly(fis); |
||||
} |
||||
} finally { |
||||
IOUtils.closeQuietly(fis); |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* get property value |
||||
* |
||||
* @param key property name |
||||
* @param defaultVal default value |
||||
* @return property value |
||||
*/ |
||||
public static String getString(String key, String defaultVal) { |
||||
String val = properties.getProperty(key.trim()); |
||||
return val == null ? defaultVal : val; |
||||
} |
||||
|
||||
/** |
||||
* get property value |
||||
* |
||||
* @param key property name |
||||
* @return property value |
||||
*/ |
||||
public static String getString(String key) { |
||||
if (key == null) { |
||||
return null; |
||||
} |
||||
return properties.getProperty(key.trim()); |
||||
} |
||||
|
||||
/** |
||||
* get property value |
||||
* |
||||
* @param key property name |
||||
* @return get property int value , if key == null, then return -1 |
||||
*/ |
||||
public static int getInt(String key) { |
||||
return getInt(key, -1); |
||||
} |
||||
|
||||
/** |
||||
* get int |
||||
* |
||||
* @param key key |
||||
* @param defaultValue default value |
||||
* @return property value |
||||
*/ |
||||
public static int getInt(String key, int defaultValue) { |
||||
String value = properties.getProperty(key.trim()); |
||||
if (value == null) { |
||||
return defaultValue; |
||||
} |
||||
try { |
||||
return Integer.parseInt(value); |
||||
} catch (NumberFormatException e) { |
||||
logger.info(e.getMessage(), e); |
||||
} |
||||
return defaultValue; |
||||
} |
||||
|
||||
/** |
||||
* get property value |
||||
* |
||||
* @param key property name |
||||
* @return property value |
||||
*/ |
||||
public static boolean getBoolean(String key) { |
||||
String value = properties.getProperty(key.trim()); |
||||
if (value == null) { |
||||
return false; |
||||
} |
||||
|
||||
return Boolean.parseBoolean(value); |
||||
} |
||||
|
||||
/** |
||||
* get property value |
||||
* |
||||
* @param key property name |
||||
* @param defaultValue default value |
||||
* @return property value |
||||
*/ |
||||
public static Boolean getBoolean(String key, boolean defaultValue) { |
||||
String value = properties.getProperty(key.trim()); |
||||
if (value == null) { |
||||
return defaultValue; |
||||
} |
||||
|
||||
return Boolean.parseBoolean(value); |
||||
} |
||||
|
||||
/** |
||||
* get property long value |
||||
* |
||||
* @param key key |
||||
* @param defaultVal default value |
||||
* @return property value |
||||
*/ |
||||
public static long getLong(String key, long defaultVal) { |
||||
String val = getString(key); |
||||
return val == null ? defaultVal : Long.parseLong(val); |
||||
} |
||||
|
||||
/** |
||||
* get long |
||||
* |
||||
* @param key key |
||||
* @return property value |
||||
*/ |
||||
public static long getLong(String key) { |
||||
return getLong(key, -1); |
||||
} |
||||
|
||||
/** |
||||
* get double |
||||
* |
||||
* @param key key |
||||
* @param defaultVal default value |
||||
* @return property value |
||||
*/ |
||||
public static double getDouble(String key, double defaultVal) { |
||||
String val = properties.getProperty(key.trim()); |
||||
return val == null ? defaultVal : Double.parseDouble(val); |
||||
} |
||||
|
||||
/** |
||||
* @param key key |
||||
* @param type type |
||||
* @param defaultValue default value |
||||
* @param <T> T |
||||
* @return get enum value |
||||
*/ |
||||
public <T extends Enum<T>> T getEnum(String key, Class<T> type, |
||||
T defaultValue) { |
||||
String val = properties.getProperty(key.trim()); |
||||
return val == null ? defaultValue : Enum.valueOf(type, val); |
||||
} |
||||
|
||||
} |
||||
@ -0,0 +1,80 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
package org.apache.dolphinscheduler.plugin.model; |
||||
|
||||
import org.junit.Before; |
||||
import org.junit.Test; |
||||
|
||||
import static org.junit.Assert.*; |
||||
|
||||
public class AlertDataTest { |
||||
|
||||
private AlertData alertData; |
||||
|
||||
@Before |
||||
public void before() { |
||||
alertData = new AlertData(); |
||||
alertData.setId(1) |
||||
.setContent("content") |
||||
.setShowType("email") |
||||
.setTitle("title") |
||||
.setReceivers("receivers") |
||||
.setReceiversCc("cc") |
||||
.setLog("log") |
||||
.setAlertGroupId(1); |
||||
} |
||||
|
||||
@Test |
||||
public void getId() { |
||||
assertEquals(1, alertData.getId()); |
||||
} |
||||
|
||||
@Test |
||||
public void getTitle() { |
||||
assertEquals("title", alertData.getTitle()); |
||||
} |
||||
|
||||
@Test |
||||
public void getContent() { |
||||
assertEquals("content", alertData.getContent()); |
||||
} |
||||
|
||||
@Test |
||||
public void getLog() { |
||||
assertEquals("log", alertData.getLog()); |
||||
} |
||||
|
||||
@Test |
||||
public void getAlertGroupId() { |
||||
assertEquals(1, alertData.getAlertGroupId()); |
||||
} |
||||
|
||||
@Test |
||||
public void getReceivers() { |
||||
assertEquals("receivers", alertData.getReceivers()); |
||||
} |
||||
|
||||
@Test |
||||
public void getReceiversCc() { |
||||
assertEquals("cc", alertData.getReceiversCc()); |
||||
} |
||||
|
||||
@Test |
||||
public void getShowType() { |
||||
assertEquals("email", alertData.getShowType()); |
||||
} |
||||
} |
||||
@ -0,0 +1,54 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
package org.apache.dolphinscheduler.plugin.model; |
||||
|
||||
import org.junit.Before; |
||||
import org.junit.Test; |
||||
|
||||
import java.util.HashMap; |
||||
import java.util.Map; |
||||
|
||||
import static org.junit.Assert.*; |
||||
|
||||
public class AlertInfoTest { |
||||
|
||||
private AlertInfo alertInfo; |
||||
|
||||
@Before |
||||
public void before() { |
||||
alertInfo = new AlertInfo(); |
||||
} |
||||
|
||||
@Test |
||||
public void getAlertProps() { |
||||
Map<String, Object> map = new HashMap<>(); |
||||
alertInfo.setAlertProps(map); |
||||
assertNotNull(alertInfo.getAlertProps()); |
||||
} |
||||
|
||||
@Test |
||||
public void getProp() { |
||||
alertInfo.addProp("k", "v"); |
||||
assertEquals("v", alertInfo.getProp("k")); |
||||
} |
||||
|
||||
@Test |
||||
public void getAlertData() { |
||||
alertInfo.setAlertData(new AlertData()); |
||||
assertNotNull(alertInfo.getAlertData()); |
||||
} |
||||
} |
||||
@ -0,0 +1,83 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
package org.apache.dolphinscheduler.plugin.utils; |
||||
|
||||
import org.junit.Test; |
||||
import org.slf4j.Logger; |
||||
import org.slf4j.LoggerFactory; |
||||
|
||||
import static org.junit.Assert.*; |
||||
|
||||
public class PropertyUtilsTest { |
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(PropertyUtilsTest.class); |
||||
|
||||
/** |
||||
* Test getString |
||||
*/ |
||||
@Test |
||||
public void testGetString() { |
||||
|
||||
String result = PropertyUtils.getString("test.string"); |
||||
logger.info(result); |
||||
assertEquals("teststring", result); |
||||
|
||||
//If key is null, then return null
|
||||
result = PropertyUtils.getString(null); |
||||
assertNull(result); |
||||
} |
||||
|
||||
|
||||
/** |
||||
* Test getBoolean |
||||
*/ |
||||
@Test |
||||
public void testGetBoolean() { |
||||
|
||||
//Expected true
|
||||
Boolean result = PropertyUtils.getBoolean("test.true"); |
||||
assertTrue(result); |
||||
|
||||
//Expected false
|
||||
result = PropertyUtils.getBoolean("test.false"); |
||||
assertFalse(result); |
||||
} |
||||
|
||||
/** |
||||
* Test getLong |
||||
*/ |
||||
@Test |
||||
public void testGetLong() { |
||||
long result = PropertyUtils.getLong("test.long"); |
||||
assertSame(100L, result); |
||||
} |
||||
|
||||
/** |
||||
* Test getDouble |
||||
*/ |
||||
@Test |
||||
public void testGetDouble() { |
||||
|
||||
//If key is undefine in alert.properties, and there is a defaultval, then return defaultval
|
||||
double result = PropertyUtils.getDouble("abc", 5.0); |
||||
assertEquals(5.0, result, 0); |
||||
|
||||
result = PropertyUtils.getDouble("cba", 5.0); |
||||
assertEquals(3.1, result, 0.01); |
||||
} |
||||
|
||||
} |
||||
@ -0,0 +1,262 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.server.master.consumer; |
||||
|
||||
import org.apache.dolphinscheduler.common.enums.CommandType; |
||||
import org.apache.dolphinscheduler.common.enums.DbType; |
||||
import org.apache.dolphinscheduler.common.enums.ExecutionStatus; |
||||
import org.apache.dolphinscheduler.common.enums.Priority; |
||||
import org.apache.dolphinscheduler.dao.entity.*; |
||||
import org.apache.dolphinscheduler.server.master.dispatch.ExecutorDispatcher; |
||||
import org.apache.dolphinscheduler.server.master.dispatch.executor.NettyExecutorManager; |
||||
import org.apache.dolphinscheduler.server.registry.DependencyConfig; |
||||
import org.apache.dolphinscheduler.server.registry.ZookeeperNodeManager; |
||||
import org.apache.dolphinscheduler.server.registry.ZookeeperRegistryCenter; |
||||
import org.apache.dolphinscheduler.server.zk.SpringZKServer; |
||||
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; |
||||
import org.apache.dolphinscheduler.service.process.ProcessService; |
||||
import org.apache.dolphinscheduler.service.queue.TaskPriorityQueue; |
||||
import org.apache.dolphinscheduler.service.queue.TaskPriorityQueueImpl; |
||||
import org.apache.dolphinscheduler.service.zk.ZookeeperCachedOperator; |
||||
import org.apache.dolphinscheduler.service.zk.ZookeeperConfig; |
||||
import org.junit.Before; |
||||
import org.junit.Test; |
||||
import org.junit.runner.RunWith; |
||||
import org.mockito.Mockito; |
||||
import org.springframework.beans.factory.annotation.Autowired; |
||||
import org.springframework.test.context.ContextConfiguration; |
||||
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; |
||||
|
||||
import java.util.Date; |
||||
|
||||
|
||||
@RunWith(SpringJUnit4ClassRunner.class) |
||||
@ContextConfiguration(classes={DependencyConfig.class, SpringApplicationContext.class, SpringZKServer.class, |
||||
NettyExecutorManager.class, ExecutorDispatcher.class, ZookeeperRegistryCenter.class, TaskPriorityQueueConsumer.class, |
||||
ZookeeperNodeManager.class, ZookeeperCachedOperator.class, ZookeeperConfig.class}) |
||||
public class TaskPriorityQueueConsumerTest { |
||||
|
||||
|
||||
@Autowired |
||||
private TaskPriorityQueue taskPriorityQueue; |
||||
|
||||
@Autowired |
||||
private TaskPriorityQueueConsumer taskPriorityQueueConsumer; |
||||
|
||||
@Autowired |
||||
private ProcessService processService; |
||||
|
||||
@Autowired |
||||
private ExecutorDispatcher dispatcher; |
||||
|
||||
@Before |
||||
public void init(){ |
||||
|
||||
Tenant tenant = new Tenant(); |
||||
tenant.setId(1); |
||||
tenant.setTenantCode("journey"); |
||||
tenant.setTenantName("journey"); |
||||
tenant.setDescription("journey"); |
||||
tenant.setQueueId(1); |
||||
tenant.setCreateTime(new Date()); |
||||
tenant.setUpdateTime(new Date()); |
||||
|
||||
Mockito.when(processService.getTenantForProcess(1,2)).thenReturn(tenant); |
||||
|
||||
Mockito.when(processService.queryUserQueueByProcessInstanceId(1)).thenReturn("default"); |
||||
} |
||||
|
||||
|
||||
@Test |
||||
public void testSHELLTask() throws Exception { |
||||
TaskInstance taskInstance = new TaskInstance(); |
||||
taskInstance.setId(1); |
||||
taskInstance.setTaskType("SHELL"); |
||||
taskInstance.setProcessDefinitionId(1); |
||||
taskInstance.setProcessInstanceId(1); |
||||
taskInstance.setState(ExecutionStatus.KILL); |
||||
taskInstance.setTaskJson("{\"conditionResult\":\"{\\\"successNode\\\":[\\\"\\\"],\\\"failedNode\\\":[\\\"\\\"]}\",\"conditionsTask\":false,\"depList\":[],\"dependence\":\"{}\",\"forbidden\":false,\"id\":\"tasks-55201\",\"maxRetryTimes\":0,\"name\":\"测试任务\",\"params\":\"{\\\"rawScript\\\":\\\"echo \\\\\\\"测试任务\\\\\\\"\\\",\\\"localParams\\\":[],\\\"resourceList\\\":[]}\",\"preTasks\":\"[]\",\"retryInterval\":1,\"runFlag\":\"NORMAL\",\"taskInstancePriority\":\"MEDIUM\",\"taskTimeoutParameter\":{\"enable\":false,\"interval\":0},\"timeout\":\"{\\\"enable\\\":false,\\\"strategy\\\":\\\"\\\"}\",\"type\":\"SHELL\",\"workerGroup\":\"default\"}"); |
||||
taskInstance.setProcessInstancePriority(Priority.MEDIUM); |
||||
taskInstance.setWorkerGroup("default"); |
||||
taskInstance.setExecutorId(2); |
||||
|
||||
ProcessInstance processInstance = new ProcessInstance(); |
||||
processInstance.setTenantId(1); |
||||
processInstance.setCommandType(CommandType.START_PROCESS); |
||||
taskInstance.setProcessInstance(processInstance); |
||||
|
||||
ProcessDefinition processDefinition = new ProcessDefinition(); |
||||
processDefinition.setUserId(2); |
||||
processDefinition.setProjectId(1); |
||||
taskInstance.setProcessDefine(processDefinition); |
||||
|
||||
Mockito.when(processService.getTaskInstanceDetailByTaskId(1)).thenReturn(taskInstance); |
||||
taskPriorityQueue.put("2_1_2_1_default"); |
||||
|
||||
Thread.sleep(10000); |
||||
} |
||||
|
||||
|
||||
@Test |
||||
public void testSQLTask() throws Exception { |
||||
TaskInstance taskInstance = new TaskInstance(); |
||||
taskInstance.setId(1); |
||||
taskInstance.setTaskType("SQL"); |
||||
taskInstance.setProcessDefinitionId(1); |
||||
taskInstance.setProcessInstanceId(1); |
||||
taskInstance.setState(ExecutionStatus.KILL); |
||||
taskInstance.setTaskJson("{\"conditionsTask\":false,\"depList\":[],\"dependence\":\"{}\",\"forbidden\":false,\"id\":\"tasks-3655\",\"maxRetryTimes\":0,\"name\":\"UDF测试\",\"params\":\"{\\\"postStatements\\\":[],\\\"connParams\\\":\\\"\\\",\\\"receiversCc\\\":\\\"\\\",\\\"udfs\\\":\\\"1\\\",\\\"type\\\":\\\"HIVE\\\",\\\"title\\\":\\\"test\\\",\\\"sql\\\":\\\"select id,name,ds,zodia(ds) from t_journey_user\\\",\\\"preStatements\\\":[],\\\"sqlType\\\":0,\\\"receivers\\\":\\\"825193156@qq.com\\\",\\\"datasource\\\":3,\\\"showType\\\":\\\"TABLE\\\",\\\"localParams\\\":[]}\",\"preTasks\":\"[]\",\"retryInterval\":1,\"runFlag\":\"NORMAL\",\"taskInstancePriority\":\"MEDIUM\",\"taskTimeoutParameter\":{\"enable\":false,\"interval\":0},\"timeout\":\"{\\\"enable\\\":false,\\\"strategy\\\":\\\"\\\"}\",\"type\":\"SQL\"}"); |
||||
taskInstance.setProcessInstancePriority(Priority.MEDIUM); |
||||
taskInstance.setWorkerGroup("default"); |
||||
taskInstance.setExecutorId(2); |
||||
|
||||
ProcessInstance processInstance = new ProcessInstance(); |
||||
processInstance.setTenantId(1); |
||||
processInstance.setCommandType(CommandType.START_PROCESS); |
||||
taskInstance.setProcessInstance(processInstance); |
||||
|
||||
ProcessDefinition processDefinition = new ProcessDefinition(); |
||||
processDefinition.setUserId(2); |
||||
processDefinition.setProjectId(1); |
||||
taskInstance.setProcessDefine(processDefinition); |
||||
|
||||
Mockito.when(processService.getTaskInstanceDetailByTaskId(1)).thenReturn(taskInstance); |
||||
taskPriorityQueue.put("2_1_2_1_default"); |
||||
|
||||
DataSource dataSource = new DataSource(); |
||||
dataSource.setId(1); |
||||
dataSource.setName("sqlDatasource"); |
||||
dataSource.setType(DbType.MYSQL); |
||||
dataSource.setUserId(2); |
||||
dataSource.setConnectionParams("{\"address\":\"jdbc:mysql://192.168.221.185:3306\",\"database\":\"dolphinscheduler_qiaozhanwei\",\"jdbcUrl\":\"jdbc:mysql://192.168.221.185:3306/dolphinscheduler_qiaozhanwei\",\"user\":\"root\",\"password\":\"root@123\"}"); |
||||
dataSource.setCreateTime(new Date()); |
||||
dataSource.setUpdateTime(new Date()); |
||||
|
||||
Mockito.when(processService.findDataSourceById(1)).thenReturn(dataSource); |
||||
|
||||
Thread.sleep(10000); |
||||
} |
||||
|
||||
|
||||
@Test |
||||
public void testDataxTask() throws Exception { |
||||
TaskInstance taskInstance = new TaskInstance(); |
||||
taskInstance.setId(1); |
||||
taskInstance.setTaskType("DATAX"); |
||||
taskInstance.setProcessDefinitionId(1); |
||||
taskInstance.setProcessInstanceId(1); |
||||
taskInstance.setState(ExecutionStatus.KILL); |
||||
taskInstance.setTaskJson("{\"conditionResult\":\"{\\\"successNode\\\":[\\\"\\\"],\\\"failedNode\\\":[\\\"\\\"]}\",\"conditionsTask\":false,\"depList\":[],\"dependence\":\"{}\",\"forbidden\":false,\"id\":\"tasks-97625\",\"maxRetryTimes\":0,\"name\":\"MySQL数据相互导入\",\"params\":\"{\\\"targetTable\\\":\\\"pv2\\\",\\\"postStatements\\\":[],\\\"jobSpeedRecord\\\":1000,\\\"customConfig\\\":0,\\\"dtType\\\":\\\"MYSQL\\\",\\\"dsType\\\":\\\"MYSQL\\\",\\\"jobSpeedByte\\\":0,\\\"dataSource\\\":80,\\\"dataTarget\\\":80,\\\"sql\\\":\\\"SELECT dt,count FROM pv\\\",\\\"preStatements\\\":[]}\",\"preTasks\":\"[]\",\"retryInterval\":1,\"runFlag\":\"NORMAL\",\"taskInstancePriority\":\"MEDIUM\",\"taskTimeoutParameter\":{\"enable\":false,\"interval\":0},\"timeout\":\"{\\\"enable\\\":false,\\\"strategy\\\":\\\"\\\"}\",\"type\":\"DATAX\",\"workerGroup\":\"default\"}"); |
||||
taskInstance.setProcessInstancePriority(Priority.MEDIUM); |
||||
taskInstance.setWorkerGroup("default"); |
||||
taskInstance.setExecutorId(2); |
||||
|
||||
ProcessInstance processInstance = new ProcessInstance(); |
||||
processInstance.setTenantId(1); |
||||
processInstance.setCommandType(CommandType.START_PROCESS); |
||||
taskInstance.setProcessInstance(processInstance); |
||||
|
||||
ProcessDefinition processDefinition = new ProcessDefinition(); |
||||
processDefinition.setUserId(2); |
||||
processDefinition.setProjectId(1); |
||||
taskInstance.setProcessDefine(processDefinition); |
||||
|
||||
Mockito.when(processService.getTaskInstanceDetailByTaskId(1)).thenReturn(taskInstance); |
||||
taskPriorityQueue.put("2_1_2_1_default"); |
||||
|
||||
|
||||
|
||||
DataSource dataSource = new DataSource(); |
||||
dataSource.setId(80); |
||||
dataSource.setName("datax"); |
||||
dataSource.setType(DbType.MYSQL); |
||||
dataSource.setUserId(2); |
||||
dataSource.setConnectionParams("{\"address\":\"jdbc:mysql://192.168.221.185:3306\",\"database\":\"dolphinscheduler_qiaozhanwei\",\"jdbcUrl\":\"jdbc:mysql://192.168.221.185:3306/dolphinscheduler_qiaozhanwei\",\"user\":\"root\",\"password\":\"root@123\"}"); |
||||
dataSource.setCreateTime(new Date()); |
||||
dataSource.setUpdateTime(new Date()); |
||||
|
||||
Mockito.when(processService.findDataSourceById(80)).thenReturn(dataSource); |
||||
|
||||
Thread.sleep(10000); |
||||
} |
||||
|
||||
|
||||
@Test |
||||
public void testSqoopTask() throws Exception { |
||||
TaskInstance taskInstance = new TaskInstance(); |
||||
taskInstance.setId(1); |
||||
taskInstance.setTaskType("SQOOP"); |
||||
taskInstance.setProcessDefinitionId(1); |
||||
taskInstance.setProcessInstanceId(1); |
||||
taskInstance.setState(ExecutionStatus.KILL); |
||||
taskInstance.setTaskJson("{\"conditionResult\":\"{\\\"successNode\\\":[\\\"\\\"],\\\"failedNode\\\":[\\\"\\\"]}\",\"conditionsTask\":false,\"depList\":[],\"dependence\":\"{}\",\"forbidden\":false,\"id\":\"tasks-63634\",\"maxRetryTimes\":0,\"name\":\"MySQL数据导入HDSF\",\"params\":\"{\\\"sourceType\\\":\\\"MYSQL\\\",\\\"targetType\\\":\\\"HDFS\\\",\\\"targetParams\\\":\\\"{\\\\\\\"targetPath\\\\\\\":\\\\\\\"/test/datatest\\\\\\\",\\\\\\\"deleteTargetDir\\\\\\\":true,\\\\\\\"fileType\\\\\\\":\\\\\\\"--as-textfile\\\\\\\",\\\\\\\"compressionCodec\\\\\\\":\\\\\\\"\\\\\\\",\\\\\\\"fieldsTerminated\\\\\\\":\\\\\\\",\\\\\\\",\\\\\\\"linesTerminated\\\\\\\":\\\\\\\"\\\\\\\\\\\\\\\\n\\\\\\\"}\\\",\\\"modelType\\\":\\\"import\\\",\\\"sourceParams\\\":\\\"{\\\\\\\"srcType\\\\\\\":\\\\\\\"MYSQL\\\\\\\",\\\\\\\"srcDatasource\\\\\\\":1,\\\\\\\"srcTable\\\\\\\":\\\\\\\"t_ds_user\\\\\\\",\\\\\\\"srcQueryType\\\\\\\":\\\\\\\"0\\\\\\\",\\\\\\\"srcQuerySql\\\\\\\":\\\\\\\"\\\\\\\",\\\\\\\"srcColumnType\\\\\\\":\\\\\\\"0\\\\\\\",\\\\\\\"srcColumns\\\\\\\":\\\\\\\"\\\\\\\",\\\\\\\"srcConditionList\\\\\\\":[],\\\\\\\"mapColumnHive\\\\\\\":[],\\\\\\\"mapColumnJava\\\\\\\":[]}\\\",\\\"localParams\\\":[],\\\"concurrency\\\":1}\",\"preTasks\":\"[]\",\"retryInterval\":1,\"runFlag\":\"NORMAL\",\"taskInstancePriority\":\"MEDIUM\",\"taskTimeoutParameter\":{\"enable\":false,\"interval\":0},\"timeout\":\"{\\\"enable\\\":false,\\\"strategy\\\":\\\"\\\"}\",\"type\":\"SQOOP\",\"workerGroup\":\"default\"}"); |
||||
taskInstance.setProcessInstancePriority(Priority.MEDIUM); |
||||
taskInstance.setWorkerGroup("default"); |
||||
taskInstance.setExecutorId(2); |
||||
|
||||
ProcessInstance processInstance = new ProcessInstance(); |
||||
processInstance.setTenantId(1); |
||||
processInstance.setCommandType(CommandType.START_PROCESS); |
||||
taskInstance.setProcessInstance(processInstance); |
||||
|
||||
ProcessDefinition processDefinition = new ProcessDefinition(); |
||||
processDefinition.setUserId(2); |
||||
processDefinition.setProjectId(1); |
||||
taskInstance.setProcessDefine(processDefinition); |
||||
|
||||
Mockito.when(processService.getTaskInstanceDetailByTaskId(1)).thenReturn(taskInstance); |
||||
taskPriorityQueue.put("2_1_2_1_default"); |
||||
|
||||
|
||||
|
||||
DataSource dataSource = new DataSource(); |
||||
dataSource.setId(1); |
||||
dataSource.setName("datax"); |
||||
dataSource.setType(DbType.MYSQL); |
||||
dataSource.setUserId(2); |
||||
dataSource.setConnectionParams("{\"address\":\"jdbc:mysql://192.168.221.185:3306\",\"database\":\"dolphinscheduler_qiaozhanwei\",\"jdbcUrl\":\"jdbc:mysql://192.168.221.185:3306/dolphinscheduler_qiaozhanwei\",\"user\":\"root\",\"password\":\"root@123\"}"); |
||||
dataSource.setCreateTime(new Date()); |
||||
dataSource.setUpdateTime(new Date()); |
||||
|
||||
Mockito.when(processService.findDataSourceById(1)).thenReturn(dataSource); |
||||
|
||||
Thread.sleep(10000); |
||||
} |
||||
|
||||
|
||||
@Test |
||||
public void testTaskInstanceIsFinalState(){ |
||||
TaskInstance taskInstance = new TaskInstance(); |
||||
taskInstance.setId(1); |
||||
taskInstance.setTaskType("SHELL"); |
||||
taskInstance.setProcessDefinitionId(1); |
||||
taskInstance.setProcessInstanceId(1); |
||||
taskInstance.setState(ExecutionStatus.KILL); |
||||
taskInstance.setTaskJson("{\"conditionResult\":\"{\\\"successNode\\\":[\\\"\\\"],\\\"failedNode\\\":[\\\"\\\"]}\",\"conditionsTask\":false,\"depList\":[],\"dependence\":\"{}\",\"forbidden\":false,\"id\":\"tasks-55201\",\"maxRetryTimes\":0,\"name\":\"测试任务\",\"params\":\"{\\\"rawScript\\\":\\\"echo \\\\\\\"测试任务\\\\\\\"\\\",\\\"localParams\\\":[],\\\"resourceList\\\":[]}\",\"preTasks\":\"[]\",\"retryInterval\":1,\"runFlag\":\"NORMAL\",\"taskInstancePriority\":\"MEDIUM\",\"taskTimeoutParameter\":{\"enable\":false,\"interval\":0},\"timeout\":\"{\\\"enable\\\":false,\\\"strategy\\\":\\\"\\\"}\",\"type\":\"SHELL\",\"workerGroup\":\"default\"}"); |
||||
taskInstance.setProcessInstancePriority(Priority.MEDIUM); |
||||
taskInstance.setWorkerGroup("default"); |
||||
taskInstance.setExecutorId(2); |
||||
|
||||
|
||||
Mockito.when( processService.findTaskInstanceById(1)).thenReturn(taskInstance); |
||||
|
||||
taskPriorityQueueConsumer.taskInstanceIsFinalState(1); |
||||
} |
||||
|
||||
|
||||
} |
||||
@ -0,0 +1,80 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.server.master.runner; |
||||
|
||||
import junit.framework.Assert; |
||||
import org.apache.dolphinscheduler.server.master.config.MasterConfig; |
||||
import org.apache.dolphinscheduler.server.master.consumer.TaskPriorityQueueConsumer; |
||||
import org.apache.dolphinscheduler.server.master.dispatch.ExecutorDispatcher; |
||||
import org.apache.dolphinscheduler.server.master.dispatch.executor.NettyExecutorManager; |
||||
import org.apache.dolphinscheduler.server.registry.DependencyConfig; |
||||
import org.apache.dolphinscheduler.server.registry.ZookeeperNodeManager; |
||||
import org.apache.dolphinscheduler.server.registry.ZookeeperRegistryCenter; |
||||
import org.apache.dolphinscheduler.server.zk.SpringZKServer; |
||||
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; |
||||
import org.apache.dolphinscheduler.service.queue.TaskPriorityQueueImpl; |
||||
import org.apache.dolphinscheduler.service.zk.ZookeeperCachedOperator; |
||||
import org.apache.dolphinscheduler.service.zk.ZookeeperConfig; |
||||
import org.junit.Test; |
||||
import org.junit.runner.RunWith; |
||||
import org.mockito.Mockito; |
||||
import org.springframework.test.context.ContextConfiguration; |
||||
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; |
||||
|
||||
import java.util.HashSet; |
||||
import java.util.Set; |
||||
|
||||
@RunWith(SpringJUnit4ClassRunner.class) |
||||
@ContextConfiguration(classes={DependencyConfig.class, SpringApplicationContext.class, SpringZKServer.class, |
||||
NettyExecutorManager.class, ExecutorDispatcher.class, ZookeeperRegistryCenter.class, TaskPriorityQueueConsumer.class, |
||||
ZookeeperNodeManager.class, ZookeeperCachedOperator.class, ZookeeperConfig.class, MasterConfig.class}) |
||||
public class MasterTaskExecThreadTest { |
||||
|
||||
@Test |
||||
public void testExistsValidWorkerGroup1(){ |
||||
ZookeeperRegistryCenter zookeeperRegistryCenter = Mockito.mock(ZookeeperRegistryCenter.class); |
||||
Mockito.when(zookeeperRegistryCenter.getWorkerGroupDirectly()).thenReturn(null); |
||||
MasterTaskExecThread masterTaskExecThread = new MasterTaskExecThread(null); |
||||
masterTaskExecThread.existsValidWorkerGroup("default"); |
||||
} |
||||
@Test |
||||
public void testExistsValidWorkerGroup2(){ |
||||
ZookeeperRegistryCenter zookeeperRegistryCenter = Mockito.mock(ZookeeperRegistryCenter.class); |
||||
Set<String> workerGorups = new HashSet<>(); |
||||
workerGorups.add("test1"); |
||||
workerGorups.add("test2"); |
||||
|
||||
Mockito.when(zookeeperRegistryCenter.getWorkerGroupDirectly()).thenReturn(workerGorups); |
||||
MasterTaskExecThread masterTaskExecThread = new MasterTaskExecThread(null); |
||||
masterTaskExecThread.existsValidWorkerGroup("default"); |
||||
} |
||||
|
||||
@Test |
||||
public void testExistsValidWorkerGroup3(){ |
||||
ZookeeperRegistryCenter zookeeperRegistryCenter = Mockito.mock(ZookeeperRegistryCenter.class); |
||||
Set<String> workerGorups = new HashSet<>(); |
||||
workerGorups.add("test1"); |
||||
|
||||
Mockito.when(zookeeperRegistryCenter.getWorkerGroupDirectly()).thenReturn(workerGorups); |
||||
Mockito.when(zookeeperRegistryCenter.getWorkerGroupNodesDirectly("test1")).thenReturn(workerGorups); |
||||
MasterTaskExecThread masterTaskExecThread = new MasterTaskExecThread(null); |
||||
masterTaskExecThread.existsValidWorkerGroup("test1"); |
||||
} |
||||
|
||||
|
||||
} |
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue