diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/configuration/dolphin-common.xml b/ambari_plugin/common-services/DOLPHIN/1.2.1/configuration/dolphin-common.xml index 41e2836e37..7d4fb8329b 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.2.1/configuration/dolphin-common.xml +++ b/ambari_plugin/common-services/DOLPHIN/1.2.1/configuration/dolphin-common.xml @@ -43,7 +43,7 @@ zookeeper.connection.timeout - 300 + 30000 int @@ -73,7 +73,7 @@ zookeeper.retry.maxtime - 5 + 10 int diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/alerts.json b/ambari_plugin/common-services/DOLPHIN/2.0.0/alerts.json new file mode 100644 index 0000000000..385c5d5599 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/2.0.0/alerts.json @@ -0,0 +1,164 @@ +{ + "DOLPHIN": { + "service": [], + "DOLPHIN_API": [ + { + "name": "dolphin_api_port_check", + "label": "dolphin_api_port_check", + "description": "dolphin_api_port_check.", + "interval": 10, + "scope": "ANY", + "source": { + "type": "PORT", + "uri": "{{dolphin-application-api/server.port}}", + "default_port": 12345, + "reporting": { + "ok": { + "text": "TCP OK - {0:.3f}s response on port {1}" + }, + "warning": { + "text": "TCP OK - {0:.3f}s response on port {1}", + "value": 1.5 + }, + "critical": { + "text": "Connection failed: {0} to {1}:{2}", + "value": 5.0 + } + } + } + } + ], + "DOLPHIN_MASTER": [ + { + "name": "DOLPHIN_MASTER_CHECK", + "label": "check dolphin scheduler master status", + "description": "", + "interval":10, + "scope": "HOST", + "enabled": true, + "source": { + "type": "SCRIPT", + "path": "DOLPHIN/2.0.0/package/alerts/alert_dolphin_scheduler_status.py", + "parameters": [ + + { + "name": "connection.timeout", + "display_name": "Connection Timeout", + "value": 5.0, + "type": "NUMERIC", + "description": "The maximum time before this alert is considered to be CRITICAL", + "units": "seconds", + "threshold": "CRITICAL" + }, + { + "name": "alertName", + "display_name": "alertName", + "value": "DOLPHIN_MASTER", + "type": "STRING", + "description": "alert name" + } + ] + } + } + ], + "DOLPHIN_WORKER": [ + { + "name": "DOLPHIN_WORKER_CHECK", + "label": "check dolphin scheduler worker status", + "description": "", + "interval":10, + "scope": "HOST", + "enabled": true, + "source": { + "type": "SCRIPT", + "path": "DOLPHIN/2.0.0/package/alerts/alert_dolphin_scheduler_status.py", + "parameters": [ + + { + "name": "connection.timeout", + "display_name": "Connection Timeout", + "value": 5.0, + "type": "NUMERIC", + "description": "The maximum time before this alert is considered to be CRITICAL", + "units": "seconds", + "threshold": "CRITICAL" + }, + { + "name": "alertName", + "display_name": "alertName", + "value": "DOLPHIN_WORKER", + "type": "STRING", + "description": "alert name" + } + ] + } + } + ], + "DOLPHIN_ALERT": [ + { + "name": "DOLPHIN_DOLPHIN_ALERT_CHECK", + "label": "check dolphin scheduler alert status", + "description": "", + "interval":10, + "scope": "HOST", + "enabled": true, + "source": { + "type": "SCRIPT", + "path": "DOLPHIN/2.0.0/package/alerts/alert_dolphin_scheduler_status.py", + "parameters": [ + + { + "name": "connection.timeout", + "display_name": "Connection Timeout", + "value": 5.0, + "type": "NUMERIC", + "description": "The maximum time before this alert is considered to be CRITICAL", + "units": "seconds", + "threshold": "CRITICAL" + }, + { + "name": "alertName", + "display_name": "alertName", + "value": "DOLPHIN_ALERT", + "type": "STRING", + "description": "alert name" + } + ] + } + } + ], + "DOLPHIN_ALERT": [ + { + "name": "DOLPHIN_DOLPHIN_LOGGER_CHECK", + "label": "check dolphin scheduler alert status", + "description": "", + "interval":10, + "scope": "HOST", + "enabled": true, + "source": { + "type": "SCRIPT", + "path": "DOLPHIN/2.0.0/package/alerts/alert_dolphin_scheduler_status.py", + "parameters": [ + + { + "name": "connection.timeout", + "display_name": "Connection Timeout", + "value": 5.0, + "type": "NUMERIC", + "description": "The maximum time before this alert is considered to be CRITICAL", + "units": "seconds", + "threshold": "CRITICAL" + }, + { + "name": "alertName", + "display_name": "alertName", + "value": "DOLPHIN_LOGGER", + "type": "STRING", + "description": "alert name" + } + ] + } + } + ] + } +} \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-alert.xml b/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-alert.xml new file mode 100644 index 0000000000..5f44a1a4c8 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-alert.xml @@ -0,0 +1,143 @@ + + + + alert.type + EMAIL + alert type is EMAIL/SMS + + + + alert.template + html + alter msg template, default is html template + + + + mail.protocol + SMTP + + + + + mail.server.host + xxx.xxx.com + + + + + mail.server.port + 25 + + int + + + + + + mail.sender + admin + + + + + mail.user + admin + + + + + mail.passwd + 000000 + + PASSWORD + + password + + + + + + mail.smtp.starttls.enable + true + + boolean + + + + + + mail.smtp.ssl.enable + true + + boolean + + + + + + mail.smtp.ssl.trust + xxx.xxx.com + + + + + + enterprise.wechat.enable + false + + + value-list + + + true + + + + false + + + + 1 + + + + + enterprise.wechat.corp.id + wechatId + + + + + enterprise.wechat.secret + secret + + + + + enterprise.wechat.agent.id + agentId + + + + + enterprise.wechat.users + wechatUsers + + + + \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-application-api.xml b/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-application-api.xml new file mode 100644 index 0000000000..766c0f477d --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-application-api.xml @@ -0,0 +1,87 @@ + + + + server.port + 12345 + + server port + + + int + + + + server.servlet.session.timeout + 7200 + + int + + + + + + server.servlet.context-path + /dolphinscheduler/ + + + + + spring.servlet.multipart.max-file-size + 1024 + + MB + int + + + + + + spring.servlet.multipart.max-request-size + 1024 + + MB + int + + + + + + server.jetty.max-http-post-size + 5000000 + + int + + + + + + spring.messages.encoding + UTF-8 + + + + spring.messages.basename + i18n/messages + + + + security.authentication.type + PASSWORD + + + \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-common.xml b/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-common.xml new file mode 100644 index 0000000000..439e21188a --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-common.xml @@ -0,0 +1,158 @@ + + + + resource.storage.type + Choose Resource Upload Startup Type + + Resource upload startup type : HDFS,S3,NONE + + NONE + + value-list + + + HDFS + + + + S3 + + + + NONE + + + + 1 + + + + + resource.upload.path + /dolphinscheduler + + resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended + + + + + data.basedir.path + /tmp/dolphinscheduler + + user data local directory path, please make sure the directory exists and have read write permissions + + + + + + hadoop.security.authentication.startup.state + false + + value-list + + + true + + + + false + + + + 1 + + whether kerberos starts + + + java.security.krb5.conf.path + /opt/krb5.conf + + java.security.krb5.conf path + + + + + login.user.keytab.username + hdfs-mycluster@ESZ.COM + + LoginUserFromKeytab user + + + + + login.user.keytab.path + /opt/hdfs.headless.keytab + + LoginUserFromKeytab path + + + + + resource.view.suffixs + txt,log,sh,conf,cfg,py,java,sql,hql,xml,properties + + + + hdfs.root.user + hdfs + + Users who have permission to create directories under the HDFS root path + + + + + fs.defaultFS + hdfs://mycluster:8020 + + HA or single namenode, + If namenode ha needs to copy core-site.xml and hdfs-site.xml to the conf directory， + support s3，for example : s3a://dolphinscheduler + + + + + fs.s3a.endpoint + http://host:9010 + + s3 need，s3 endpoint + + + + + fs.s3a.access.key + A3DXS30FO22544RE + + s3 need，s3 access key + + + + + fs.s3a.secret.key + OloCLq3n+8+sdPHUhJ21XrSxTC+JK + + s3 need，s3 secret key + + + + + kerberos.expire.time + 7 + + + + \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-datasource.xml b/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-datasource.xml new file mode 100644 index 0000000000..6e50a1b649 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-datasource.xml @@ -0,0 +1,467 @@ + + + + spring.datasource.initialSize + 5 + + Init connection number + + + int + + + + + spring.datasource.minIdle + 5 + + Min connection number + + + int + + + + + spring.datasource.maxActive + 50 + + Max connection number + + + int + + + + + spring.datasource.maxWait + 60000 + + Max wait time for get a connection in milliseconds. + If configuring maxWait, fair locks are enabled by default and concurrency efficiency decreases. + If necessary, unfair locks can be used by configuring the useUnfairLock attribute to true. + + + int + + + + + spring.datasource.timeBetweenEvictionRunsMillis + 60000 + + Milliseconds for check to close free connections + + + int + + + + + spring.datasource.timeBetweenConnectErrorMillis + 60000 + + The Destroy thread detects the connection interval and closes the physical connection in milliseconds + if the connection idle time is greater than or equal to minEvictableIdleTimeMillis. + + + int + + + + + spring.datasource.minEvictableIdleTimeMillis + 300000 + + The longest time a connection remains idle without being evicted, in milliseconds + + + int + + + + + spring.datasource.validationQuery + SELECT 1 + + The SQL used to check whether the connection is valid requires a query statement. + If validation Query is null, testOnBorrow, testOnReturn, and testWhileIdle will not work. + + + + + spring.datasource.validationQueryTimeout + 3 + + int + + + Check whether the connection is valid for timeout, in seconds + + + + + spring.datasource.testWhileIdle + true + + boolean + + + When applying for a connection, + if it is detected that the connection is idle longer than time Between Eviction Runs Millis, + validation Query is performed to check whether the connection is valid + + + + + spring.datasource.testOnBorrow + true + + boolean + + + Execute validation to check if the connection is valid when applying for a connection + + + + + spring.datasource.testOnReturn + false + + boolean + + + Execute validation to check if the connection is valid when the connection is returned + + + + + spring.datasource.defaultAutoCommit + true + + boolean + + + + + + + spring.datasource.keepAlive + false + + boolean + + + + + + + + spring.datasource.poolPreparedStatements + true + + boolean + + + Open PSCache, specify count PSCache for every connection + + + + + spring.datasource.maxPoolPreparedStatementPerConnectionSize + 20 + + int + + + + + + spring.datasource.spring.datasource.filters + stat,wall,log4j + + + + + spring.datasource.connectionProperties + druid.stat.mergeSql=true;druid.stat.slowSqlMillis=5000 + + + + + + mybatis-plus.mapper-locations + classpath*:/org.apache.dolphinscheduler.dao.mapper/*.xml + + + + + mybatis-plus.typeEnumsPackage + org.apache.dolphinscheduler.*.enums + + + + + mybatis-plus.typeAliasesPackage + org.apache.dolphinscheduler.dao.entity + + Entity scan, where multiple packages are separated by a comma or semicolon + + + + + mybatis-plus.global-config.db-config.id-type + AUTO + + value-list + + + AUTO + + + + INPUT + + + + ID_WORKER + + + + UUID + + + + 1 + + + Primary key type AUTO:" database ID AUTO ", + INPUT:" user INPUT ID", + ID_WORKER:" global unique ID (numeric type unique ID)", + UUID:" global unique ID UUID"; + + + + + mybatis-plus.global-config.db-config.field-strategy + NOT_NULL + + value-list + + + IGNORED + + + + NOT_NULL + + + + NOT_EMPTY + + + + 1 + + + Field policy IGNORED:" ignore judgment ", + NOT_NULL:" not NULL judgment "), + NOT_EMPTY:" not NULL judgment" + + + + + mybatis-plus.global-config.db-config.column-underline + true + + boolean + + + + + + mybatis-plus.global-config.db-config.logic-delete-value + 1 + + int + + + + + + mybatis-plus.global-config.db-config.logic-not-delete-value + 0 + + int + + + + + + mybatis-plus.global-config.db-config.banner + true + + boolean + + + + + + + mybatis-plus.configuration.map-underscore-to-camel-case + true + + boolean + + + + + + mybatis-plus.configuration.cache-enabled + false + + boolean + + + + + + mybatis-plus.configuration.call-setters-on-nulls + true + + boolean + + + + + + mybatis-plus.configuration.jdbc-type-for-null + null + + + + + master.exec.threads + 100 + + int + + + + + + master.exec.task.num + 20 + + int + + + + + + master.heartbeat.interval + 10 + + int + + + + + + master.task.commit.retryTimes + 5 + + int + + + + + + master.task.commit.interval + 1000 + + int + + + + + + master.max.cpuload.avg + 100 + + int + + + + + + master.reserved.memory + 0.1 + + float + + + + + + worker.exec.threads + 100 + + int + + + + + + worker.heartbeat.interval + 10 + + int + + + + + + worker.fetch.task.num + 3 + + int + + + + + + worker.max.cpuload.avg + 100 + + int + + + + + + worker.reserved.memory + 0.1 + + float + + + + + + \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-env.xml b/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-env.xml new file mode 100644 index 0000000000..8e14716d05 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-env.xml @@ -0,0 +1,123 @@ + + + + dolphin.database.type + mysql + Dolphin Scheduler DataBase Type Which Is Select + Dolphin Database Type + + value-list + + + mysql + + + + postgresql + + + + 1 + + + + + + dolphin.database.host + + Dolphin Database Host + + + + + dolphin.database.port + + Dolphin Database Port + + + + + dolphin.database.username + + Dolphin Database Username + + + + + dolphin.database.password + + Dolphin Database Password + PASSWORD + + password + + + + + + dolphin.user + + Which user to install and admin dolphin scheduler + Deploy User + + + + dolphin.group + + Which user to install and admin dolphin scheduler + Deploy Group + + + + + dolphinscheduler-env-content + Dolphinscheduler Env template + This is the jinja template for dolphinscheduler.env.sh file + # +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +export HADOOP_HOME=/opt/soft/hadoop +export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop +export SPARK_HOME1=/opt/soft/spark1 +export SPARK_HOME2=/opt/soft/spark2 +export PYTHON_HOME=/opt/soft/python +export JAVA_HOME=/opt/soft/java +export HIVE_HOME=/opt/soft/hive +export FLINK_HOME=/opt/soft/flink + + content + false + false + + + + \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-master.xml b/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-master.xml new file mode 100644 index 0000000000..c8eec047fc --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-master.xml @@ -0,0 +1,88 @@ + + + + master.exec.threads + 100 + + int + + master execute thread num + + + + master.exec.task.num + 20 + + int + + master execute task number in parallel + + + + master.heartbeat.interval + 10 + + int + + master heartbeat interval + + + + master.task.commit.retryTimes + 5 + + int + + master commit task retry times + + + + master.task.commit.interval + 1000 + + int + + master commit task interval + + + + master.max.cpuload.avg + 100 + + int + + only less than cpu avg load, master server can work. default value : the number of cpu cores * 2 + + + + master.reserved.memory + 0.3 + only larger than reserved memory, master server can work. default value : physical memory * 1/10, unit is G. + + + + + master.listen.port + 5678 + + int + + master listen port + + + \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-quartz.xml b/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-quartz.xml new file mode 100644 index 0000000000..7a0c68b051 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-quartz.xml @@ -0,0 +1,126 @@ + + + + org.quartz.scheduler.instanceName + DolphinScheduler + + + + + org.quartz.scheduler.instanceId + AUTO + + + + org.quartz.scheduler.makeSchedulerThreadDaemon + true + + boolean + + + + + org.quartz.jobStore.useProperties + false + + boolean + + + + + org.quartz.threadPool.class + org.quartz.simpl.SimpleThreadPool + + + + org.quartz.threadPool.makeThreadsDaemons + true + + boolean + + + + + org.quartz.threadPool.threadCount + 25 + + int + + + + + org.quartz.threadPool.threadPriority + 5 + + int + + + + + org.quartz.jobStore.class + org.quartz.impl.jdbcjobstore.JobStoreTX + + + + org.quartz.jobStore.tablePrefix + QRTZ_ + + + + org.quartz.jobStore.isClustered + true + + boolean + + + + + org.quartz.jobStore.misfireThreshold + 60000 + + int + + + + + org.quartz.jobStore.clusterCheckinInterval + 5000 + + int + + + + + org.quartz.jobStore.acquireTriggersWithinLock + true + + boolean + + + + + org.quartz.jobStore.dataSource + myDs + + + + org.quartz.dataSource.myDs.connectionProvider.class + org.apache.dolphinscheduler.service.quartz.DruidConnectionProvider + + + \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-worker.xml b/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-worker.xml new file mode 100644 index 0000000000..97beade1bc --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-worker.xml @@ -0,0 +1,76 @@ + + + + worker.exec.threads + 100 + + int + + worker execute thread num + + + + worker.heartbeat.interval + 10 + + int + + worker heartbeat interval + + + + worker.fetch.task.num + 3 + + int + + submit the number of tasks at a time + + + + worker.max.cpuload.avg + 100 + + int + + only less than cpu avg load, worker server can work. default value : the number of cpu cores * 2 + + + + worker.reserved.memory + 0.3 + only larger than reserved memory, worker server can work. default value : physical memory * 1/10, unit is G. + + + + + worker.listen.port + 1234 + + int + + worker listen port + + + + worker.group + default + default worker group + + + \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-zookeeper.xml b/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-zookeeper.xml new file mode 100644 index 0000000000..5882162254 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-zookeeper.xml @@ -0,0 +1,84 @@ + + + + dolphinscheduler.queue.impl + zookeeper + + Task queue implementation, default "zookeeper" + + + + + zookeeper.dolphinscheduler.root + /dolphinscheduler + + dolphinscheduler root directory + + + + + zookeeper.session.timeout + 300 + + int + + + + + + + zookeeper.connection.timeout + 300 + + int + + + + + + + zookeeper.retry.base.sleep + 100 + + int + + + + + + + zookeeper.retry.max.sleep + 30000 + + int + + + + + + + zookeeper.retry.maxtime + 5 + + int + + + + + + \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/metainfo.xml b/ambari_plugin/common-services/DOLPHIN/2.0.0/metainfo.xml new file mode 100644 index 0000000000..b3c14e33cb --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/2.0.0/metainfo.xml @@ -0,0 +1,137 @@ + + + + 2.0 + + + DOLPHIN + Dolphin Scheduler + 分布式易扩展的可视化DAG工作流任务调度系统 + 2.0.0 + + + DOLPHIN_MASTER + DS Master + MASTER + 1+ + + + PYTHON + 600 + + + + + DOLPHIN_LOGGER + DS Logger + SLAVE + 1+ + + + PYTHON + 600 + + + + + DOLPHIN_WORKER + DS Worker + SLAVE + 1+ + + + DOLPHIN/DOLPHIN_LOGGER + host + + true + + + + + + PYTHON + 600 + + + + + DOLPHIN_ALERT + DS Alert + SLAVE + 1 + + + PYTHON + 600 + + + + + DOLPHIN_API + DS_Api + SLAVE + 1 + + + PYTHON + 600 + + + + + + ZOOKEEPER + + + + + any + + + apache-dolphinscheduler-incubating-1.2.1* + + + + + + + dolphin-alert + dolphin-app-api + dolphin-app-dao + dolphin-common + dolphin-env + dolphin-quartz + + + + + theme.json + true + + + + quicklinks + + + quicklinks.json + true + + + + + diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/alerts/alert_dolphin_scheduler_status.py b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/alerts/alert_dolphin_scheduler_status.py new file mode 100644 index 0000000000..87cc7b453b --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/alerts/alert_dolphin_scheduler_status.py @@ -0,0 +1,124 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import socket +import urllib2 +import os +import logging +import ambari_simplejson as json +from resource_management.libraries.script.script import Script +import sys +reload(sys) +sys.setdefaultencoding('utf-8') + +logger = logging.getLogger('ambari_alerts') + +config = Script.get_config() + + +def get_tokens(): + """ + Returns a tuple of tokens in the format {{site/property}} that will be used + to build the dictionary passed into execute + + :rtype tuple + """ + +def get_info(url, connection_timeout): + response = None + + try: + response = urllib2.urlopen(url, timeout=connection_timeout) + json_data = response.read() + return json_data + finally: + if response is not None: + try: + response.close() + except: + pass + + +def execute(configurations={}, parameters={}, host_name=None): + """ + Returns a tuple containing the result code and a pre-formatted result label + + Keyword arguments: + configurations : a mapping of configuration key to value + parameters : a mapping of script parameter key to value + host_name : the name of this host where the alert is running + + :type configurations dict + :type parameters dict + :type host_name str + """ + + alert_name = parameters['alertName'] + + dolphin_pidfile_dir = "/opt/soft/run/dolphinscheduler" + + pid = "0" + + + from resource_management.core import sudo + + is_running = True + pid_file_path = "" + if alert_name == 'DOLPHIN_MASTER': + pid_file_path = dolphin_pidfile_dir + "/master-server.pid" + elif alert_name == 'DOLPHIN_WORKER': + pid_file_path = dolphin_pidfile_dir + "/worker-server.pid" + elif alert_name == 'DOLPHIN_ALERT': + pid_file_path = dolphin_pidfile_dir + "/alert-server.pid" + elif alert_name == 'DOLPHIN_LOGGER': + pid_file_path = dolphin_pidfile_dir + "/logger-server.pid" + elif alert_name == 'DOLPHIN_API': + pid_file_path = dolphin_pidfile_dir + "/api-server.pid" + + if not pid_file_path or not os.path.isfile(pid_file_path): + is_running = False + + try: + pid = int(sudo.read_file(pid_file_path)) + except: + is_running = False + + try: + # Kill will not actually kill the process + # From the doc: + # If sig is 0, then no signal is sent, but error checking is still + # performed; this can be used to check for the existence of a + # process ID or process group ID. + sudo.kill(pid, 0) + except OSError: + is_running = False + + if host_name is None: + host_name = socket.getfqdn() + + if not is_running: + result_code = "CRITICAL" + else: + result_code = "OK" + + label = "The comment {0} of DOLPHIN_SCHEDULER on {1} is {2}".format(alert_name, host_name, result_code) + + return ((result_code, [label])) + +if __name__ == "__main__": + pass diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_alert_service.py b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_alert_service.py new file mode 100644 index 0000000000..62255a3432 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_alert_service.py @@ -0,0 +1,61 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import time +from resource_management import * + +from dolphin_env import dolphin_env + + +class DolphinAlertService(Script): + def install(self, env): + import params + env.set_params(params) + self.install_packages(env) + Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True) + + def configure(self, env): + import params + params.pika_slave = True + env.set_params(params) + + dolphin_env() + + def start(self, env): + import params + env.set_params(params) + self.configure(env) + no_op_test = format("ls {dolphin_pidfile_dir}/alert-server.pid >/dev/null 2>&1 && ps `cat {dolphin_pidfile_dir}/alert-server.pid` | grep `cat {dolphin_pidfile_dir}/alert-server.pid` >/dev/null 2>&1") + + start_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh start alert-server") + Execute(start_cmd, user=params.dolphin_user, not_if=no_op_test) + + def stop(self, env): + import params + env.set_params(params) + stop_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh stop alert-server") + Execute(stop_cmd, user=params.dolphin_user) + time.sleep(5) + + def status(self, env): + import status_params + env.set_params(status_params) + check_process_status(status_params.dolphin_run_dir + "alert-server.pid") + + +if __name__ == "__main__": + DolphinAlertService().execute() diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_api_service.py b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_api_service.py new file mode 100644 index 0000000000..bdc18fb602 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_api_service.py @@ -0,0 +1,70 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import time +from resource_management import * + +from dolphin_env import dolphin_env + + +class DolphinApiService(Script): + def install(self, env): + import params + env.set_params(params) + self.install_packages(env) + Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True) + + def configure(self, env): + import params + params.pika_slave = True + env.set_params(params) + + dolphin_env() + + def start(self, env): + import params + env.set_params(params) + self.configure(env) + + #init + init_cmd=format("sh " + params.dolphin_home + "/script/create-dolphinscheduler.sh") + Execute(init_cmd, user=params.dolphin_user) + + #upgrade + upgrade_cmd=format("sh " + params.dolphin_home + "/script/upgrade-dolphinscheduler.sh") + Execute(upgrade_cmd, user=params.dolphin_user) + + no_op_test = format("ls {dolphin_pidfile_dir}/api-server.pid >/dev/null 2>&1 && ps `cat {dolphin_pidfile_dir}/api-server.pid` | grep `cat {dolphin_pidfile_dir}/api-server.pid` >/dev/null 2>&1") + + start_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh start api-server") + Execute(start_cmd, user=params.dolphin_user, not_if=no_op_test) + + def stop(self, env): + import params + env.set_params(params) + stop_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh stop api-server") + Execute(stop_cmd, user=params.dolphin_user) + time.sleep(5) + + def status(self, env): + import status_params + env.set_params(status_params) + check_process_status(status_params.dolphin_run_dir + "api-server.pid") + + +if __name__ == "__main__": + DolphinApiService().execute() diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_env.py b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_env.py new file mode 100644 index 0000000000..1661d76c75 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_env.py @@ -0,0 +1,123 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * + + +def dolphin_env(): + import params + + Directory(params.dolphin_pidfile_dir, + mode=0777, + owner=params.dolphin_user, + group=params.dolphin_group, + create_parents=True + ) + Directory(params.dolphin_log_dir, + mode=0777, + owner=params.dolphin_user, + group=params.dolphin_group, + create_parents=True + ) + Directory(params.dolphin_conf_dir, + mode=0777, + owner=params.dolphin_user, + group=params.dolphin_group, + create_parents=True + ) + + Directory(params.dolphin_common_map['data.basedir.path'], + mode=0777, + owner=params.dolphin_user, + group=params.dolphin_group, + create_parents=True + ) + + + File(format(params.dolphin_env_path), + mode=0777, + content=InlineTemplate(params.dolphin_env_content), + owner=params.dolphin_user, + group=params.dolphin_group + ) + + + File(format(params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh"), + mode=0755, + content=Template("dolphin-daemon.sh.j2"), + owner=params.dolphin_user, + group=params.dolphin_group + ) + + File(format(params.dolphin_conf_dir + "/master.properties"), + mode=0755, + content=Template("master.properties.j2"), + owner=params.dolphin_user, + group=params.dolphin_group + ) + + File(format(params.dolphin_conf_dir + "/worker.properties"), + mode=0755, + content=Template("worker.properties.j2"), + owner=params.dolphin_user, + group=params.dolphin_group + ) + + + File(format(params.dolphin_conf_dir + "/alert.properties"), + mode=0755, + content=Template("alert.properties.j2"), + owner=params.dolphin_user, + group=params.dolphin_group + ) + + File(format(params.dolphin_conf_dir + "/datasource.properties"), + mode=0755, + content=Template("datasource.properties.j2"), + owner=params.dolphin_user, + group=params.dolphin_group + ) + + File(format(params.dolphin_conf_dir + "/application-api.properties"), + mode=0755, + content=Template("application-api.properties.j2"), + owner=params.dolphin_user, + group=params.dolphin_group + ) + + File(format(params.dolphin_conf_dir + "/common.properties"), + mode=0755, + content=Template("common.properties.j2"), + owner=params.dolphin_user, + group=params.dolphin_group + ) + + File(format(params.dolphin_conf_dir + "/quartz.properties"), + mode=0755, + content=Template("quartz.properties.j2"), + owner=params.dolphin_user, + group=params.dolphin_group + ) + + File(format(params.dolphin_conf_dir + "/zookeeper.properties"), + mode=0755, + content=Template("zookeeper.properties.j2"), + owner=params.dolphin_user, + group=params.dolphin_group + ) diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_logger_service.py b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_logger_service.py new file mode 100644 index 0000000000..f1c19bd66f --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_logger_service.py @@ -0,0 +1,61 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import time +from resource_management import * + +from dolphin_env import dolphin_env + + +class DolphinLoggerService(Script): + def install(self, env): + import params + env.set_params(params) + self.install_packages(env) + Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True) + + def configure(self, env): + import params + params.pika_slave = True + env.set_params(params) + + dolphin_env() + + def start(self, env): + import params + env.set_params(params) + self.configure(env) + no_op_test = format("ls {dolphin_pidfile_dir}/logger-server.pid >/dev/null 2>&1 && ps `cat {dolphin_pidfile_dir}/logger-server.pid` | grep `cat {dolphin_pidfile_dir}/logger-server.pid` >/dev/null 2>&1") + + start_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh start logger-server") + Execute(start_cmd, user=params.dolphin_user, not_if=no_op_test) + + def stop(self, env): + import params + env.set_params(params) + stop_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh stop logger-server") + Execute(stop_cmd, user=params.dolphin_user) + time.sleep(5) + + def status(self, env): + import status_params + env.set_params(status_params) + check_process_status(status_params.dolphin_run_dir + "logger-server.pid") + + +if __name__ == "__main__": + DolphinLoggerService().execute() diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_master_service.py b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_master_service.py new file mode 100644 index 0000000000..6ee7ecfcf3 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_master_service.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import time +from resource_management import * + +from dolphin_env import dolphin_env + + +class DolphinMasterService(Script): + def install(self, env): + import params + env.set_params(params) + self.install_packages(env) + Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True) + + def configure(self, env): + import params + params.pika_slave = True + env.set_params(params) + + dolphin_env() + + def start(self, env): + import params + env.set_params(params) + self.configure(env) + no_op_test = format("ls {dolphin_pidfile_dir}/master-server.pid >/dev/null 2>&1 && ps `cat {dolphin_pidfile_dir}/master-server.pid` | grep `cat {dolphin_pidfile_dir}/master-server.pid` >/dev/null 2>&1") + start_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh start master-server") + Execute(start_cmd, user=params.dolphin_user, not_if=no_op_test) + + def stop(self, env): + import params + env.set_params(params) + stop_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh stop master-server") + Execute(stop_cmd, user=params.dolphin_user) + time.sleep(5) + + def status(self, env): + import status_params + env.set_params(status_params) + check_process_status(status_params.dolphin_run_dir + "master-server.pid") + + +if __name__ == "__main__": + DolphinMasterService().execute() diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_worker_service.py b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_worker_service.py new file mode 100644 index 0000000000..2d145ee730 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_worker_service.py @@ -0,0 +1,60 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import time +from resource_management import * + +from dolphin_env import dolphin_env + + +class DolphinWorkerService(Script): + def install(self, env): + import params + env.set_params(params) + self.install_packages(env) + Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True) + + def configure(self, env): + import params + params.pika_slave = True + env.set_params(params) + + dolphin_env() + + def start(self, env): + import params + env.set_params(params) + self.configure(env) + no_op_test = format("ls {dolphin_pidfile_dir}/worker-server.pid >/dev/null 2>&1 && ps `cat {dolphin_pidfile_dir}/worker-server.pid` | grep `cat {dolphin_pidfile_dir}/worker-server.pid` >/dev/null 2>&1") + start_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh start worker-server") + Execute(start_cmd, user=params.dolphin_user, not_if=no_op_test) + + def stop(self, env): + import params + env.set_params(params) + stop_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh stop worker-server") + Execute(stop_cmd, user=params.dolphin_user) + time.sleep(5) + + def status(self, env): + import status_params + env.set_params(status_params) + check_process_status(status_params.dolphin_run_dir + "worker-server.pid") + + +if __name__ == "__main__": + DolphinWorkerService().execute() diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/params.py b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/params.py new file mode 100644 index 0000000000..b09b2589f4 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/params.py @@ -0,0 +1,154 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + + +import sys +from resource_management import * +from resource_management.core.logger import Logger +from resource_management.libraries.functions import default + +Logger.initialize_logger() +reload(sys) +sys.setdefaultencoding('utf-8') + +# server configurations +config = Script.get_config() + +# conf_dir = "/etc/" +dolphin_home = "/opt/soft/dolphinscheduler" +dolphin_conf_dir = dolphin_home + "/conf" +dolphin_log_dir = dolphin_home + "/logs" +dolphin_bin_dir = dolphin_home + "/bin" +dolphin_lib_jars = dolphin_home + "/lib/*" +dolphin_pidfile_dir = "/opt/soft/run/dolphinscheduler" + +rmHosts = default("/clusterHostInfo/rm_host", []) + +# dolphin-env +dolphin_env_map = {} +dolphin_env_map.update(config['configurations']['dolphin-env']) + +# which user to install and admin dolphin scheduler +dolphin_user = dolphin_env_map['dolphin.user'] +dolphin_group = dolphin_env_map['dolphin.group'] + +# .dolphinscheduler_env.sh +dolphin_env_path = dolphin_conf_dir + '/env/dolphinscheduler_env.sh' +dolphin_env_content = dolphin_env_map['dolphinscheduler-env-content'] + +# database config +dolphin_database_config = {} +dolphin_database_config['dolphin_database_type'] = dolphin_env_map['dolphin.database.type'] +dolphin_database_config['dolphin_database_username'] = dolphin_env_map['dolphin.database.username'] +dolphin_database_config['dolphin_database_password'] = dolphin_env_map['dolphin.database.password'] +if 'mysql' == dolphin_database_config['dolphin_database_type']: + dolphin_database_config['dolphin_database_driver'] = 'com.mysql.jdbc.Driver' + dolphin_database_config['driverDelegateClass'] = 'org.quartz.impl.jdbcjobstore.StdJDBCDelegate' + dolphin_database_config['dolphin_database_url'] = 'jdbc:mysql://' + dolphin_env_map['dolphin.database.host'] \ + + ':' + dolphin_env_map['dolphin.database.port'] \ + + '/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8' +else: + dolphin_database_config['dolphin_database_driver'] = 'org.postgresql.Driver' + dolphin_database_config['driverDelegateClass'] = 'org.quartz.impl.jdbcjobstore.PostgreSQLDelegate' + dolphin_database_config['dolphin_database_url'] = 'jdbc:postgresql://' + dolphin_env_map['dolphin.database.host'] \ + + ':' + dolphin_env_map['dolphin.database.port'] \ + + '/dolphinscheduler' + + + + + +# application-alert.properties +dolphin_alert_map = {} +wechat_push_url = 'https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=$token' +wechat_token_url = 'https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid=$corpId&corpsecret=$secret' +wechat_team_send_msg = '{\"toparty\":\"$toParty\",\"agentid\":\"$agentId\",\"msgtype\":\"text\",\"text\":{\"content\":\"$msg\"},\"safe\":\"0\"}' +wechat_user_send_msg = '{\"touser\":\"$toUser\",\"agentid\":\"$agentId\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"$msg\"}}' + +dolphin_alert_config_map = config['configurations']['dolphin-alert'] + +if dolphin_alert_config_map['enterprise.wechat.enable']: + dolphin_alert_map['enterprise.wechat.push.ur'] = wechat_push_url + dolphin_alert_map['enterprise.wechat.token.url'] = wechat_token_url + dolphin_alert_map['enterprise.wechat.team.send.msg'] = wechat_team_send_msg + dolphin_alert_map['enterprise.wechat.user.send.msg'] = wechat_user_send_msg + +dolphin_alert_map.update(dolphin_alert_config_map) + + + +# application-api.properties +dolphin_app_api_map = {} +dolphin_app_api_map.update(config['configurations']['dolphin-application-api']) + + +# common.properties +dolphin_common_map = {} + +if 'yarn-site' in config['configurations'] and \ + 'yarn.resourcemanager.webapp.address' in config['configurations']['yarn-site']: + yarn_resourcemanager_webapp_address = config['configurations']['yarn-site']['yarn.resourcemanager.webapp.address'] + yarn_application_status_address = 'http://' + yarn_resourcemanager_webapp_address + '/ws/v1/cluster/apps/%s' + dolphin_common_map['yarn.application.status.address'] = yarn_application_status_address + +rmHosts = default("/clusterHostInfo/rm_host", []) +if len(rmHosts) > 1: + dolphin_common_map['yarn.resourcemanager.ha.rm.ids'] = ','.join(rmHosts) +else: + dolphin_common_map['yarn.resourcemanager.ha.rm.ids'] = '' + +dolphin_common_map_tmp = config['configurations']['dolphin-common'] +data_basedir_path = dolphin_common_map_tmp['data.basedir.path'] +process_exec_basepath = data_basedir_path + '/exec' +data_download_basedir_path = data_basedir_path + '/download' +dolphin_common_map['process.exec.basepath'] = process_exec_basepath +dolphin_common_map['data.download.basedir.path'] = data_download_basedir_path +dolphin_common_map['dolphinscheduler.env.path'] = dolphin_env_path +dolphin_common_map.update(config['configurations']['dolphin-common']) + +# datasource.properties +dolphin_datasource_map = {} +dolphin_datasource_map['spring.datasource.type'] = 'com.alibaba.druid.pool.DruidDataSource' +dolphin_datasource_map['spring.datasource.driver-class-name'] = dolphin_database_config['dolphin_database_driver'] +dolphin_datasource_map['spring.datasource.url'] = dolphin_database_config['dolphin_database_url'] +dolphin_datasource_map['spring.datasource.username'] = dolphin_database_config['dolphin_database_username'] +dolphin_datasource_map['spring.datasource.password'] = dolphin_database_config['dolphin_database_password'] +dolphin_datasource_map.update(config['configurations']['dolphin-datasource']) + +# master.properties +dolphin_master_map = config['configurations']['dolphin-master'] + +# quartz.properties +dolphin_quartz_map = {} +dolphin_quartz_map['org.quartz.jobStore.driverDelegateClass'] = dolphin_database_config['driverDelegateClass'] +dolphin_quartz_map.update(config['configurations']['dolphin-quartz']) + +# worker.properties +dolphin_worker_map = config['configurations']['dolphin-worker'] + +# zookeeper.properties +dolphin_zookeeper_map={} +zookeeperHosts = default("/clusterHostInfo/zookeeper_hosts", []) +if len(zookeeperHosts) > 0 and "clientPort" in config['configurations']['zoo.cfg']: + clientPort = config['configurations']['zoo.cfg']['clientPort'] + zookeeperPort = ":" + clientPort + "," + dolphin_zookeeper_map['zookeeper.quorum'] = zookeeperPort.join(zookeeperHosts) + ":" + clientPort +dolphin_zookeeper_map.update(config['configurations']['dolphin-zookeeper']) + + + diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/service_check.py b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/service_check.py new file mode 100644 index 0000000000..0e12f69932 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/service_check.py @@ -0,0 +1,31 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * +from resource_management.libraries.functions import get_unique_id_and_date + +class ServiceCheck(Script): + def service_check(self, env): + import params + #env.set_params(params) + + # Execute(format("which pika_server")) + +if __name__ == "__main__": + ServiceCheck().execute() diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/status_params.py b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/status_params.py new file mode 100644 index 0000000000..24b2c8b1bc --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/status_params.py @@ -0,0 +1,23 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from resource_management import * + +config = Script.get_config() + +dolphin_run_dir = "/opt/soft/run/dolphinscheduler/" diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/alert.properties.j2 b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/alert.properties.j2 new file mode 100644 index 0000000000..73840b8c18 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/alert.properties.j2 @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{% for key, value in dolphin_alert_map.iteritems() -%} + {{key}}={{value}} +{% endfor %} \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/application-api.properties.j2 b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/application-api.properties.j2 new file mode 100644 index 0000000000..70118003b9 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/application-api.properties.j2 @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{% for key, value in dolphin_app_api_map.iteritems() -%} + {{key}}={{value}} +{% endfor %} \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/common.properties.j2 b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/common.properties.j2 new file mode 100644 index 0000000000..2220c4effa --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/common.properties.j2 @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{% for key, value in dolphin_common_map.iteritems() -%} + {{key}}={{value}} +{% endfor %} \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/datasource.properties.j2 b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/datasource.properties.j2 new file mode 100644 index 0000000000..40aed83543 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/datasource.properties.j2 @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{% for key, value in dolphin_datasource_map.iteritems() -%} + {{key}}={{value}} +{% endfor %} \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/dolphin-daemon.sh.j2 b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/dolphin-daemon.sh.j2 new file mode 100644 index 0000000000..0802b74750 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/dolphin-daemon.sh.j2 @@ -0,0 +1,116 @@ +#!/bin/sh +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +usage="Usage: dolphinscheduler-daemon.sh (start|stop) " + +# if no args specified, show usage +if [ $# -le 1 ]; then + echo $usage + exit 1 +fi + +startStop=$1 +shift +command=$1 +shift + +echo "Begin $startStop $command......" + +BIN_DIR=`dirname $0` +BIN_DIR=`cd "$BIN_DIR"; pwd` +DOLPHINSCHEDULER_HOME=$BIN_DIR/.. + +export HOSTNAME=`hostname` + +DOLPHINSCHEDULER_LIB_JARS={{dolphin_lib_jars}} + +DOLPHINSCHEDULER_OPTS="-server -Xmx16g -Xms4g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70" +STOP_TIMEOUT=5 + +log={{dolphin_log_dir}}/dolphinscheduler-$command-$HOSTNAME.out +pid={{dolphin_pidfile_dir}}/$command.pid + +cd $DOLPHINSCHEDULER_HOME + +if [ "$command" = "api-server" ]; then + LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/logback-api.xml -Dspring.profiles.active=api" + CLASS=org.apache.dolphinscheduler.api.ApiApplicationServer +elif [ "$command" = "master-server" ]; then + LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/logback-master.xml -Ddruid.mysql.usePingMethod=false" + CLASS=org.apache.dolphinscheduler.server.master.MasterServer +elif [ "$command" = "worker-server" ]; then + LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/logback-worker.xml -Ddruid.mysql.usePingMethod=false" + CLASS=org.apache.dolphinscheduler.server.worker.WorkerServer +elif [ "$command" = "alert-server" ]; then + LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/logback-alert.xml" + CLASS=org.apache.dolphinscheduler.alert.AlertServer +elif [ "$command" = "logger-server" ]; then + CLASS=org.apache.dolphinscheduler.server.log.LoggerServer +else + echo "Error: No command named \`$command' was found." + exit 1 +fi + +case $startStop in + (start) + + if [ -f $pid ]; then + if kill -0 `cat $pid` > /dev/null 2>&1; then + echo $command running as process `cat $pid`. Stop it first. + exit 1 + fi + fi + + echo starting $command, logging to $log + + exec_command="$LOG_FILE $DOLPHINSCHEDULER_OPTS -classpath {{dolphin_conf_dir}}:{{dolphin_lib_jars}} $CLASS" + + echo "nohup java $exec_command > $log 2>&1 < /dev/null &" + nohup java $exec_command > $log 2>&1 < /dev/null & + echo $! > $pid + ;; + + (stop) + + if [ -f $pid ]; then + TARGET_PID=`cat $pid` + if kill -0 $TARGET_PID > /dev/null 2>&1; then + echo stopping $command + kill $TARGET_PID + sleep $STOP_TIMEOUT + if kill -0 $TARGET_PID > /dev/null 2>&1; then + echo "$command did not stop gracefully after $STOP_TIMEOUT seconds: killing with kill -9" + kill -9 $TARGET_PID + fi + else + echo no $command to stop + fi + rm -f $pid + else + echo no $command to stop + fi + ;; + + (*) + echo $usage + exit 1 + ;; + +esac + +echo "End $startStop $command." \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/master.properties.j2 b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/master.properties.j2 new file mode 100644 index 0000000000..d9b85e14cf --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/master.properties.j2 @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{% for key, value in dolphin_master_map.iteritems() -%} + {{key}}={{value}} +{% endfor %} \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/quartz.properties.j2 b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/quartz.properties.j2 new file mode 100644 index 0000000000..e027a263b5 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/quartz.properties.j2 @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{% for key, value in dolphin_quartz_map.iteritems() -%} + {{key}}={{value}} +{% endfor %} \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/worker.properties.j2 b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/worker.properties.j2 new file mode 100644 index 0000000000..a008b74084 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/worker.properties.j2 @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{% for key, value in dolphin_worker_map.iteritems() -%} + {{key}}={{value}} +{% endfor %} \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/zookeeper.properties.j2 b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/zookeeper.properties.j2 new file mode 100644 index 0000000000..9eb14eaef3 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/zookeeper.properties.j2 @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{% for key, value in dolphin_zookeeper_map.iteritems() -%} + {{key}}={{value}} +{% endfor %} \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/quicklinks/quicklinks.json b/ambari_plugin/common-services/DOLPHIN/2.0.0/quicklinks/quicklinks.json new file mode 100755 index 0000000000..8753004fef --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/2.0.0/quicklinks/quicklinks.json @@ -0,0 +1,26 @@ +{ + "name": "default", + "description": "default quick links configuration", + "configuration": { + "protocol": + { + "type":"http" + }, + + "links": [ + { + "name": "dolphin-application-ui", + "label": "DolphinApplication UI", + "requires_user_name": "false", + "component_name": "DOLPHIN_API", + "url": "%@://%@:%@/dolphinscheduler/ui/view/login/index.html", + "port":{ + "http_property": "server.port", + "http_default_port": "12345", + "regex": "^(\\d+)$", + "site": "dolphin-application-api" + } + } + ] + } +} \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/themes/theme.json b/ambari_plugin/common-services/DOLPHIN/2.0.0/themes/theme.json new file mode 100644 index 0000000000..953e2323f8 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/2.0.0/themes/theme.json @@ -0,0 +1,661 @@ +{ + "name": "default", + "description": "Default theme for Dolphin Scheduler service", + "configuration": { + "layouts": [ + { + "name": "default", + "tabs": [ + { + "name": "settings", + "display-name": "Settings", + "layout": { + "tab-rows": "3", + "tab-columns": "3", + "sections": [ + { + "name": "dolphin-env-config", + "display-name": "Dolphin Env Config", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "2", + "section-rows": "1", + "section-columns": "2", + "subsections": [ + { + "name": "env-row1-col1", + "display-name": "Deploy User Info", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + }, + { + "name": "env-row1-col2", + "display-name": "System Env Optimization", + "row-index": "0", + "column-index": "1", + "row-span": "1", + "column-span": "1" + } + ] + }, + { + "name": "dolphin-database-config", + "display-name": "Database Config", + "row-index": "1", + "column-index": "0", + "row-span": "1", + "column-span": "2", + "section-rows": "1", + "section-columns": "3", + "subsections": [ + { + "name": "database-row1-col1", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + }, + { + "name": "database-row1-col2", + "row-index": "0", + "column-index": "1", + "row-span": "1", + "column-span": "1" + }, + { + "name": "database-row1-col3", + "row-index": "0", + "column-index": "2", + "row-span": "1", + "column-span": "1" + } + ] + }, + { + "name": "dynamic-config", + "row-index": "2", + "column-index": "0", + "row-span": "1", + "column-span": "2", + "section-rows": "1", + "section-columns": "3", + "subsections": [ + { + "name": "dynamic-row1-col1", + "display-name": "Resource FS Config", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + }, + { + "name": "dynamic-row1-col2", + "display-name": "Kerberos Info", + "row-index": "0", + "column-index": "1", + "row-span": "1", + "column-span": "1" + }, + { + "name": "dynamic-row1-col3", + "display-name": "Wechat Info", + "row-index": "0", + "column-index": "1", + "row-span": "1", + "column-span": "1" + } + ] + } + ] + } + } + ] + } + ], + "placement": { + "configuration-layout": "default", + "configs": [ + { + "config": "dolphin-env/dolphin.database.type", + "subsection-name": "database-row1-col1" + }, + { + "config": "dolphin-env/dolphin.database.host", + "subsection-name": "database-row1-col2" + }, + { + "config": "dolphin-env/dolphin.database.port", + "subsection-name": "database-row1-col2" + }, + { + "config": "dolphin-env/dolphin.database.username", + "subsection-name": "database-row1-col3" + }, + { + "config": "dolphin-env/dolphin.database.password", + "subsection-name": "database-row1-col3" + }, + { + "config": "dolphin-env/dolphin.user", + "subsection-name": "env-row1-col1" + }, + { + "config": "dolphin-env/dolphin.group", + "subsection-name": "env-row1-col1" + }, + { + "config": "dolphin-env/dolphinscheduler-env-content", + "subsection-name": "env-row1-col2" + }, + { + "config": "dolphin-common/resource.storage.type", + "subsection-name": "dynamic-row1-col1" + }, + { + "config": "dolphin-common/resource.upload.path", + "subsection-name": "dynamic-row1-col1", + "depends-on": [ + { + "configs":[ + "dolphin-common/resource.storage.type" + ], + "if": "${dolphin-common/resource.storage.type} === HDFS || ${dolphin-common/resource.storage.type} === S3", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "dolphin-common/hdfs.root.user", + "subsection-name": "dynamic-row1-col1", + "depends-on": [ + { + "configs":[ + "dolphin-common/resource.storage.type" + ], + "if": "${dolphin-common/resource.storage.type} === HDFS", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "dolphin-common/data.store2hdfs.basepath", + "subsection-name": "dynamic-row1-col1", + "depends-on": [ + { + "configs":[ + "dolphin-common/resource.storage.type" + ], + "if": "${dolphin-common/resource.storage.type} === HDFS", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "dolphin-common/fs.defaultFS", + "subsection-name": "dynamic-row1-col1", + "depends-on": [ + { + "configs":[ + "dolphin-common/resource.storage.type" + ], + "if": "${dolphin-common/resource.storage.type} === HDFS", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "dolphin-common/fs.s3a.endpoint", + "subsection-name": "dynamic-row1-col1", + "depends-on": [ + { + "configs":[ + "dolphin-common/resource.storage.type" + ], + "if": "${dolphin-common/resource.storage.type} === S3", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "dolphin-common/fs.s3a.access.key", + "subsection-name": "dynamic-row1-col1", + "depends-on": [ + { + "configs":[ + "dolphin-common/resource.storage.type" + ], + "if": "${dolphin-common/resource.storage.type} === S3", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "dolphin-common/fs.s3a.secret.key", + "subsection-name": "dynamic-row1-col1", + "depends-on": [ + { + "configs":[ + "dolphin-common/resource.storage.type" + ], + "if": "${dolphin-common/resource.storage.type} === S3", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "dolphin-common/hadoop.security.authentication.startup.state", + "subsection-name": "dynamic-row1-col2" + }, + { + "config": "dolphin-common/java.security.krb5.conf.path", + "subsection-name": "dynamic-row1-col2", + "depends-on": [ + { + "configs":[ + "dolphin-common/hadoop.security.authentication.startup.state" + ], + "if": "${dolphin-common/hadoop.security.authentication.startup.state}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "dolphin-common/login.user.keytab.username", + "subsection-name": "dynamic-row1-col2", + "depends-on": [ + { + "configs":[ + "dolphin-common/hadoop.security.authentication.startup.state" + ], + "if": "${dolphin-common/hadoop.security.authentication.startup.state}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "dolphin-common/login.user.keytab.path", + "subsection-name": "dynamic-row1-col2", + "depends-on": [ + { + "configs":[ + "dolphin-common/hadoop.security.authentication.startup.state" + ], + "if": "${dolphin-common/hadoop.security.authentication.startup.state}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "dolphin-common/kerberos.expire.time", + "subsection-name": "dynamic-row1-col2", + "depends-on": [ + { + "configs":[ + "dolphin-common/hadoop.security.authentication.startup.state" + ], + "if": "${dolphin-common/hadoop.security.authentication.startup.state}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "dolphin-alert/enterprise.wechat.enable", + "subsection-name": "dynamic-row1-col3" + }, + { + "config": "dolphin-alert/enterprise.wechat.corp.id", + "subsection-name": "dynamic-row1-col3", + "depends-on": [ + { + "configs":[ + "dolphin-alert/enterprise.wechat.enable" + ], + "if": "${dolphin-alert/enterprise.wechat.enable}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "dolphin-alert/enterprise.wechat.secret", + "subsection-name": "dynamic-row1-col3", + "depends-on": [ + { + "configs":[ + "dolphin-alert/enterprise.wechat.enable" + ], + "if": "${dolphin-alert/enterprise.wechat.enable}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "dolphin-alert/enterprise.wechat.agent.id", + "subsection-name": "dynamic-row1-col3", + "depends-on": [ + { + "configs":[ + "dolphin-alert/enterprise.wechat.enable" + ], + "if": "${dolphin-alert/enterprise.wechat.enable}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "dolphin-alert/enterprise.wechat.users", + "subsection-name": "dynamic-row1-col3", + "depends-on": [ + { + "configs":[ + "dolphin-alert/enterprise.wechat.enable" + ], + "if": "${dolphin-alert/enterprise.wechat.enable}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + } + ] + }, + "widgets": [ + { + "config": "dolphin-env/dolphin.database.type", + "widget": { + "type": "combo" + } + }, + { + "config": "dolphin-env/dolphin.database.host", + "widget": { + "type": "text-field" + } + }, + { + "config": "dolphin-env/dolphin.database.port", + "widget": { + "type": "text-field", + "units": [ + { + "unit-name": "int" + } + ] + } + }, + { + "config": "dolphin-env/dolphin.database.username", + "widget": { + "type": "text-field" + } + }, + { + "config": "dolphin-env/dolphin.database.password", + "widget": { + "type": "password" + } + }, + { + "config": "dolphin-env/dolphin.user", + "widget": { + "type": "text-field" + } + }, + { + "config": "dolphin-env/dolphin.group", + "widget": { + "type": "text-field" + } + }, + { + "config": "dolphin-env/dolphinscheduler-env-content", + "widget": { + "type": "text-area" + } + }, + { + "config": "dolphin-common/resource.storage.type", + "widget": { + "type": "combo" + } + }, + { + "config": "dolphin-common/resource.upload.path", + "widget": { + "type": "text-field" + } + }, + { + "config": "dolphin-common/hdfs.root.user", + "widget": { + "type": "text-field" + } + }, + { + "config": "dolphin-common/data.store2hdfs.basepath", + "widget": { + "type": "text-field" + } + }, + { + "config": "dolphin-common/fs.defaultFS", + "widget": { + "type": "text-field" + } + }, + { + "config": "dolphin-common/fs.s3a.endpoint", + "widget": { + "type": "text-field" + } + }, + { + "config": "dolphin-common/fs.s3a.access.key", + "widget": { + "type": "text-field" + } + }, + { + "config": "dolphin-common/fs.s3a.secret.key", + "widget": { + "type": "text-field" + } + }, + { + "config": "dolphin-common/hadoop.security.authentication.startup.state", + "widget": { + "type": "toggle" + } + }, + { + "config": "dolphin-common/java.security.krb5.conf.path", + "widget": { + "type": "text-field" + } + }, + { + "config": "dolphin-common/login.user.keytab.username", + "widget": { + "type": "text-field" + } + }, + { + "config": "dolphin-common/login.user.keytab.path", + "widget": { + "type": "text-field" + } + }, + { + "config": "dolphin-common/kerberos.expire.time", + "widget": { + "type": "text-field" + } + }, + { + "config": "dolphin-alert/enterprise.wechat.enable", + "widget": { + "type": "toggle" + } + }, + { + "config": "dolphin-alert/enterprise.wechat.corp.id", + "widget": { + "type": "text-field" + } + }, + { + "config": "dolphin-alert/enterprise.wechat.secret", + "widget": { + "type": "text-field" + } + }, + { + "config": "dolphin-alert/enterprise.wechat.agent.id", + "widget": { + "type": "text-field" + } + }, + { + "config": "dolphin-alert/enterprise.wechat.users", + "widget": { + "type": "text-field" + } + } + ] + } +} diff --git a/ambari_plugin/readme.pdf b/ambari_plugin/readme.pdf deleted file mode 100644 index 1209375701..0000000000 Binary files a/ambari_plugin/readme.pdf and /dev/null differ diff --git a/ambari_plugin/statcks/DOLPHIN/metainfo.xml b/ambari_plugin/statcks/DOLPHIN/metainfo.xml index c41db5f513..3af7f383eb 100755 --- a/ambari_plugin/statcks/DOLPHIN/metainfo.xml +++ b/ambari_plugin/statcks/DOLPHIN/metainfo.xml @@ -20,7 +20,7 @@ DOLPHIN - common-services/DOLPHIN/1.2.1 + common-services/DOLPHIN/2.0.0 \ No newline at end of file diff --git a/dockerfile/conf/dolphinscheduler/zookeeper.properties.tpl b/dockerfile/conf/dolphinscheduler/zookeeper.properties.tpl index a0ef72dc8f..362a4e85ad 100644 --- a/dockerfile/conf/dolphinscheduler/zookeeper.properties.tpl +++ b/dockerfile/conf/dolphinscheduler/zookeeper.properties.tpl @@ -22,8 +22,8 @@ zookeeper.quorum=${ZOOKEEPER_QUORUM} #zookeeper.dolphinscheduler.root=/dolphinscheduler # dolphinscheduler failover directory -#zookeeper.session.timeout=300 -#zookeeper.connection.timeout=300 +#zookeeper.session.timeout=60000 +#zookeeper.connection.timeout=30000 #zookeeper.retry.base.sleep=100 #zookeeper.retry.max.sleep=30000 -#zookeeper.retry.maxtime=5 \ No newline at end of file +#zookeeper.retry.maxtime=10 \ No newline at end of file diff --git a/dolphinscheduler-service/src/main/resources/zookeeper.properties b/dolphinscheduler-service/src/main/resources/zookeeper.properties index a0df570d47..2204467ac9 100644 --- a/dolphinscheduler-service/src/main/resources/zookeeper.properties +++ b/dolphinscheduler-service/src/main/resources/zookeeper.properties @@ -23,7 +23,7 @@ zookeeper.quorum=localhost:2181 # dolphinscheduler failover directory #zookeeper.session.timeout=60000 -#zookeeper.connection.timeout=300 +#zookeeper.connection.timeout=30000 #zookeeper.retry.base.sleep=100 #zookeeper.retry.max.sleep=30000 -#zookeeper.retry.maxtime=5 \ No newline at end of file +#zookeeper.retry.maxtime=10 \ No newline at end of file diff --git a/install.sh b/install.sh index 20b293f697..a1323952a7 100644 --- a/install.sh +++ b/install.sh @@ -52,11 +52,8 @@ sed -i ${txt} "s#hadoop.security.authentication.startup.state.*#hadoop.security. sed -i ${txt} "s#java.security.krb5.conf.path.*#java.security.krb5.conf.path=${krb5ConfPath}#g" conf/common.properties sed -i ${txt} "s#login.user.keytab.username.*#login.user.keytab.username=${keytabUserName}#g" conf/common.properties sed -i ${txt} "s#login.user.keytab.path.*#login.user.keytab.path=${keytabPath}#g" conf/common.properties -sed -i ${txt} "s#zookeeper.quorum.*#zookeeper.quorum=${zkQuorum}#g" conf/common.properties - +sed -i ${txt} "s#zookeeper.quorum.*#zookeeper.quorum=${zkQuorum}#g" conf/zookeeper.properties sed -i ${txt} "s#server.port.*#server.port=${apiServerPort}#g" conf/application-api.properties - - sed -i ${txt} "s#mail.server.host.*#mail.server.host=${mailServerHost}#g" conf/alert.properties sed -i ${txt} "s#mail.server.port.*#mail.server.port=${mailServerPort}#g" conf/alert.properties sed -i ${txt} "s#mail.sender.*#mail.sender=${mailSender}#g" conf/alert.properties