From 39b3a37abdb1bed088e5494de41769c1862acac8 Mon Sep 17 00:00:00 2001 From: zhangchunyang Date: Sun, 28 Jun 2020 11:37:59 +0800 Subject: [PATCH 1/3] [bugfix] remove Conflicting configuration sudo and user (#3038) Co-authored-by: zhangchunyang1024 --- .../DOLPHIN/1.3.0/package/scripts/dolphin_alert_service.py | 3 ++- .../DOLPHIN/1.3.0/package/scripts/dolphin_api_service.py | 3 ++- .../DOLPHIN/1.3.0/package/scripts/dolphin_logger_service.py | 4 ++-- .../DOLPHIN/1.3.0/package/scripts/dolphin_master_service.py | 3 ++- .../DOLPHIN/1.3.0/package/scripts/dolphin_worker_service.py | 3 ++- 5 files changed, 10 insertions(+), 6 deletions(-) diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_alert_service.py b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_alert_service.py index 62255a3432..e78c38d272 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_alert_service.py +++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_alert_service.py @@ -26,7 +26,8 @@ class DolphinAlertService(Script): import params env.set_params(params) self.install_packages(env) - Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True) + Execute(('chmod', '-R', '777', params.dolphin_home)) + Execute(('chown', '-R', params.dolphin_user + ":" + params.dolphin_group, params.dolphin_home)) def configure(self, env): import params diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_api_service.py b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_api_service.py index bdc18fb602..5a28924a9a 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_api_service.py +++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_api_service.py @@ -26,7 +26,8 @@ class DolphinApiService(Script): import params env.set_params(params) self.install_packages(env) - Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True) + Execute(('chmod', '-R', '777', params.dolphin_home)) + Execute(('chown', '-R', params.dolphin_user + ":" + params.dolphin_group, params.dolphin_home)) def configure(self, env): import params diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_logger_service.py b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_logger_service.py index f1c19bd66f..fb47e132e1 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_logger_service.py +++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_logger_service.py @@ -26,8 +26,8 @@ class DolphinLoggerService(Script): import params env.set_params(params) self.install_packages(env) - Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True) - + Execute(('chmod', '-R', '777', params.dolphin_home)) + Execute(('chown', '-R', params.dolphin_user + ":" + params.dolphin_group, params.dolphin_home)) def configure(self, env): import params params.pika_slave = True diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_master_service.py b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_master_service.py index 6ee7ecfcf3..8d64935d26 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_master_service.py +++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_master_service.py @@ -27,7 +27,8 @@ class DolphinMasterService(Script): import params env.set_params(params) self.install_packages(env) - Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True) + Execute(('chmod', '-R', '777', params.dolphin_home)) + Execute(('chown', '-R', params.dolphin_user + ":" + params.dolphin_group, params.dolphin_home)) def configure(self, env): import params diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_worker_service.py b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_worker_service.py index 2d145ee730..1f542c06c2 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_worker_service.py +++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_worker_service.py @@ -26,7 +26,8 @@ class DolphinWorkerService(Script): import params env.set_params(params) self.install_packages(env) - Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True) + Execute(('chmod', '-R', '777', params.dolphin_home)) + Execute(('chown', '-R', params.dolphin_user + ":" + params.dolphin_group, params.dolphin_home)) def configure(self, env): import params From 0fc49b2023c4b16e77b7472a529874ddebed89eb Mon Sep 17 00:00:00 2001 From: zhangchunyang Date: Tue, 30 Jun 2020 10:44:52 +0800 Subject: [PATCH 2/3] [bugfix] 1. fix alert.json includes repeated tag 'DOLPHIN_ALERT' 2. metainfo.xml The version number of the RPM package is no longer specified in the metainfo.xml (#3032) Co-authored-by: zhangchunyang1024 --- .../common-services/DOLPHIN/1.3.0/alerts.json | 60 +++++++++---------- .../DOLPHIN/1.3.0/metainfo.xml | 4 +- 2 files changed, 29 insertions(+), 35 deletions(-) diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/alerts.json b/ambari_plugin/common-services/DOLPHIN/1.3.0/alerts.json index 2a9800e59a..4949f7da7c 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.3.0/alerts.json +++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/alerts.json @@ -28,6 +28,33 @@ } } ], + "DOLPHIN_LOGGER": [ + { + "name": "dolphin_logger_port_check", + "label": "dolphin_logger_port_check", + "description": "dolphin_logger_port_check.", + "interval": 10, + "scope": "ANY", + "source": { + "type": "PORT", + "uri": "{{dolphin-common/loggerserver.rpc.port}}", + "default_port": 50051, + "reporting": { + "ok": { + "text": "TCP OK - {0:.3f}s response on port {1}" + }, + "warning": { + "text": "TCP OK - {0:.3f}s response on port {1}", + "value": 1.5 + }, + "critical": { + "text": "Connection failed: {0} to {1}:{2}", + "value": 5.0 + } + } + } + } + ], "DOLPHIN_MASTER": [ { "name": "DOLPHIN_MASTER_CHECK", @@ -126,39 +153,6 @@ ] } } - ], - "DOLPHIN_ALERT": [ - { - "name": "DOLPHIN_DOLPHIN_LOGGER_CHECK", - "label": "check dolphin scheduler alert status", - "description": "", - "interval":10, - "scope": "HOST", - "enabled": true, - "source": { - "type": "SCRIPT", - "path": "DOLPHIN/1.3.0/package/alerts/alert_dolphin_scheduler_status.py", - "parameters": [ - - { - "name": "connection.timeout", - "display_name": "Connection Timeout", - "value": 5.0, - "type": "NUMERIC", - "description": "The maximum time before this alert is considered to be CRITICAL", - "units": "seconds", - "threshold": "CRITICAL" - }, - { - "name": "alertName", - "display_name": "alertName", - "value": "DOLPHIN_LOGGER", - "type": "STRING", - "description": "alert name" - } - ] - } - } ] } } \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/metainfo.xml b/ambari_plugin/common-services/DOLPHIN/1.3.0/metainfo.xml index a559085f03..183221a1c2 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.3.0/metainfo.xml +++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/metainfo.xml @@ -103,7 +103,7 @@ any - apache-dolphinscheduler-incubating-1.3.0* + apache-dolphinscheduler-incubating* @@ -134,4 +134,4 @@ - + \ No newline at end of file From 8c29e2a1c0fc1603d3947657fce020b36ed26bbb Mon Sep 17 00:00:00 2001 From: zhangchunyang Date: Tue, 30 Jun 2020 11:02:25 +0800 Subject: [PATCH 3/3] [feature] readme.md adds instructions for changing environment variables (#3031) Co-authored-by: zhangchunyang1024 --- ambari_plugin/README.md | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/ambari_plugin/README.md b/ambari_plugin/README.md index 817b7abbcd..17782661b3 100644 --- a/ambari_plugin/README.md +++ b/ambari_plugin/README.md @@ -3,7 +3,8 @@ #### Note 1. This document is intended for users with a basic understanding of Ambari -2. This document is a description of adding the Dolphin Scheduler(version 1.3.0) service to the installed Ambari service +2. This document is a description of adding the Dolphin Scheduler service to the installed Ambari service +3. This document is based on version 2.5.2 of Ambari #### Installation preparation @@ -16,8 +17,12 @@ 3. Install with rpm package - Manual installation (recommended）： - - Copy the prepared RPM packages to each node of the cluster - - Execute with DS installation user: ```rpm -ivh apache-dolphinscheduler-incubating-xxx.noarch.rpm``` + - Copy the prepared RPM packages to each node of the cluster. + - Execute with DS installation user: ```rpm -ivh apache-dolphinscheduler-incubating-xxx.noarch.rpm``` + - Mysql-connector-java packaged using the default POM file will not be included. + - The RPM package was packaged in the project with the installation path of /opt/soft. + If you use mysql as the database, you need add it manually. + - Automatic installation with ambari - Each node of the cluster needs to configure the local yum source - Copy the prepared RPM packages to each node local yum source @@ -45,6 +50,7 @@ #### Ambari Install Dolphin Scheduler +- **NOTE: You have to install zookeeper first** 1. Install Dolphin Scheduler on ambari web interface @@ -62,21 +68,25 @@ ![](https://github.com/apache/incubator-dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_004.png) -5. Configure the database information (same as in the initialization database in step 1) +5. System Env Optimization will export some system environment config. Modify according to actual situation + + ![](https://github.com/apache/incubator-dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_020.png) + +6. Configure the database information (same as in the initialization database in step 1) ![](https://github.com/apache/incubator-dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_005.png) -6. Configure additional information if needed +7. Configure additional information if needed ![](https://github.com/apache/incubator-dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_006.png) ![](https://github.com/apache/incubator-dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_007.png) -7. Perform the next steps as normal +8. Perform the next steps as normal ![](https://github.com/apache/incubator-dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_008.png) -8. The interface after successful installation +9. The interface after successful installation ![](https://github.com/apache/incubator-dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_009.png) @@ -111,7 +121,6 @@ ![DS2_AMBARI_016](https://github.com/apache/incubator-dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_016.png) - #### Remove the component from the node with Ambari 1. Stop the component in the corresponding node