From 9545cb2008c92cdd887893814df8ca69c266fcc9 Mon Sep 17 00:00:00 2001 From: bao liang <29528966+lenboo@users.noreply.github.com> Date: Thu, 31 Oct 2019 18:52:20 +0800 Subject: [PATCH] add postgre performance monitor (#1137) * update english documents * refactor zk client * update documents * update zkclient * update zkclient * update documents * add architecture-design * change i18n * update i18n * update english documents * add architecture-design * update english documents * update en-US documents * add architecture-design * update demo site * add mybatis plus model * modify mybatisplus * modify mybatisplus * change interface by mybatisplus * add unit test * refactor dao interface. * add unit test for dao... * add unit test for dao... * add unit test for dao... * Merge remote-tracking branch 'upstream/dev-db' into dev-db # Conflicts: # dolphinscheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProjectMapper.xml # dolphinscheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ScheduleMapper.xml # escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProcessInstanceMapper.xml # escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProjectUserMapper.xml # escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/QueueMapper.xml # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ProcessInstanceMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ProjectUserMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/QueueMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ResourceUserMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ScheduleMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/SessionMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/TenantMapperTest.java * Merge remote-tracking branch 'upstream/dev-db' into dev-db # Conflicts: # dolphinscheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProjectMapper.xml # dolphinscheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ScheduleMapper.xml # escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProcessInstanceMapper.xml # escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProjectUserMapper.xml # escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/QueueMapper.xml # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ProcessInstanceMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ProjectUserMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/QueueMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ResourceUserMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ScheduleMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/SessionMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/TenantMapperTest.java * Merge remote-tracking branch 'upstream/dev-db' into dev-db # Conflicts: # dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.xml # dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.xml # dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml * update some dao bugs * update for some bugs * update some bugs * Merge remote-tracking branch 'upstream/dev-db' into dev-db # Conflicts: # dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.xml # dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.xml # dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml * update * update * add multiply settings for application.yml * add multiply settings for application.yml * revert * update configuration settings in task record dao... * change application_master to application-master * change application_master to application-master * update application.yml to application.properties * revert * revert * add properties * add properties * revert * revert * add api start up.. add alert send try catch * update dao info level * fix bug: task cannot submit when recovery failover * fix bug: task cannot submit when recovery failover * merge from dev-db * revert * revert * fix bug: get process definition list failed. * fix bug: process instance interval is error * revert * revert * update * support stop submit success tasks * update kill process * update for stop process * update for stop process * add some logs for stop process * update for small bug. * add check strategy before submit task * revert * update * update * revert * wait task instance exists if null. * revert * update * change desc to description. * add check user and definitions function when delete tenant * update * change desc to description. * change desc to description. * change desc to description. * remove check resources when delete tenant * change desc to description. * change mybatisplus version to 3.2.0 * update * change the notice to apache. * update * update postgre sql * fix bug: phone can be empty. * fix bug: postgre test error. * update create table for postgre quartz * fix some bugs about postgre. * update create table for postgre quartz * add postgre db performance monitor * add postgre performance monitor * update performance monitor * revert * revert --- .../dolphinscheduler/dao/MonitorDBDao.java | 117 ++++++------------ .../dao/entity/MonitorRecord.java | 20 ++- .../dao/utils/BaseDBPerformance.java | 35 ++++++ .../dao/utils/MysqlPerformance.java | 83 +++++++++++++ .../dao/utils/PostgrePerformance.java | 76 ++++++++++++ .../src/main/resources/application.properties | 7 +- 6 files changed, 252 insertions(+), 86 deletions(-) create mode 100644 dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/BaseDBPerformance.java create mode 100644 dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/MysqlPerformance.java create mode 100644 dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PostgrePerformance.java diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/MonitorDBDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/MonitorDBDao.java index 73cb5c104f..7da83c1bf7 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/MonitorDBDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/MonitorDBDao.java @@ -16,17 +16,21 @@ */ package org.apache.dolphinscheduler.dao; +import com.alibaba.druid.pool.DruidDataSource; import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.dao.datasource.ConnectionFactory; import org.apache.dolphinscheduler.dao.entity.MonitorRecord; import org.apache.commons.configuration.Configuration; import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.PropertiesConfiguration; +import org.apache.dolphinscheduler.dao.utils.MysqlPerformance; +import org.apache.dolphinscheduler.dao.utils.PostgrePerformance; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.sql.*; import java.util.ArrayList; -import java.util.Date; import java.util.List; @@ -52,99 +56,52 @@ public class MonitorDBDao { } } - - /** - * create connection - * @return - */ - private static Connection getConn() { - String url = conf.getString(Constants.SPRING_DATASOURCE_URL); - String username = conf.getString(Constants.SPRING_DATASOURCE_USERNAME); - String password = conf.getString(Constants.SPRING_DATASOURCE_PASSWORD); - Connection conn = null; - try { - //classloader,load driver - Class.forName(Constants.JDBC_MYSQL_CLASS_NAME); - conn = DriverManager.getConnection(url, username, password); - } catch (ClassNotFoundException e) { - logger.error("ClassNotFoundException ", e); - } catch (SQLException e) { - logger.error("SQLException ", e); - } - return conn; - } - - - /** - * query database state - * @return - */ - public static List queryDatabaseState() { - List list = new ArrayList<>(1); - + public static MonitorRecord getCurrentDbPerformance(){ + MonitorRecord monitorRecord = null; Connection conn = null; - Statement pstmt = null; - long maxConnections = 0; - long maxUsedConnections = 0; - long threadsConnections = 0; - long threadsRunningConnections = 0; - //mysql running state - int state = 1; - - - MonitorRecord monitorRecord = new MonitorRecord(); - try { - conn = getConn(); + DruidDataSource dataSource = null; + try{ + dataSource = ConnectionFactory.getDataSource(); + dataSource.setInitialSize(2); + dataSource.setMinIdle(2); + dataSource.setMaxActive(2); + conn = dataSource.getConnection(); if(conn == null){ - return list; + return monitorRecord; } - - pstmt = conn.createStatement(); - - ResultSet rs1 = pstmt.executeQuery("show global variables"); - while(rs1.next()){ - if(rs1.getString(VARIABLE_NAME).toUpperCase().equals("MAX_CONNECTIONS")){ - maxConnections= Long.parseLong(rs1.getString("value")); - } - } - - ResultSet rs2 = pstmt.executeQuery("show global status"); - while(rs2.next()){ - if(rs2.getString(VARIABLE_NAME).toUpperCase().equals("MAX_USED_CONNECTIONS")){ - maxUsedConnections = Long.parseLong(rs2.getString("value")); - }else if(rs2.getString(VARIABLE_NAME).toUpperCase().equals("THREADS_CONNECTED")){ - threadsConnections = Long.parseLong(rs2.getString("value")); - }else if(rs2.getString(VARIABLE_NAME).toUpperCase().equals("THREADS_RUNNING")){ - threadsRunningConnections= Long.parseLong(rs2.getString("value")); - } + if(conf.getString(Constants.SPRING_DATASOURCE_DRIVER_CLASS_NAME).contains(DbType.MYSQL.toString().toLowerCase())) { + return new MysqlPerformance().getMonitorRecord(conn); + } else if(conf.getString(Constants.SPRING_DATASOURCE_DRIVER_CLASS_NAME).contains(DbType.POSTGRESQL.toString().toLowerCase())){ + return new PostgrePerformance().getMonitorRecord(conn); } - - - } catch (SQLException e) { - logger.error("SQLException ", e); - state = 0; + }catch (Exception e) { + logger.error("SQLException " + e); }finally { try { - if(pstmt != null) { - pstmt.close(); - } - if(conn != null){ + if (conn != null) { conn.close(); } + if(dataSource != null){ + dataSource.close(); + } } catch (SQLException e) { logger.error("SQLException ", e); } } + return monitorRecord; + } - monitorRecord.setDate(new Date()); - monitorRecord.setMaxConnections(maxConnections); - monitorRecord.setMaxUsedConnections(maxUsedConnections); - monitorRecord.setThreadsConnections(threadsConnections); - monitorRecord.setThreadsRunningConnections(threadsRunningConnections); - monitorRecord.setState(state); - - list.add(monitorRecord); + /** + * query database state + * @return + */ + public static List queryDatabaseState() { + List list = new ArrayList<>(1); + MonitorRecord monitorRecord = getCurrentDbPerformance(); + if(monitorRecord != null){ + list.add(monitorRecord); + } return list; } } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/MonitorRecord.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/MonitorRecord.java index 23b17a37d5..259827ed80 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/MonitorRecord.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/MonitorRecord.java @@ -16,6 +16,9 @@ */ package org.apache.dolphinscheduler.dao.entity; +import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.common.enums.Flag; + import java.util.Date; /** @@ -23,10 +26,12 @@ import java.util.Date; */ public class MonitorRecord { + private DbType dbType; + /** * is normal or not , 1: normal */ - private int state; + private Flag state; /** * max connections @@ -53,11 +58,11 @@ public class MonitorRecord { */ private Date date; - public int getState() { + public Flag getState() { return state; } - public void setState(int state) { + public void setState(Flag state) { this.state = state; } @@ -105,6 +110,7 @@ public class MonitorRecord { public String toString() { return "MonitorRecord{" + "state=" + state + + ", dbType=" + dbType + ", maxConnections=" + maxConnections + ", maxUsedConnections=" + maxUsedConnections + ", threadsConnections=" + threadsConnections + @@ -112,4 +118,12 @@ public class MonitorRecord { ", date=" + date + '}'; } + + public DbType getDbType() { + return dbType; + } + + public void setDbType(DbType dbType) { + this.dbType = dbType; + } } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/BaseDBPerformance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/BaseDBPerformance.java new file mode 100644 index 0000000000..aa077aad83 --- /dev/null +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/BaseDBPerformance.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.dao.utils; + +import org.apache.dolphinscheduler.dao.entity.MonitorRecord; + +import java.sql.Connection; + +/** + * + */ +public abstract class BaseDBPerformance { + + + /** + * return the current database performance + * @return + */ + protected abstract MonitorRecord getMonitorRecord(Connection conn); + +} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/MysqlPerformance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/MysqlPerformance.java new file mode 100644 index 0000000000..7b264a4432 --- /dev/null +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/MysqlPerformance.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.dao.utils; + + +import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.common.enums.Flag; +import org.apache.dolphinscheduler.dao.MonitorDBDao; +import org.apache.dolphinscheduler.dao.entity.MonitorRecord; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Date; + +import static org.apache.dolphinscheduler.dao.MonitorDBDao.VARIABLE_NAME; + +public class MysqlPerformance extends BaseDBPerformance{ + + private static Logger logger = LoggerFactory.getLogger(MonitorDBDao.class); + + + @Override + public MonitorRecord getMonitorRecord(Connection conn) { + MonitorRecord monitorRecord = new MonitorRecord(); + monitorRecord.setDate(new Date()); + monitorRecord.setDbType(DbType.MYSQL); + monitorRecord.setState(Flag.YES); + Statement pstmt= null; + try{ + pstmt = conn.createStatement(); + + ResultSet rs1 = pstmt.executeQuery("show global variables"); + while(rs1.next()){ + if(rs1.getString(VARIABLE_NAME).toUpperCase().equals("MAX_CONNECTIONS")){ + monitorRecord.setMaxConnections( Long.parseLong(rs1.getString("value"))); + } + } + + ResultSet rs2 = pstmt.executeQuery("show global status"); + while(rs2.next()){ + if(rs2.getString(VARIABLE_NAME).toUpperCase().equals("MAX_USED_CONNECTIONS")){ + monitorRecord.setMaxUsedConnections(Long.parseLong(rs2.getString("value"))); + }else if(rs2.getString(VARIABLE_NAME).toUpperCase().equals("THREADS_CONNECTED")){ + monitorRecord.setThreadsConnections(Long.parseLong(rs2.getString("value"))); + }else if(rs2.getString(VARIABLE_NAME).toUpperCase().equals("THREADS_RUNNING")){ + monitorRecord.setThreadsRunningConnections(Long.parseLong(rs2.getString("value"))); + } + } + }catch (Exception e) { + monitorRecord.setState(Flag.NO); + logger.error("SQLException " + e); + }finally { + try { + if (pstmt != null) { + pstmt.close(); + } + }catch (SQLException e) { + logger.error("SQLException ", e); + } + } + return monitorRecord; + } + + +} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PostgrePerformance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PostgrePerformance.java new file mode 100644 index 0000000000..b203ce719f --- /dev/null +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PostgrePerformance.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.dao.utils; + +import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.common.enums.Flag; +import org.apache.dolphinscheduler.dao.MonitorDBDao; +import org.apache.dolphinscheduler.dao.entity.MonitorRecord; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Date; + +public class PostgrePerformance extends BaseDBPerformance { + + private static Logger logger = LoggerFactory.getLogger(MonitorDBDao.class); + + @Override + public MonitorRecord getMonitorRecord(Connection conn) { + MonitorRecord monitorRecord = new MonitorRecord(); + monitorRecord.setDate(new Date()); + monitorRecord.setState(Flag.YES); + monitorRecord.setDbType(DbType.POSTGRESQL); + Statement pstmt= null; + try{ + pstmt = conn.createStatement(); + ResultSet rs1 = pstmt.executeQuery("select count(*) from pg_stat_activity;"); + while(rs1.next()){ + monitorRecord.setThreadsConnections(rs1.getInt("count")); + break; + } + + ResultSet rs2 = pstmt.executeQuery("show max_connections"); + while(rs2.next()){ + monitorRecord.setMaxConnections( rs2.getInt("max_connections")); + break; + } + + ResultSet rs3 = pstmt.executeQuery("select count(*) from pg_stat_activity pg where pg.state = 'active';"); + while(rs3.next()){ + monitorRecord.setThreadsRunningConnections(rs3.getInt("count")); + break; + } + }catch (Exception e) { + monitorRecord.setState(Flag.NO); + logger.error("SQLException " + e); + }finally { + try { + if (pstmt != null) { + pstmt.close(); + } + }catch (SQLException e) { + logger.error("SQLException ", e); + } + } + return monitorRecord; + } +} diff --git a/dolphinscheduler-dao/src/main/resources/application.properties b/dolphinscheduler-dao/src/main/resources/application.properties index 9f26be6079..e482e33bd7 100644 --- a/dolphinscheduler-dao/src/main/resources/application.properties +++ b/dolphinscheduler-dao/src/main/resources/application.properties @@ -1,8 +1,9 @@ -# mysql -# url=jdbc:postgresql://192.168.220.154:5432/dolphinscheduler # base spring data source configuration spring.datasource.type=com.alibaba.druid.pool.DruidDataSource +# postgre #spring.datasource.driver-class-name=org.postgresql.Driver +#spring.datasource.url=jdbc:postgresql://192.168.220.154:5432/dolphinscheduler +# mysql spring.datasource.driver-class-name=com.mysql.jdbc.Driver spring.datasource.url=jdbc:mysql://192.168.220.188:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 spring.datasource.username=root @@ -83,7 +84,7 @@ mybatis-plus.configuration.call-setters-on-nulls=true mybatis-plus.configuration.jdbc-type-for-null=null # data quality analysis is not currently in use. please ignore the following configuration -# task record flag +# task record task.record.flag=false task.record.datasource.url=jdbc:mysql://192.168.xx.xx:3306/etl?characterEncoding=UTF-8 task.record.datasource.username=xx