From 33084d281f303701b345051f306fb505f78a0152 Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Sat, 14 Oct 2023 17:58:40 +0800 Subject: [PATCH] Add dolphinscheduler-dao-plugin module (#15019) --- .../src/main/resources/application.yaml | 2 + dolphinscheduler-api/pom.xml | 6 ++ .../api/service/impl/MonitorServiceImpl.java | 9 +-- .../src/main/resources/application.yaml | 2 + .../api/service/MonitorServiceTest.java | 20 ++--- .../src/test/resources/application.yaml | 7 +- .../common/sql/ClasspathSqlScriptParser.java | 8 ++ .../dolphinscheduler-dao-api/pom.xml | 46 +++++++++++ .../plugin/api/DaoPluginConfiguration.java | 36 +++++++++ .../plugin/api/monitor/DatabaseMetrics.java | 55 +++++++++++++ .../plugin/api/monitor/DatabaseMonitor.java | 27 +++++++ .../dolphinscheduler-dao-h2/pom.xml | 42 ++++++++++ .../plugin/h2/H2DaoPluginConfiguration.java | 52 ++++++++++++ .../dao/plugin/h2/monitor/H2Monitor.java | 71 +++++++++++++++++ .../dolphinscheduler-dao-mysql/pom.xml | 41 ++++++++++ .../mysql/MysqlDaoPluginConfiguration.java | 50 ++++++++++++ .../plugin/mysql/monitor/MysqlMonitor.java | 79 +++++++++++++++++++ .../dolphinscheduler-dao-plugin-all/pom.xml | 56 +++++++++++++ .../dolphinscheduler-dao-postgresql/pom.xml | 41 ++++++++++ .../PostgresqlDaoPluginConfiguration.java | 51 ++++++++++++ .../postgresql/monitor/PostgresqlMonitor.java | 79 +++++++++++++++++++ dolphinscheduler-dao-plugin/pom.xml | 50 ++++++++++++ dolphinscheduler-dao/pom.xml | 19 ++--- .../dolphinscheduler/dao/MonitorDBDao.java | 77 ------------------ .../datasource/SpringConnectionFactory.java | 47 ++++------- .../dao/utils/H2Performance.java | 65 --------------- .../dao/utils/MySQLPerformance.java | 78 ------------------ .../dao/utils/PostgreSQLPerformance.java | 73 ----------------- .../src/test/resources/application.yaml | 7 +- dolphinscheduler-dist/release-docs/LICENSE | 2 +- .../src/main/resources/application.yaml | 2 + dolphinscheduler-standalone-server/pom.xml | 5 -- .../src/main/resources/application.yaml | 2 + pom.xml | 31 ++++++++ tools/dependencies/known-dependencies.txt | 2 - 35 files changed, 875 insertions(+), 365 deletions(-) create mode 100644 dolphinscheduler-dao-plugin/dolphinscheduler-dao-api/pom.xml create mode 100644 dolphinscheduler-dao-plugin/dolphinscheduler-dao-api/src/main/java/org/apache/dolphinscheduler/dao/plugin/api/DaoPluginConfiguration.java create mode 100644 dolphinscheduler-dao-plugin/dolphinscheduler-dao-api/src/main/java/org/apache/dolphinscheduler/dao/plugin/api/monitor/DatabaseMetrics.java create mode 100644 dolphinscheduler-dao-plugin/dolphinscheduler-dao-api/src/main/java/org/apache/dolphinscheduler/dao/plugin/api/monitor/DatabaseMonitor.java create mode 100644 dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/pom.xml create mode 100644 dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/H2DaoPluginConfiguration.java create mode 100644 dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/monitor/H2Monitor.java create mode 100644 dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/pom.xml create mode 100644 dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/MysqlDaoPluginConfiguration.java create mode 100644 dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/monitor/MysqlMonitor.java create mode 100644 dolphinscheduler-dao-plugin/dolphinscheduler-dao-plugin-all/pom.xml create mode 100644 dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/pom.xml create mode 100644 dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/PostgresqlDaoPluginConfiguration.java create mode 100644 dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/monitor/PostgresqlMonitor.java create mode 100644 dolphinscheduler-dao-plugin/pom.xml delete mode 100644 dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/MonitorDBDao.java delete mode 100644 dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/H2Performance.java delete mode 100644 dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/MySQLPerformance.java delete mode 100644 dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PostgreSQLPerformance.java diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml index a82718ec23..5ea68f9588 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml @@ -16,6 +16,8 @@ # spring: + profiles: + active: postgresql jackson: time-zone: UTC date-format: "yyyy-MM-dd HH:mm:ss" diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index 467826bc41..40fe41c190 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -50,6 +50,12 @@ + + + org.apache.dolphinscheduler + dolphinscheduler-dao + + org.apache.dolphinscheduler dolphinscheduler-meter diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/MonitorServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/MonitorServiceImpl.java index 945fabe72f..5ab42d3089 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/MonitorServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/MonitorServiceImpl.java @@ -22,9 +22,8 @@ import org.apache.dolphinscheduler.api.service.MonitorService; import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.model.Server; import org.apache.dolphinscheduler.common.model.WorkerServerModel; -import org.apache.dolphinscheduler.dao.MonitorDBDao; -import org.apache.dolphinscheduler.dao.entity.MonitorRecord; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.plugin.api.monitor.DatabaseMonitor; import org.apache.dolphinscheduler.registry.api.RegistryClient; import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType; @@ -39,6 +38,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import com.google.common.collect.Lists; import com.google.common.collect.Sets; /** @@ -49,7 +49,7 @@ import com.google.common.collect.Sets; public class MonitorServiceImpl extends BaseServiceImpl implements MonitorService { @Autowired - private MonitorDBDao monitorDBDao; + private DatabaseMonitor databaseMonitor; @Autowired private RegistryClient registryClient; @@ -63,8 +63,7 @@ public class MonitorServiceImpl extends BaseServiceImpl implements MonitorServic @Override public Map queryDatabaseState(User loginUser) { Map result = new HashMap<>(); - List monitorRecordList = monitorDBDao.queryDatabaseState(); - result.put(Constants.DATA_LIST, monitorRecordList); + result.put(Constants.DATA_LIST, Lists.newArrayList(databaseMonitor.getDatabaseMetrics())); putMsg(result, Status.SUCCESS); return result; } diff --git a/dolphinscheduler-api/src/main/resources/application.yaml b/dolphinscheduler-api/src/main/resources/application.yaml index 798fa6d6fc..e58983f0a3 100644 --- a/dolphinscheduler-api/src/main/resources/application.yaml +++ b/dolphinscheduler-api/src/main/resources/application.yaml @@ -31,6 +31,8 @@ server: custom-format: '%{client}a - %u %t "%r" %s %O %{ms}Tms' spring: + profiles: + active: postgresql banner: charset: UTF-8 jackson: diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/MonitorServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/MonitorServiceTest.java index 3aba92d24b..54808a0290 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/MonitorServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/MonitorServiceTest.java @@ -26,12 +26,12 @@ import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.enums.AuthorizationType; import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.model.Server; -import org.apache.dolphinscheduler.dao.MonitorDBDao; import org.apache.dolphinscheduler.dao.entity.MonitorRecord; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.plugin.api.monitor.DatabaseMetrics; +import org.apache.dolphinscheduler.dao.plugin.api.monitor.DatabaseMonitor; import org.apache.dolphinscheduler.registry.api.RegistryClient; import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType; -import org.apache.dolphinscheduler.spi.enums.DbType; import org.apache.commons.collections4.CollectionUtils; @@ -53,6 +53,8 @@ import org.mockito.quality.Strictness; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.baomidou.mybatisplus.annotation.DbType; + /** * monitor service test */ @@ -66,7 +68,7 @@ public class MonitorServiceTest { private MonitorServiceImpl monitorService; @Mock - private MonitorDBDao monitorDBDao; + private DatabaseMonitor databaseMonitor; @Mock private ResourcePermissionCheckService resourcePermissionCheckService; @@ -88,7 +90,7 @@ public class MonitorServiceTest { @Test public void testQueryDatabaseState() { mockPermissionCheck(ApiFuncIdentificationConstant.MONITOR_DATABASES_VIEW, true); - Mockito.when(monitorDBDao.queryDatabaseState()).thenReturn(getList()); + Mockito.when(databaseMonitor.getDatabaseMetrics()).thenReturn(getDatabaseMetrics()); Map result = monitorService.queryDatabaseState(user); logger.info(result.toString()); Assertions.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); @@ -138,14 +140,8 @@ public class MonitorServiceTest { serviceLogger)).thenReturn(true); } - private List getList() { - List monitorRecordList = new ArrayList<>(); - monitorRecordList.add(getEntity()); - return monitorRecordList; - } - - private MonitorRecord getEntity() { - MonitorRecord monitorRecord = new MonitorRecord(); + private DatabaseMetrics getDatabaseMetrics() { + DatabaseMetrics monitorRecord = new DatabaseMetrics(); monitorRecord.setDbType(DbType.MYSQL); return monitorRecord; } diff --git a/dolphinscheduler-api/src/test/resources/application.yaml b/dolphinscheduler-api/src/test/resources/application.yaml index cdd4f16eb0..b11c16d362 100644 --- a/dolphinscheduler-api/src/test/resources/application.yaml +++ b/dolphinscheduler-api/src/test/resources/application.yaml @@ -16,11 +16,16 @@ # spring: + profiles: + active: h2 main: banner-mode: off + sql: + init: + schema-locations: classpath:sql/dolphinscheduler_h2.sql datasource: driver-class-name: org.h2.Driver - url: jdbc:h2:mem:dolphinscheduler;MODE=MySQL;DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=true;INIT=runscript from 'classpath:sql/dolphinscheduler_h2.sql' + url: jdbc:h2:mem:dolphinscheduler;MODE=MySQL;DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=true; username: sa password: "" diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/sql/ClasspathSqlScriptParser.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/sql/ClasspathSqlScriptParser.java index 3b4b3765f3..e0e367747b 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/sql/ClasspathSqlScriptParser.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/sql/ClasspathSqlScriptParser.java @@ -27,11 +27,15 @@ import java.io.Reader; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import lombok.extern.slf4j.Slf4j; + import org.springframework.core.io.ClassPathResource; import org.springframework.core.io.Resource; +@Slf4j public class ClasspathSqlScriptParser implements SqlScriptParser { private final String sqlScriptPath; @@ -46,6 +50,10 @@ public class ClasspathSqlScriptParser implements SqlScriptParser { @Override public List getAllSql() throws IOException { Resource sqlScriptResource = new ClassPathResource(sqlScriptPath); + if (!sqlScriptResource.exists()) { + log.warn("The sql script file {} doesn't exist", sqlScriptPath); + return Collections.emptyList(); + } List result = new ArrayList<>(); try ( InputStream inputStream = sqlScriptResource.getInputStream(); diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-api/pom.xml b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-api/pom.xml new file mode 100644 index 0000000000..d8fad0b6a0 --- /dev/null +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-api/pom.xml @@ -0,0 +1,46 @@ + + + + 4.0.0 + + org.apache.dolphinscheduler + dolphinscheduler-dao-plugin + dev-SNAPSHOT + + + dolphinscheduler-dao-api + + + + + org.springframework.boot + spring-boot-autoconfigure + + + + org.springframework + spring-context + + + + com.baomidou + mybatis-plus + + + diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-api/src/main/java/org/apache/dolphinscheduler/dao/plugin/api/DaoPluginConfiguration.java b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-api/src/main/java/org/apache/dolphinscheduler/dao/plugin/api/DaoPluginConfiguration.java new file mode 100644 index 0000000000..ba3577f10e --- /dev/null +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-api/src/main/java/org/apache/dolphinscheduler/dao/plugin/api/DaoPluginConfiguration.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +package org.apache.dolphinscheduler.dao.plugin.api; + +import org.apache.dolphinscheduler.dao.plugin.api.monitor.DatabaseMonitor; + +import com.baomidou.mybatisplus.annotation.DbType; + +/** + * DaoPluginConfiguration used to configure the dao plugin. + */ +public interface DaoPluginConfiguration { + + DbType dbType(); + + DatabaseMonitor databaseMonitor(); + +} diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-api/src/main/java/org/apache/dolphinscheduler/dao/plugin/api/monitor/DatabaseMetrics.java b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-api/src/main/java/org/apache/dolphinscheduler/dao/plugin/api/monitor/DatabaseMetrics.java new file mode 100644 index 0000000000..fbe967bd8a --- /dev/null +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-api/src/main/java/org/apache/dolphinscheduler/dao/plugin/api/monitor/DatabaseMetrics.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +package org.apache.dolphinscheduler.dao.plugin.api.monitor; + +import java.util.Date; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import com.baomidou.mybatisplus.annotation.DbType; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class DatabaseMetrics { + + private DbType dbType; + + private DatabaseHealthStatus state; + + private long maxConnections; + + private long maxUsedConnections; + + private long threadsConnections; + + private long threadsRunningConnections; + + private Date date; + + public enum DatabaseHealthStatus { + YES, + NO + } + +} diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-api/src/main/java/org/apache/dolphinscheduler/dao/plugin/api/monitor/DatabaseMonitor.java b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-api/src/main/java/org/apache/dolphinscheduler/dao/plugin/api/monitor/DatabaseMonitor.java new file mode 100644 index 0000000000..e254d00968 --- /dev/null +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-api/src/main/java/org/apache/dolphinscheduler/dao/plugin/api/monitor/DatabaseMonitor.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +package org.apache.dolphinscheduler.dao.plugin.api.monitor; + +public interface DatabaseMonitor { + + DatabaseMetrics getDatabaseMetrics(); + +} diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/pom.xml b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/pom.xml new file mode 100644 index 0000000000..1adebd3174 --- /dev/null +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/pom.xml @@ -0,0 +1,42 @@ + + + + 4.0.0 + + org.apache.dolphinscheduler + dolphinscheduler-dao-plugin + dev-SNAPSHOT + + + dolphinscheduler-dao-h2 + + + + + org.apache.dolphinscheduler + dolphinscheduler-dao-api + + + + com.h2database + h2 + + + + diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/H2DaoPluginConfiguration.java b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/H2DaoPluginConfiguration.java new file mode 100644 index 0000000000..dca8b29a04 --- /dev/null +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/H2DaoPluginConfiguration.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +package org.apache.dolphinscheduler.dao.plugin.h2; + +import org.apache.dolphinscheduler.dao.plugin.api.DaoPluginConfiguration; +import org.apache.dolphinscheduler.dao.plugin.api.monitor.DatabaseMonitor; +import org.apache.dolphinscheduler.dao.plugin.h2.monitor.H2Monitor; + +import javax.sql.DataSource; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Profile; + +import com.baomidou.mybatisplus.annotation.DbType; + +@Profile("h2") +@Configuration +public class H2DaoPluginConfiguration implements DaoPluginConfiguration { + + @Autowired + private DataSource dataSource; + + @Override + public DbType dbType() { + return DbType.H2; + } + + @Override + public DatabaseMonitor databaseMonitor() { + return new H2Monitor(dataSource); + } + +} diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/monitor/H2Monitor.java b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/monitor/H2Monitor.java new file mode 100644 index 0000000000..74ecea100d --- /dev/null +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/monitor/H2Monitor.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +package org.apache.dolphinscheduler.dao.plugin.h2.monitor; + +import org.apache.dolphinscheduler.dao.plugin.api.monitor.DatabaseMetrics; +import org.apache.dolphinscheduler.dao.plugin.api.monitor.DatabaseMonitor; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.Date; + +import javax.sql.DataSource; + +import lombok.SneakyThrows; + +import com.baomidou.mybatisplus.annotation.DbType; + +public class H2Monitor implements DatabaseMonitor { + + private final DataSource dataSource; + + public H2Monitor(DataSource dataSource) { + this.dataSource = dataSource; + } + + @SneakyThrows + @Override + public DatabaseMetrics getDatabaseMetrics() { + DatabaseMetrics monitorRecord = new DatabaseMetrics(); + monitorRecord.setDate(new Date()); + monitorRecord.setDbType(DbType.H2); + monitorRecord.setState(DatabaseMetrics.DatabaseHealthStatus.YES); + + try ( + Connection connection = dataSource.getConnection(); + Statement pstmt = connection.createStatement()) { + + try ( + ResultSet rs1 = pstmt + .executeQuery("select count(1) as total from information_schema.sessions;")) { + if (rs1.next()) { + int currentSessions = rs1.getInt("total"); + monitorRecord.setThreadsConnections(currentSessions); + monitorRecord.setMaxUsedConnections(currentSessions); + } + } + + } + return monitorRecord; + } + +} diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/pom.xml b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/pom.xml new file mode 100644 index 0000000000..a05537909e --- /dev/null +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/pom.xml @@ -0,0 +1,41 @@ + + + + 4.0.0 + + org.apache.dolphinscheduler + dolphinscheduler-dao-plugin + dev-SNAPSHOT + + + dolphinscheduler-dao-mysql + + + + org.apache.dolphinscheduler + dolphinscheduler-dao-api + + + + mysql + mysql-connector-java + + + + diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/MysqlDaoPluginConfiguration.java b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/MysqlDaoPluginConfiguration.java new file mode 100644 index 0000000000..25f6c34374 --- /dev/null +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/MysqlDaoPluginConfiguration.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.dolphinscheduler.dao.plugin.mysql; + +import org.apache.dolphinscheduler.dao.plugin.api.DaoPluginConfiguration; +import org.apache.dolphinscheduler.dao.plugin.api.monitor.DatabaseMonitor; +import org.apache.dolphinscheduler.dao.plugin.mysql.monitor.MysqlMonitor; + +import javax.sql.DataSource; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Profile; + +import com.baomidou.mybatisplus.annotation.DbType; + +@Profile("mysql") +@Configuration +public class MysqlDaoPluginConfiguration implements DaoPluginConfiguration { + + @Autowired + private DataSource dataSource; + + @Override + public DbType dbType() { + return DbType.MYSQL; + } + + @Override + public DatabaseMonitor databaseMonitor() { + return new MysqlMonitor(dataSource); + } +} diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/monitor/MysqlMonitor.java b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/monitor/MysqlMonitor.java new file mode 100644 index 0000000000..8a48057568 --- /dev/null +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/monitor/MysqlMonitor.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +package org.apache.dolphinscheduler.dao.plugin.mysql.monitor; + +import org.apache.dolphinscheduler.dao.plugin.api.monitor.DatabaseMetrics; +import org.apache.dolphinscheduler.dao.plugin.api.monitor.DatabaseMonitor; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.Date; + +import javax.sql.DataSource; + +import lombok.SneakyThrows; + +import com.baomidou.mybatisplus.annotation.DbType; + +public class MysqlMonitor implements DatabaseMonitor { + + private final DataSource dataSource; + + public MysqlMonitor(DataSource dataSource) { + this.dataSource = dataSource; + } + + @SneakyThrows + @Override + public DatabaseMetrics getDatabaseMetrics() { + DatabaseMetrics monitorRecord = new DatabaseMetrics(); + monitorRecord.setDate(new Date()); + monitorRecord.setDbType(DbType.MYSQL); + monitorRecord.setState(DatabaseMetrics.DatabaseHealthStatus.YES); + + try ( + Connection connection = dataSource.getConnection(); + Statement pstmt = connection.createStatement()) { + + try (ResultSet rs1 = pstmt.executeQuery("show global variables")) { + while (rs1.next()) { + if ("MAX_CONNECTIONS".equalsIgnoreCase(rs1.getString("variable_name"))) { + monitorRecord.setMaxConnections(Long.parseLong(rs1.getString("value"))); + } + } + } + + try (ResultSet rs2 = pstmt.executeQuery("show global status")) { + while (rs2.next()) { + if ("MAX_USED_CONNECTIONS".equalsIgnoreCase(rs2.getString("variable_name"))) { + monitorRecord.setMaxUsedConnections(Long.parseLong(rs2.getString("value"))); + } else if ("THREADS_CONNECTED".equalsIgnoreCase(rs2.getString("variable_name"))) { + monitorRecord.setThreadsConnections(Long.parseLong(rs2.getString("value"))); + } else if ("THREADS_RUNNING".equalsIgnoreCase(rs2.getString("variable_name"))) { + monitorRecord.setThreadsRunningConnections(Long.parseLong(rs2.getString("value"))); + } + } + } + } + return monitorRecord; + } +} diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-plugin-all/pom.xml b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-plugin-all/pom.xml new file mode 100644 index 0000000000..429336ec4e --- /dev/null +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-plugin-all/pom.xml @@ -0,0 +1,56 @@ + + + + 4.0.0 + + org.apache.dolphinscheduler + dolphinscheduler-dao-plugin + dev-SNAPSHOT + + + dolphinscheduler-dao-plugin-all + + + + org.apache.dolphinscheduler + dolphinscheduler-dao-api + ${project.version} + + + + org.apache.dolphinscheduler + dolphinscheduler-dao-h2 + ${project.version} + + + + org.apache.dolphinscheduler + dolphinscheduler-dao-postgresql + ${project.version} + + + + org.apache.dolphinscheduler + dolphinscheduler-dao-mysql + ${project.version} + + + + + diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/pom.xml b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/pom.xml new file mode 100644 index 0000000000..51489f5f75 --- /dev/null +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/pom.xml @@ -0,0 +1,41 @@ + + + + 4.0.0 + + org.apache.dolphinscheduler + dolphinscheduler-dao-plugin + dev-SNAPSHOT + + + dolphinscheduler-dao-postgresql + + + + org.apache.dolphinscheduler + dolphinscheduler-dao-api + + + + org.postgresql + postgresql + + + + diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/PostgresqlDaoPluginConfiguration.java b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/PostgresqlDaoPluginConfiguration.java new file mode 100644 index 0000000000..ee9c290f36 --- /dev/null +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/PostgresqlDaoPluginConfiguration.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +package org.apache.dolphinscheduler.dao.plugin.postgresql; + +import org.apache.dolphinscheduler.dao.plugin.api.DaoPluginConfiguration; +import org.apache.dolphinscheduler.dao.plugin.api.monitor.DatabaseMonitor; +import org.apache.dolphinscheduler.dao.plugin.postgresql.monitor.PostgresqlMonitor; + +import javax.sql.DataSource; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Profile; + +import com.baomidou.mybatisplus.annotation.DbType; + +@Profile("postgresql") +@Configuration +public class PostgresqlDaoPluginConfiguration implements DaoPluginConfiguration { + + @Autowired + private DataSource dataSource; + + @Override + public DbType dbType() { + return DbType.POSTGRE_SQL; + } + + @Override + public DatabaseMonitor databaseMonitor() { + return new PostgresqlMonitor(dataSource); + } +} diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/monitor/PostgresqlMonitor.java b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/monitor/PostgresqlMonitor.java new file mode 100644 index 0000000000..1446147826 --- /dev/null +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/monitor/PostgresqlMonitor.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +package org.apache.dolphinscheduler.dao.plugin.postgresql.monitor; + +import org.apache.dolphinscheduler.dao.plugin.api.monitor.DatabaseMetrics; +import org.apache.dolphinscheduler.dao.plugin.api.monitor.DatabaseMonitor; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.Date; + +import javax.sql.DataSource; + +import lombok.SneakyThrows; + +import com.baomidou.mybatisplus.annotation.DbType; + +public class PostgresqlMonitor implements DatabaseMonitor { + + private final DataSource dataSource; + + public PostgresqlMonitor(DataSource dataSource) { + this.dataSource = dataSource; + } + + @Override + @SneakyThrows + public DatabaseMetrics getDatabaseMetrics() { + DatabaseMetrics monitorRecord = new DatabaseMetrics(); + monitorRecord.setDate(new Date()); + monitorRecord.setState(DatabaseMetrics.DatabaseHealthStatus.YES); + monitorRecord.setDbType(DbType.POSTGRE_SQL); + + try ( + Connection connection = dataSource.getConnection(); + Statement pstmt = connection.createStatement()) { + + try (ResultSet rs1 = pstmt.executeQuery("select count(*) from pg_stat_activity;")) { + if (rs1.next()) { + monitorRecord.setThreadsConnections(rs1.getInt("count")); + } + } + + try (ResultSet rs2 = pstmt.executeQuery("show max_connections")) { + if (rs2.next()) { + monitorRecord.setMaxConnections(rs2.getInt("max_connections")); + } + } + + try ( + ResultSet rs3 = + pstmt.executeQuery("select count(*) from pg_stat_activity pg where pg.state = 'active';")) { + if (rs3.next()) { + monitorRecord.setThreadsRunningConnections(rs3.getInt("count")); + } + } + } + return monitorRecord; + } +} diff --git a/dolphinscheduler-dao-plugin/pom.xml b/dolphinscheduler-dao-plugin/pom.xml new file mode 100644 index 0000000000..ad72272261 --- /dev/null +++ b/dolphinscheduler-dao-plugin/pom.xml @@ -0,0 +1,50 @@ + + + + 4.0.0 + + org.apache.dolphinscheduler + dolphinscheduler + dev-SNAPSHOT + + + dolphinscheduler-dao-plugin + pom + + + dolphinscheduler-dao-api + dolphinscheduler-dao-plugin-all + dolphinscheduler-dao-h2 + dolphinscheduler-dao-mysql + dolphinscheduler-dao-postgresql + + + + + + org.apache.dolphinscheduler + dolphinscheduler-bom + ${project.version} + pom + import + + + + + diff --git a/dolphinscheduler-dao/pom.xml b/dolphinscheduler-dao/pom.xml index 2c2fdb5b6c..f46817c0cc 100644 --- a/dolphinscheduler-dao/pom.xml +++ b/dolphinscheduler-dao/pom.xml @@ -50,6 +50,11 @@ dolphinscheduler-task-api + + org.apache.dolphinscheduler + dolphinscheduler-dao-plugin-all + + org.springframework.boot spring-boot-starter @@ -83,15 +88,6 @@ - - org.postgresql - postgresql - - - - mysql - mysql-connector-java - com.fasterxml.jackson.core @@ -107,11 +103,6 @@ spring-boot-starter-test test - - com.h2database - h2 - test - org.apache.dolphinscheduler dolphinscheduler-storage-api diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/MonitorDBDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/MonitorDBDao.java deleted file mode 100644 index 54b9809eea..0000000000 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/MonitorDBDao.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.dao; - -import org.apache.dolphinscheduler.dao.entity.MonitorRecord; -import org.apache.dolphinscheduler.dao.utils.H2Performance; -import org.apache.dolphinscheduler.dao.utils.MySQLPerformance; -import org.apache.dolphinscheduler.dao.utils.PostgreSQLPerformance; -import org.apache.dolphinscheduler.spi.enums.DbType; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.util.ArrayList; -import java.util.List; - -import javax.sql.DataSource; - -import lombok.extern.slf4j.Slf4j; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -@Component -@Slf4j -public class MonitorDBDao { - - public static final String VARIABLE_NAME = "variable_name"; - - @Autowired - private DataSource dataSource; - - private MonitorRecord getCurrentDbPerformance() { - try (final Connection conn = dataSource.getConnection()) { - String driverClassName = DriverManager.getDriver(conn.getMetaData().getURL()).getClass().getName(); - if (driverClassName.contains(DbType.MYSQL.toString().toLowerCase())) { - return new MySQLPerformance().getMonitorRecord(conn); - } else if (driverClassName.contains(DbType.POSTGRESQL.toString().toLowerCase())) { - return new PostgreSQLPerformance().getMonitorRecord(conn); - } else if (driverClassName.contains(DbType.H2.toString().toLowerCase())) { - return new H2Performance().getMonitorRecord(conn); - } - } catch (Exception e) { - log.error("SQLException: {}", e.getMessage(), e); - } - return null; - } - - /** - * query database state - * - * @return MonitorRecord list - */ - public List queryDatabaseState() { - List list = new ArrayList<>(1); - - MonitorRecord monitorRecord = getCurrentDbPerformance(); - if (monitorRecord != null) { - list.add(monitorRecord); - } - return list; - } -} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SpringConnectionFactory.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SpringConnectionFactory.java index 51b198a1ff..c640761350 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SpringConnectionFactory.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SpringConnectionFactory.java @@ -17,21 +17,18 @@ package org.apache.dolphinscheduler.dao.datasource; -import org.apache.ibatis.mapping.DatabaseIdProvider; -import org.apache.ibatis.mapping.VendorDatabaseIdProvider; +import org.apache.dolphinscheduler.dao.plugin.api.DaoPluginConfiguration; +import org.apache.dolphinscheduler.dao.plugin.api.monitor.DatabaseMonitor; + import org.apache.ibatis.session.SqlSessionFactory; import org.apache.ibatis.type.JdbcType; -import java.util.Properties; - import javax.sql.DataSource; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.jdbc.init.DataSourceScriptDatabaseInitializer; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; -import org.springframework.context.annotation.Profile; import org.springframework.core.io.support.PathMatchingResourcePatternResolver; import org.springframework.core.io.support.ResourcePatternResolver; import org.springframework.jdbc.datasource.DataSourceTransactionManager; @@ -53,6 +50,12 @@ public class SpringConnectionFactory { @Autowired(required = false) public DataSourceScriptDatabaseInitializer dataSourceScriptDatabaseInitializer; + /** + * Inject this field to make sure the DaoPluginConfiguration is initialized before SpringConnectionFactory. + */ + @Autowired + public DaoPluginConfiguration daoPluginConfiguration; + @Bean public MybatisPlusInterceptor paginationInterceptor(DbType dbType) { MybatisPlusInterceptor interceptor = new MybatisPlusInterceptor(); @@ -66,7 +69,8 @@ public class SpringConnectionFactory { } @Bean - public SqlSessionFactory sqlSessionFactory(DataSource dataSource, GlobalConfig globalConfig, + public SqlSessionFactory sqlSessionFactory(DataSource dataSource, + GlobalConfig globalConfig, DbType dbType) throws Exception { MybatisConfiguration configuration = new MybatisConfiguration(); configuration.setMapUnderscoreToCamelCase(true); @@ -82,9 +86,9 @@ public class SpringConnectionFactory { sqlSessionFactoryBean.setGlobalConfig(globalConfig); sqlSessionFactoryBean.setTypeAliasesPackage("org.apache.dolphinscheduler.dao.entity"); ResourcePatternResolver resolver = new PathMatchingResourcePatternResolver(); + // todo: if the different database has different sql, we need to add the different mapper. sqlSessionFactoryBean .setMapperLocations(resolver.getResources("org/apache/dolphinscheduler/dao/mapper/*Mapper.xml")); - sqlSessionFactoryBean.setDatabaseIdProvider(databaseIdProvider()); return sqlSessionFactoryBean.getObject(); } @@ -95,32 +99,13 @@ public class SpringConnectionFactory { } @Bean - public DatabaseIdProvider databaseIdProvider() { - DatabaseIdProvider databaseIdProvider = new VendorDatabaseIdProvider(); - Properties properties = new Properties(); - properties.setProperty("MySQL", "mysql"); - properties.setProperty("PostgreSQL", "pg"); - properties.setProperty("h2", "h2"); - databaseIdProvider.setProperties(properties); - return databaseIdProvider; + public DbType dbType() { + return daoPluginConfiguration.dbType(); } @Bean - @Primary - @Profile("mysql") - public DbType mysql() { - return DbType.MYSQL; + public DatabaseMonitor databaseMonitor() { + return daoPluginConfiguration.databaseMonitor(); } - @Bean - public DbType h2() { - return DbType.H2; - } - - @Bean - @Primary - @Profile("postgresql") - public DbType postgresql() { - return DbType.POSTGRE_SQL; - } } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/H2Performance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/H2Performance.java deleted file mode 100644 index 70d68dea9b..0000000000 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/H2Performance.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.dao.utils; - -import org.apache.dolphinscheduler.common.enums.Flag; -import org.apache.dolphinscheduler.dao.entity.MonitorRecord; -import org.apache.dolphinscheduler.spi.enums.DbType; - -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.Date; - -import lombok.extern.slf4j.Slf4j; - -/** - * H2 MEMORY DB Performance Monitor - */ -@Slf4j -public class H2Performance extends BaseDBPerformance { - - /** - * return the current database performance - * - * @param conn connection - * @return MonitorRecord - */ - @Override - public MonitorRecord getMonitorRecord(Connection conn) { - MonitorRecord monitorRecord = new MonitorRecord(); - monitorRecord.setDate(new Date()); - monitorRecord.setDbType(DbType.H2); - monitorRecord.setState(Flag.YES); - - try (Statement pstmt = conn.createStatement()) { - try ( - ResultSet rs1 = pstmt - .executeQuery("select count(1) as total from information_schema.sessions;")) { - if (rs1.next()) { - monitorRecord.setThreadsConnections(rs1.getInt("total")); - } - } - } catch (SQLException e) { - monitorRecord.setState(Flag.NO); - log.error("SQLException ", e); - } - return monitorRecord; - } -} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/MySQLPerformance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/MySQLPerformance.java deleted file mode 100644 index f09483f5e7..0000000000 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/MySQLPerformance.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.dao.utils; - -import static org.apache.dolphinscheduler.dao.MonitorDBDao.VARIABLE_NAME; - -import org.apache.dolphinscheduler.common.enums.Flag; -import org.apache.dolphinscheduler.dao.entity.MonitorRecord; -import org.apache.dolphinscheduler.spi.enums.DbType; - -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.Statement; -import java.util.Date; - -import lombok.extern.slf4j.Slf4j; - -/** - * MySQL performance - */ -@Slf4j -public class MySQLPerformance extends BaseDBPerformance { - - /** - * get monitor record - * @param conn connection - * @return MonitorRecord - */ - @Override - public MonitorRecord getMonitorRecord(Connection conn) { - MonitorRecord monitorRecord = new MonitorRecord(); - monitorRecord.setDate(new Date()); - monitorRecord.setDbType(DbType.MYSQL); - monitorRecord.setState(Flag.YES); - - try (Statement pstmt = conn.createStatement()) { - try (ResultSet rs1 = pstmt.executeQuery("show global variables")) { - while (rs1.next()) { - if ("MAX_CONNECTIONS".equalsIgnoreCase(rs1.getString(VARIABLE_NAME))) { - monitorRecord.setMaxConnections(Long.parseLong(rs1.getString("value"))); - } - } - } - - try (ResultSet rs2 = pstmt.executeQuery("show global status")) { - while (rs2.next()) { - if ("MAX_USED_CONNECTIONS".equalsIgnoreCase(rs2.getString(VARIABLE_NAME))) { - monitorRecord.setMaxUsedConnections(Long.parseLong(rs2.getString("value"))); - } else if ("THREADS_CONNECTED".equalsIgnoreCase(rs2.getString(VARIABLE_NAME))) { - monitorRecord.setThreadsConnections(Long.parseLong(rs2.getString("value"))); - } else if ("THREADS_RUNNING".equalsIgnoreCase(rs2.getString(VARIABLE_NAME))) { - monitorRecord.setThreadsRunningConnections(Long.parseLong(rs2.getString("value"))); - } - } - } - } catch (Exception e) { - monitorRecord.setState(Flag.NO); - log.error("SQLException ", e); - } - return monitorRecord; - } - -} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PostgreSQLPerformance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PostgreSQLPerformance.java deleted file mode 100644 index 08454ff5ac..0000000000 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PostgreSQLPerformance.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.dao.utils; - -import org.apache.dolphinscheduler.common.enums.Flag; -import org.apache.dolphinscheduler.dao.entity.MonitorRecord; -import org.apache.dolphinscheduler.spi.enums.DbType; - -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.Statement; -import java.util.Date; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class PostgreSQLPerformance extends BaseDBPerformance { - - /** - * get monitor record - * - * @param conn connection - * @return MonitorRecord - */ - @Override - public MonitorRecord getMonitorRecord(Connection conn) { - MonitorRecord monitorRecord = new MonitorRecord(); - monitorRecord.setDate(new Date()); - monitorRecord.setState(Flag.YES); - monitorRecord.setDbType(DbType.POSTGRESQL); - - try (Statement pstmt = conn.createStatement()) { - try (ResultSet rs1 = pstmt.executeQuery("select count(*) from pg_stat_activity;")) { - if (rs1.next()) { - monitorRecord.setThreadsConnections(rs1.getInt("count")); - } - } - - try (ResultSet rs2 = pstmt.executeQuery("show max_connections")) { - if (rs2.next()) { - monitorRecord.setMaxConnections(rs2.getInt("max_connections")); - } - } - - try ( - ResultSet rs3 = - pstmt.executeQuery("select count(*) from pg_stat_activity pg where pg.state = 'active';")) { - if (rs3.next()) { - monitorRecord.setThreadsRunningConnections(rs3.getInt("count")); - } - } - } catch (Exception e) { - monitorRecord.setState(Flag.NO); - log.error("SQLException ", e); - } - return monitorRecord; - } -} diff --git a/dolphinscheduler-dao/src/test/resources/application.yaml b/dolphinscheduler-dao/src/test/resources/application.yaml index 22087b9afa..03c09d3d84 100644 --- a/dolphinscheduler-dao/src/test/resources/application.yaml +++ b/dolphinscheduler-dao/src/test/resources/application.yaml @@ -16,8 +16,13 @@ # spring: + profiles: + active: h2 + sql: + init: + schema-locations: classpath:sql/dolphinscheduler_h2.sql datasource: driver-class-name: org.h2.Driver - url: jdbc:h2:mem:dolphinscheduler;MODE=MySQL;DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=true;INIT=runscript from 'classpath:sql/dolphinscheduler_h2.sql' + url: jdbc:h2:mem:dolphinscheduler;MODE=MySQL;DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=true; username: sa password: "" diff --git a/dolphinscheduler-dist/release-docs/LICENSE b/dolphinscheduler-dist/release-docs/LICENSE index 53e23d938a..94ec6048a5 100644 --- a/dolphinscheduler-dist/release-docs/LICENSE +++ b/dolphinscheduler-dist/release-docs/LICENSE @@ -511,7 +511,6 @@ The text of each license is also included at licenses/LICENSE-[project].txt. zt-zip 1.15: https://github.com/zeroturnaround/zt-zip/blob/master/LICENSE, Apache 2.0 content-type 2.2: https://mvnrepository.com/artifact/com.nimbusds/content-type/2.2, Apache 2.0 jackson-dataformat-xml 2.13.3: https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.13.3, Apache 2.0 - jna-platform 5.6.0: https://mvnrepository.com/artifact/net.java.dev.jna/jna-platform/5.6.0, Apache 2.0 lang-tag 1.6: https://mvnrepository.com/artifact/com.nimbusds/lang-tag/1.6, Apache 2.0 netty-codec-dns 4.1.53.Final: https://mvnrepository.com/artifact/io.netty/netty-codec-dns/4.1.53.Final, Apache 2.0 netty-codec-socks 4.1.53.Final: https://mvnrepository.com/artifact/io.netty/netty-codec-socks/4.1.53.Final, Apache 2.0 @@ -651,6 +650,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt. slf4j-api 1.7.36: https://mvnrepository.com/artifact/org.slf4j/slf4j-api/1.7.36, MIT animal-sniffer-annotations 1.19 https://mvnrepository.com/artifact/org.codehaus.mojo/animal-sniffer-annotations/1.19, MIT checker-qual 3.12.0 https://mvnrepository.com/artifact/org.checkerframework/checker-qual/3.12.0, MIT + GPLv2 + checker-qual 3.19.0 https://mvnrepository.com/artifact/org.checkerframework/checker-qual/3.19.0, MIT + GPLv2 Java-WebSocket 1.5.1: https://github.com/TooTallNate/Java-WebSocket, MIT oshi-core 6.1.1: https://mvnrepository.com/artifact/com.github.oshi/oshi-core/6.1.1, MIT unirest-java 3.7.04-standalone: https://mvnrepository.com/artifact/com.konghq/unirest-java/3.7.04, MIT diff --git a/dolphinscheduler-master/src/main/resources/application.yaml b/dolphinscheduler-master/src/main/resources/application.yaml index bab902fb59..b98d5f5e65 100644 --- a/dolphinscheduler-master/src/main/resources/application.yaml +++ b/dolphinscheduler-master/src/main/resources/application.yaml @@ -15,6 +15,8 @@ # limitations under the License. # spring: + profiles: + active: postgresql banner: charset: UTF-8 jackson: diff --git a/dolphinscheduler-standalone-server/pom.xml b/dolphinscheduler-standalone-server/pom.xml index 7b77b82fa6..921eb6bf8c 100644 --- a/dolphinscheduler-standalone-server/pom.xml +++ b/dolphinscheduler-standalone-server/pom.xml @@ -56,11 +56,6 @@ dolphinscheduler-alert-server - - com.h2database - h2 - - org.apache.curator curator-test diff --git a/dolphinscheduler-standalone-server/src/main/resources/application.yaml b/dolphinscheduler-standalone-server/src/main/resources/application.yaml index fe525b45e7..9f9e1bb961 100644 --- a/dolphinscheduler-standalone-server/src/main/resources/application.yaml +++ b/dolphinscheduler-standalone-server/src/main/resources/application.yaml @@ -16,6 +16,8 @@ # spring: + profiles: + active: h2 jackson: time-zone: UTC date-format: "yyyy-MM-dd HH:mm:ss" diff --git a/pom.xml b/pom.xml index a6c5f185e5..14266903e0 100755 --- a/pom.xml +++ b/pom.xml @@ -56,6 +56,7 @@ dolphinscheduler-scheduler-plugin dolphinscheduler-storage-plugin dolphinscheduler-extract + dolphinscheduler-dao-plugin @@ -259,6 +260,36 @@ ${project.version} + + org.apache.dolphinscheduler + dolphinscheduler-dao-plugin-all + ${project.version} + + + + org.apache.dolphinscheduler + dolphinscheduler-dao-api + ${project.version} + + + + org.apache.dolphinscheduler + dolphinscheduler-dao-h2 + ${project.version} + + + + org.apache.dolphinscheduler + dolphinscheduler-dao-mysql + ${project.version} + + + + org.apache.dolphinscheduler + dolphinscheduler-dao-postgresql + ${project.version} + + org.junit junit-bom diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt index 303c3d4e6f..10e6967193 100644 --- a/tools/dependencies/known-dependencies.txt +++ b/tools/dependencies/known-dependencies.txt @@ -32,7 +32,6 @@ bucket4j-core-6.2.0.jar caffeine-2.9.3.jar checker-qual-3.12.0.jar checker-qual-3.19.0.jar -checker-qual-3.5.0.jar classgraph-4.8.147.jar classmate-1.5.1.jar clickhouse-jdbc-0.4.6.jar @@ -381,7 +380,6 @@ azure-core-http-netty-1.13.0.jar azure-identity-1.7.1.jar content-type-2.2.jar jackson-dataformat-xml-2.13.3.jar -jna-platform-5.6.0.jar lang-tag-1.6.jar msal4j-1.13.3.jar msal4j-persistence-extension-1.1.0.jar