diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml index 5ea68f9588..e9d2554832 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml @@ -40,6 +40,20 @@ spring: leak-detection-threshold: 0 initialization-fail-timeout: 1 +# Mybatis-plus configuration, you don't need to change it +mybatis-plus: + mapper-locations: classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml + type-aliases-package: org.apache.dolphinscheduler.dao.entity + configuration: + cache-enabled: false + call-setters-on-nulls: true + map-underscore-to-camel-case: true + jdbc-type-for-null: NULL + global-config: + db-config: + id-type: auto + banner: false + server: port: 50053 diff --git a/dolphinscheduler-api/src/main/resources/application.yaml b/dolphinscheduler-api/src/main/resources/application.yaml index e58983f0a3..c93fd99a59 100644 --- a/dolphinscheduler-api/src/main/resources/application.yaml +++ b/dolphinscheduler-api/src/main/resources/application.yaml @@ -88,6 +88,20 @@ springdoc: path: /swagger-ui.html packages-to-scan: org.apache.dolphinscheduler.api +# Mybatis-plus configuration, you don't need to change it +mybatis-plus: + mapper-locations: classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml + type-aliases-package: org.apache.dolphinscheduler.dao.entity + configuration: + cache-enabled: false + call-setters-on-nulls: true + map-underscore-to-camel-case: true + jdbc-type-for-null: NULL + global-config: + db-config: + id-type: auto + banner: false + management: endpoints: web: diff --git a/dolphinscheduler-api/src/test/resources/application.yaml b/dolphinscheduler-api/src/test/resources/application.yaml index b11c16d362..26536d631f 100644 --- a/dolphinscheduler-api/src/test/resources/application.yaml +++ b/dolphinscheduler-api/src/test/resources/application.yaml @@ -29,6 +29,19 @@ spring: username: sa password: "" +mybatis-plus: + mapper-locations: classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml + type-aliases-package: org.apache.dolphinscheduler.dao.entity + configuration: + cache-enabled: false + call-setters-on-nulls: true + map-underscore-to-camel-case: true + jdbc-type-for-null: NULL + global-config: + db-config: + id-type: auto + banner: false + registry: type: zookeeper diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-api/src/main/java/org/apache/dolphinscheduler/dao/plugin/api/DaoPluginConfiguration.java b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-api/src/main/java/org/apache/dolphinscheduler/dao/plugin/api/DaoPluginConfiguration.java index ba3577f10e..36443b012a 100644 --- a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-api/src/main/java/org/apache/dolphinscheduler/dao/plugin/api/DaoPluginConfiguration.java +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-api/src/main/java/org/apache/dolphinscheduler/dao/plugin/api/DaoPluginConfiguration.java @@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.dao.plugin.api; +import org.apache.dolphinscheduler.dao.plugin.api.dialect.DatabaseDialect; import org.apache.dolphinscheduler.dao.plugin.api.monitor.DatabaseMonitor; import com.baomidou.mybatisplus.annotation.DbType; @@ -33,4 +34,6 @@ public interface DaoPluginConfiguration { DatabaseMonitor databaseMonitor(); + DatabaseDialect databaseDialect(); + } diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-api/src/main/java/org/apache/dolphinscheduler/dao/plugin/api/dialect/DatabaseDialect.java b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-api/src/main/java/org/apache/dolphinscheduler/dao/plugin/api/dialect/DatabaseDialect.java new file mode 100644 index 0000000000..a07f050fda --- /dev/null +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-api/src/main/java/org/apache/dolphinscheduler/dao/plugin/api/dialect/DatabaseDialect.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.plugin.api.dialect; + +public interface DatabaseDialect { + + boolean tableExists(String tableName); + + boolean columnExists(String tableName, String columnName); + +} diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/H2DaoPluginConfiguration.java b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/H2DaoPluginConfiguration.java index dca8b29a04..9aea94f77d 100644 --- a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/H2DaoPluginConfiguration.java +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/H2DaoPluginConfiguration.java @@ -21,7 +21,9 @@ package org.apache.dolphinscheduler.dao.plugin.h2; import org.apache.dolphinscheduler.dao.plugin.api.DaoPluginConfiguration; +import org.apache.dolphinscheduler.dao.plugin.api.dialect.DatabaseDialect; import org.apache.dolphinscheduler.dao.plugin.api.monitor.DatabaseMonitor; +import org.apache.dolphinscheduler.dao.plugin.h2.dialect.H2Dialect; import org.apache.dolphinscheduler.dao.plugin.h2.monitor.H2Monitor; import javax.sql.DataSource; @@ -49,4 +51,9 @@ public class H2DaoPluginConfiguration implements DaoPluginConfiguration { return new H2Monitor(dataSource); } + @Override + public DatabaseDialect databaseDialect() { + return new H2Dialect(); + } + } diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/dialect/H2Dialect.java b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/dialect/H2Dialect.java new file mode 100644 index 0000000000..2cb3f60529 --- /dev/null +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/dialect/H2Dialect.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.plugin.h2.dialect; + +import org.apache.dolphinscheduler.dao.plugin.api.dialect.DatabaseDialect; + +public class H2Dialect implements DatabaseDialect { + + @Override + public boolean tableExists(String tableName) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean columnExists(String tableName, String columnName) { + throw new UnsupportedOperationException(); + } +} diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/MysqlDaoPluginConfiguration.java b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/MysqlDaoPluginConfiguration.java index 25f6c34374..8b37fca67b 100644 --- a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/MysqlDaoPluginConfiguration.java +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/MysqlDaoPluginConfiguration.java @@ -20,7 +20,9 @@ package org.apache.dolphinscheduler.dao.plugin.mysql; import org.apache.dolphinscheduler.dao.plugin.api.DaoPluginConfiguration; +import org.apache.dolphinscheduler.dao.plugin.api.dialect.DatabaseDialect; import org.apache.dolphinscheduler.dao.plugin.api.monitor.DatabaseMonitor; +import org.apache.dolphinscheduler.dao.plugin.mysql.dialect.MysqlDialect; import org.apache.dolphinscheduler.dao.plugin.mysql.monitor.MysqlMonitor; import javax.sql.DataSource; @@ -47,4 +49,9 @@ public class MysqlDaoPluginConfiguration implements DaoPluginConfiguration { public DatabaseMonitor databaseMonitor() { return new MysqlMonitor(dataSource); } + + @Override + public DatabaseDialect databaseDialect() { + return new MysqlDialect(dataSource); + } } diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/dialect/MysqlDialect.java b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/dialect/MysqlDialect.java new file mode 100644 index 0000000000..ef8c20e021 --- /dev/null +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/dialect/MysqlDialect.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +package org.apache.dolphinscheduler.dao.plugin.mysql.dialect; + +import org.apache.dolphinscheduler.dao.plugin.api.dialect.DatabaseDialect; + +import java.sql.Connection; +import java.sql.ResultSet; + +import javax.sql.DataSource; + +import lombok.SneakyThrows; + +public class MysqlDialect implements DatabaseDialect { + + private final DataSource dataSource; + + public MysqlDialect(DataSource dataSource) { + this.dataSource = dataSource; + } + + @SneakyThrows + @Override + public boolean tableExists(String tableName) { + try ( + Connection conn = dataSource.getConnection(); + ResultSet rs = conn.getMetaData().getTables(conn.getCatalog(), conn.getSchema(), tableName, null)) { + return rs.next(); + } + } + + @SneakyThrows + @Override + public boolean columnExists(String tableName, String columnName) { + try ( + Connection conn = dataSource.getConnection(); + ResultSet rs = + conn.getMetaData().getColumns(conn.getCatalog(), conn.getSchema(), tableName, columnName)) { + return rs.next(); + + } + } +} diff --git a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/PostgresqlDaoPluginConfiguration.java b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/PostgresqlDaoPluginConfiguration.java index ee9c290f36..e57c84fab9 100644 --- a/dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/PostgresqlDaoPluginConfiguration.java +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/PostgresqlDaoPluginConfiguration.java @@ -21,7 +21,9 @@ package org.apache.dolphinscheduler.dao.plugin.postgresql; import org.apache.dolphinscheduler.dao.plugin.api.DaoPluginConfiguration; +import org.apache.dolphinscheduler.dao.plugin.api.dialect.DatabaseDialect; import org.apache.dolphinscheduler.dao.plugin.api.monitor.DatabaseMonitor; +import org.apache.dolphinscheduler.dao.plugin.postgresql.dialect.PostgresqlDialect; import org.apache.dolphinscheduler.dao.plugin.postgresql.monitor.PostgresqlMonitor; import javax.sql.DataSource; @@ -48,4 +50,9 @@ public class PostgresqlDaoPluginConfiguration implements DaoPluginConfiguration public DatabaseMonitor databaseMonitor() { return new PostgresqlMonitor(dataSource); } + + @Override + public DatabaseDialect databaseDialect() { + return new PostgresqlDialect(dataSource); + } } diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/PostgreSQLUpgradeDao.java b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/dialect/PostgresqlDialect.java similarity index 53% rename from dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/PostgreSQLUpgradeDao.java rename to dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/dialect/PostgresqlDialect.java index c2d4b637f1..7cda5294a2 100644 --- a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/PostgreSQLUpgradeDao.java +++ b/dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/dialect/PostgresqlDialect.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.tools.datasource.dao; +package org.apache.dolphinscheduler.dao.plugin.postgresql.dialect; -import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.dao.plugin.api.dialect.DatabaseDialect; import java.sql.Connection; import java.sql.PreparedStatement; @@ -26,82 +26,47 @@ import java.sql.SQLException; import javax.sql.DataSource; -import lombok.extern.slf4j.Slf4j; +import lombok.SneakyThrows; -import org.springframework.context.annotation.Profile; -import org.springframework.stereotype.Service; +public class PostgresqlDialect implements DatabaseDialect { -@Service -@Slf4j -@Profile("postgresql") -public class PostgreSQLUpgradeDao extends UpgradeDao { + private final DataSource dataSource; - private PostgreSQLUpgradeDao(DataSource dataSource) { - super(dataSource); + public PostgresqlDialect(DataSource dataSource) { + this.dataSource = dataSource; } + @SneakyThrows @Override - protected String initSqlPath() { - return "create/release-1.2.0_schema/postgresql"; - } - - @Override - public DbType getDbType() { - return DbType.POSTGRESQL; - } - - public String getSchema() { + public boolean tableExists(String tableName) { try ( Connection conn = dataSource.getConnection(); - PreparedStatement pstmt = conn.prepareStatement("select current_schema()"); - ResultSet resultSet = pstmt.executeQuery()) { - while (resultSet.next()) { - if (resultSet.isFirst()) { - return resultSet.getString(1); - } - } - - } catch (SQLException e) { - log.error(e.getMessage(), e); + ResultSet rs = conn.getMetaData().getTables(conn.getCatalog(), getSchema(), tableName, null)) { + return rs.next(); } - return ""; } - /** - * determines whether a table exists - * - * @param tableName tableName - * @return if table exist return true，else return false - */ + @SneakyThrows @Override - public boolean isExistsTable(String tableName) { + public boolean columnExists(String tableName, String columnName) { try ( Connection conn = dataSource.getConnection(); ResultSet rs = conn.getMetaData().getTables(conn.getCatalog(), getSchema(), tableName, null)) { return rs.next(); - } catch (SQLException e) { - log.error(e.getMessage(), e); - throw new RuntimeException(e.getMessage(), e); } } - /** - * determines whether a field exists in the specified table - * - * @param tableName tableName - * @param columnName columnName - * @return if column name exist return true，else return false - */ - @Override - public boolean isExistsColumn(String tableName, String columnName) { + private String getSchema() throws SQLException { try ( Connection conn = dataSource.getConnection(); - ResultSet rs = conn.getMetaData().getColumns(conn.getCatalog(), getSchema(), tableName, columnName)) { - return rs.next(); - } catch (SQLException e) { - log.error(e.getMessage(), e); - throw new RuntimeException(e.getMessage(), e); + PreparedStatement pstmt = conn.prepareStatement("select current_schema()"); + ResultSet resultSet = pstmt.executeQuery()) { + while (resultSet.next()) { + if (resultSet.isFirst()) { + return resultSet.getString(1); + } + } } + return ""; } - } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/DaoConfiguration.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/DaoConfiguration.java index d1de5630a8..2587bf0dc2 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/DaoConfiguration.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/DaoConfiguration.java @@ -20,12 +20,60 @@ package org.apache.dolphinscheduler.dao; +import org.apache.dolphinscheduler.dao.plugin.api.DaoPluginConfiguration; +import org.apache.dolphinscheduler.dao.plugin.api.dialect.DatabaseDialect; +import org.apache.dolphinscheduler.dao.plugin.api.monitor.DatabaseMonitor; + import org.mybatis.spring.annotation.MapperScan; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.jdbc.init.DataSourceScriptDatabaseInitializer; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; +import com.baomidou.mybatisplus.annotation.DbType; +import com.baomidou.mybatisplus.extension.plugins.MybatisPlusInterceptor; +import com.baomidou.mybatisplus.extension.plugins.inner.PaginationInnerInterceptor; + @Configuration @EnableAutoConfiguration +@ComponentScan({"org.apache.dolphinscheduler.dao.plugin"}) @MapperScan(basePackages = "org.apache.dolphinscheduler.dao.mapper", sqlSessionFactoryRef = "sqlSessionFactory") public class DaoConfiguration { + + /** + * Inject this field to make sure the database is initialized, this can solve the table not found issue #8432. + */ + @Autowired(required = false) + public DataSourceScriptDatabaseInitializer dataSourceScriptDatabaseInitializer; + + /** + * Inject this field to make sure the DaoPluginConfiguration is initialized before SpringConnectionFactory. + */ + @Autowired + public DaoPluginConfiguration daoPluginConfiguration; + + @Bean + public MybatisPlusInterceptor paginationInterceptor(DbType dbType) { + MybatisPlusInterceptor interceptor = new MybatisPlusInterceptor(); + interceptor.addInnerInterceptor(new PaginationInnerInterceptor(dbType)); + return interceptor; + } + + @Bean + public DbType dbType() { + return daoPluginConfiguration.dbType(); + } + + @Bean + public DatabaseMonitor databaseMonitor() { + return daoPluginConfiguration.databaseMonitor(); + } + + @Bean + public DatabaseDialect databaseDialect() { + return daoPluginConfiguration.databaseDialect(); + } + } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SpringConnectionFactory.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SpringConnectionFactory.java deleted file mode 100644 index c640761350..0000000000 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SpringConnectionFactory.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.dao.datasource; - -import org.apache.dolphinscheduler.dao.plugin.api.DaoPluginConfiguration; -import org.apache.dolphinscheduler.dao.plugin.api.monitor.DatabaseMonitor; - -import org.apache.ibatis.session.SqlSessionFactory; -import org.apache.ibatis.type.JdbcType; - -import javax.sql.DataSource; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.jdbc.init.DataSourceScriptDatabaseInitializer; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.core.io.support.PathMatchingResourcePatternResolver; -import org.springframework.core.io.support.ResourcePatternResolver; -import org.springframework.jdbc.datasource.DataSourceTransactionManager; - -import com.baomidou.mybatisplus.annotation.DbType; -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.core.MybatisConfiguration; -import com.baomidou.mybatisplus.core.config.GlobalConfig; -import com.baomidou.mybatisplus.extension.plugins.MybatisPlusInterceptor; -import com.baomidou.mybatisplus.extension.plugins.inner.PaginationInnerInterceptor; -import com.baomidou.mybatisplus.extension.spring.MybatisSqlSessionFactoryBean; - -@Configuration -public class SpringConnectionFactory { - - /** - * Inject this field to make sure the database is initialized, this can solve the table not found issue #8432. - */ - @Autowired(required = false) - public DataSourceScriptDatabaseInitializer dataSourceScriptDatabaseInitializer; - - /** - * Inject this field to make sure the DaoPluginConfiguration is initialized before SpringConnectionFactory. - */ - @Autowired - public DaoPluginConfiguration daoPluginConfiguration; - - @Bean - public MybatisPlusInterceptor paginationInterceptor(DbType dbType) { - MybatisPlusInterceptor interceptor = new MybatisPlusInterceptor(); - interceptor.addInnerInterceptor(new PaginationInnerInterceptor(dbType)); - return interceptor; - } - - @Bean - public DataSourceTransactionManager transactionManager(DataSource dataSource) { - return new DataSourceTransactionManager(dataSource); - } - - @Bean - public SqlSessionFactory sqlSessionFactory(DataSource dataSource, - GlobalConfig globalConfig, - DbType dbType) throws Exception { - MybatisConfiguration configuration = new MybatisConfiguration(); - configuration.setMapUnderscoreToCamelCase(true); - configuration.setCacheEnabled(false); - configuration.setCallSettersOnNulls(true); - configuration.setJdbcTypeForNull(JdbcType.NULL); - configuration.addInterceptor(paginationInterceptor(dbType)); - - MybatisSqlSessionFactoryBean sqlSessionFactoryBean = new MybatisSqlSessionFactoryBean(); - sqlSessionFactoryBean.setConfiguration(configuration); - sqlSessionFactoryBean.setDataSource(dataSource); - - sqlSessionFactoryBean.setGlobalConfig(globalConfig); - sqlSessionFactoryBean.setTypeAliasesPackage("org.apache.dolphinscheduler.dao.entity"); - ResourcePatternResolver resolver = new PathMatchingResourcePatternResolver(); - // todo: if the different database has different sql, we need to add the different mapper. - sqlSessionFactoryBean - .setMapperLocations(resolver.getResources("org/apache/dolphinscheduler/dao/mapper/*Mapper.xml")); - return sqlSessionFactoryBean.getObject(); - } - - @Bean - public GlobalConfig globalConfig() { - return new GlobalConfig().setDbConfig(new GlobalConfig.DbConfig() - .setIdType(IdType.AUTO)).setBanner(false); - } - - @Bean - public DbType dbType() { - return daoPluginConfiguration.dbType(); - } - - @Bean - public DatabaseMonitor databaseMonitor() { - return daoPluginConfiguration.databaseMonitor(); - } - -} diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/ProcessDefinitionDaoTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/ProcessDefinitionDaoTest.java deleted file mode 100644 index 6a1599360b..0000000000 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/ProcessDefinitionDaoTest.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.dao.upgrade; - -import java.util.HashMap; -import java.util.Map; - -import javax.sql.DataSource; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.context.ActiveProfiles; - -@ActiveProfiles("h2") -public class ProcessDefinitionDaoTest { - - @Autowired - private DataSource dataSource; - final ProcessDefinitionDao processDefinitionDao = new ProcessDefinitionDao(); - - @Test - public void testQueryAllProcessDefinition() { - // Map processDefinitionJsonMap = - // processDefinitionDao.queryAllProcessDefinition(dataSource.getConnection()); - // assertThat(processDefinitionJsonMap.size(),greaterThanOrEqualTo(0)); - } - - @Test - public void testUpdateProcessDefinitionJson() { - Map processDefinitionJsonMap = new HashMap<>(); - processDefinitionJsonMap.put(1, "test"); - // processDefinitionDao.updateProcessDefinitionJson(dataSource.getConnection(),processDefinitionJsonMap); - } - - @Test - public void testQueryAllProcessDefinitionException() { - // processDefinitionDao.queryAllProcessDefinition(null); - } - - @Test - public void testUpdateProcessDefinitionJsonException() { - Assertions.assertThrows(Exception.class, () -> processDefinitionDao.updateProcessDefinitionJson(null, null)); - } -} diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/WorkerGroupDaoTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/WorkerGroupDaoTest.java deleted file mode 100644 index 0c36d5f136..0000000000 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/WorkerGroupDaoTest.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.dolphinscheduler.dao.upgrade; - -import org.apache.dolphinscheduler.dao.DaoConfiguration; - -import javax.sql.DataSource; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.junit.jupiter.MockitoExtension; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.ActiveProfiles; - -@ActiveProfiles("h2") -@SpringBootTest(classes = DaoConfiguration.class) -@ExtendWith(MockitoExtension.class) -@SpringBootApplication(scanBasePackageClasses = DaoConfiguration.class) -public class WorkerGroupDaoTest { - - @Autowired - protected DataSource dataSource; - - @Test - public void testQueryQueryAllOldWorkerGroupException() throws Exception { - Assertions.assertThrows(Exception.class, () -> { - WorkerGroupDao workerGroupDao = new WorkerGroupDao(); - workerGroupDao.queryAllOldWorkerGroup(null); - }); - - } - -} diff --git a/dolphinscheduler-dao/src/test/resources/application.yaml b/dolphinscheduler-dao/src/test/resources/application.yaml index 03c09d3d84..ffc6b9762d 100644 --- a/dolphinscheduler-dao/src/test/resources/application.yaml +++ b/dolphinscheduler-dao/src/test/resources/application.yaml @@ -26,3 +26,16 @@ spring: url: jdbc:h2:mem:dolphinscheduler;MODE=MySQL;DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=true; username: sa password: "" + +mybatis-plus: + mapper-locations: classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml + type-aliases-package: org.apache.dolphinscheduler.dao.entity + configuration: + cache-enabled: false + call-setters-on-nulls: true + map-underscore-to-camel-case: true + jdbc-type-for-null: NULL + global-config: + db-config: + id-type: auto + banner: false \ No newline at end of file diff --git a/dolphinscheduler-master/src/main/resources/application.yaml b/dolphinscheduler-master/src/main/resources/application.yaml index b98d5f5e65..cbf25079f3 100644 --- a/dolphinscheduler-master/src/main/resources/application.yaml +++ b/dolphinscheduler-master/src/main/resources/application.yaml @@ -71,6 +71,21 @@ spring: org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.PostgreSQLDelegate org.quartz.jobStore.clusterCheckinInterval: 5000 +# Mybatis-plus configuration, you don't need to change it +mybatis-plus: + mapper-locations: classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml + type-aliases-package: org.apache.dolphinscheduler.dao.entity + configuration: + cache-enabled: false + call-setters-on-nulls: true + map-underscore-to-camel-case: true + jdbc-type-for-null: NULL + global-config: + db-config: + id-type: auto + banner: false + + registry: type: zookeeper zookeeper: diff --git a/dolphinscheduler-standalone-server/src/main/resources/application.yaml b/dolphinscheduler-standalone-server/src/main/resources/application.yaml index 9f9e1bb961..dda509733e 100644 --- a/dolphinscheduler-standalone-server/src/main/resources/application.yaml +++ b/dolphinscheduler-standalone-server/src/main/resources/application.yaml @@ -76,6 +76,19 @@ spring: pathmatch: matching-strategy: ANT_PATH_MATCHER +mybatis-plus: + mapper-locations: classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml + type-aliases-package: org.apache.dolphinscheduler.dao.entity + configuration: + cache-enabled: false + call-setters-on-nulls: true + map-underscore-to-camel-case: true + jdbc-type-for-null: NULL + global-config: + db-config: + id-type: auto + banner: false + registry: type: zookeeper zookeeper: diff --git a/dolphinscheduler-tools/pom.xml b/dolphinscheduler-tools/pom.xml index 42007b7cb8..dbba85d10d 100644 --- a/dolphinscheduler-tools/pom.xml +++ b/dolphinscheduler-tools/pom.xml @@ -130,19 +130,16 @@ org.testcontainers mysql - test mysql mysql-connector-java - test org.testcontainers postgresql - test diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/DolphinSchedulerManager.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/DolphinSchedulerManager.java index 04b906beb5..7f0b7b9d7a 100644 --- a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/DolphinSchedulerManager.java +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/DolphinSchedulerManager.java @@ -17,57 +17,45 @@ package org.apache.dolphinscheduler.tools.datasource; -import org.apache.dolphinscheduler.dao.upgrade.SchemaUtils; -import org.apache.dolphinscheduler.spi.enums.DbType; -import org.apache.dolphinscheduler.tools.datasource.dao.UpgradeDao; +import org.apache.dolphinscheduler.dao.plugin.api.dialect.DatabaseDialect; import org.apache.dolphinscheduler.tools.datasource.upgrader.DolphinSchedulerUpgrader; import org.apache.dolphinscheduler.tools.datasource.upgrader.DolphinSchedulerVersion; +import org.apache.dolphinscheduler.tools.datasource.upgrader.UpgradeDao; +import org.apache.dolphinscheduler.tools.datasource.utils.SchemaUtils; import org.apache.commons.collections4.CollectionUtils; import java.io.IOException; -import java.sql.Connection; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; -import javax.sql.DataSource; - import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @Service @Slf4j public class DolphinSchedulerManager { - private final UpgradeDao upgradeDao; + @Autowired + private UpgradeDao upgradeDao; + + @Autowired + private DatabaseDialect databaseDialect; private Map upgraderMap = new HashMap<>(); - public DolphinSchedulerManager(DataSource dataSource, List daos, - List dolphinSchedulerUpgraders) throws Exception { - final DbType type = getCurrentDbType(dataSource); - upgradeDao = daos.stream() - .filter(it -> it.getDbType() == type) - .findFirst() - .orElseThrow(() -> new RuntimeException( - "Cannot find UpgradeDao implementation for db type: " + type)); + public DolphinSchedulerManager(List dolphinSchedulerUpgraders) throws Exception { if (CollectionUtils.isNotEmpty(dolphinSchedulerUpgraders)) { upgraderMap = dolphinSchedulerUpgraders.stream() .collect(Collectors.toMap(DolphinSchedulerUpgrader::getCurrentVersion, Function.identity())); } } - private DbType getCurrentDbType(DataSource dataSource) throws Exception { - try (Connection conn = dataSource.getConnection()) { - String name = conn.getMetaData().getDatabaseProductName().toUpperCase(); - return DbType.valueOf(name); - } - } - public void initDolphinScheduler() { this.initDolphinSchedulerSchema(); } @@ -78,9 +66,9 @@ public class DolphinSchedulerManager { */ public boolean schemaIsInitialized() { // Determines whether the dolphinscheduler table structure has been init - if (upgradeDao.isExistsTable("t_escheduler_version") - || upgradeDao.isExistsTable("t_ds_version") - || upgradeDao.isExistsTable("t_escheduler_queue")) { + if (databaseDialect.tableExists("t_escheduler_version") + || databaseDialect.tableExists("t_ds_version") + || databaseDialect.tableExists("t_escheduler_queue")) { log.info("The database has been initialized. Skip the initialization step"); return true; } @@ -100,13 +88,13 @@ public class DolphinSchedulerManager { } else { String version; // Gets the version of the current system - if (upgradeDao.isExistsTable("t_escheduler_version")) { + if (databaseDialect.tableExists("t_escheduler_version")) { version = upgradeDao.getCurrentVersion("t_escheduler_version"); - } else if (upgradeDao.isExistsTable("t_ds_version")) { + } else if (databaseDialect.tableExists("t_ds_version")) { version = upgradeDao.getCurrentVersion("t_ds_version"); - } else if (upgradeDao.isExistsColumn("t_escheduler_queue", "create_time")) { + } else if (databaseDialect.columnExists("t_escheduler_queue", "create_time")) { version = "1.0.1"; - } else if (upgradeDao.isExistsTable("t_escheduler_queue")) { + } else if (databaseDialect.tableExists("t_escheduler_queue")) { version = "1.0.0"; } else { log.error("Unable to determine current software version, so cannot upgrade"); diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/InitDolphinScheduler.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/InitDolphinScheduler.java deleted file mode 100644 index c6aa05190d..0000000000 --- a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/InitDolphinScheduler.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.tools.datasource; - -import org.apache.dolphinscheduler.dao.DaoConfiguration; - -import lombok.extern.slf4j.Slf4j; - -import org.springframework.boot.CommandLineRunner; -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.ImportAutoConfiguration; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.context.annotation.Profile; -import org.springframework.stereotype.Component; - -@ImportAutoConfiguration(DaoConfiguration.class) -@SpringBootApplication -public class InitDolphinScheduler { - - public static void main(String[] args) { - SpringApplication.run(InitDolphinScheduler.class, args); - } - - @Component - @Profile("init") - @Slf4j - static class InitRunner implements CommandLineRunner { - - private final DolphinSchedulerManager dolphinSchedulerManager; - - InitRunner(DolphinSchedulerManager dolphinSchedulerManager) { - this.dolphinSchedulerManager = dolphinSchedulerManager; - } - - @Override - public void run(String... args) { - dolphinSchedulerManager.initDolphinScheduler(); - log.info("init DolphinScheduler finished"); - } - } -} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/JsonSplitDao.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/JsonSplitDao.java similarity index 99% rename from dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/JsonSplitDao.java rename to dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/JsonSplitDao.java index d453651782..2a411df994 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/JsonSplitDao.java +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/JsonSplitDao.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.dao.upgrade; +package org.apache.dolphinscheduler.tools.datasource.dao; import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog; import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelationLog; @@ -28,6 +28,7 @@ import java.util.List; import lombok.extern.slf4j.Slf4j; +@Deprecated @Slf4j public class JsonSplitDao { diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/MySQLUpgradeDao.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/MySQLUpgradeDao.java deleted file mode 100644 index ba7e5d28e9..0000000000 --- a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/MySQLUpgradeDao.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.tools.datasource.dao; - -import org.apache.dolphinscheduler.spi.enums.DbType; - -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; - -import javax.sql.DataSource; - -import lombok.extern.slf4j.Slf4j; - -import org.springframework.context.annotation.Profile; -import org.springframework.stereotype.Service; - -@Service -@Slf4j -@Profile("mysql") -public class MySQLUpgradeDao extends UpgradeDao { - - private MySQLUpgradeDao(DataSource dataSource) { - super(dataSource); - } - - @Override - protected String initSqlPath() { - return "create/release-1.0.0_schema/mysql"; - } - - @Override - public DbType getDbType() { - return DbType.MYSQL; - } - - /** - * determines whether a table exists - * @param tableName tableName - * @return if table exist return true，else return false - */ - @Override - public boolean isExistsTable(String tableName) { - try ( - Connection conn = dataSource.getConnection(); - ResultSet rs = conn.getMetaData().getTables(conn.getCatalog(), conn.getSchema(), tableName, null)) { - return rs.next(); - } catch (SQLException e) { - log.error(e.getMessage(), e); - throw new RuntimeException(e.getMessage(), e); - } - - } - - /** - * determines whether a field exists in the specified table - * @param tableName tableName - * @param columnName columnName - * @return if column name exist return true，else return false - */ - @Override - public boolean isExistsColumn(String tableName, String columnName) { - try ( - Connection conn = dataSource.getConnection(); - ResultSet rs = - conn.getMetaData().getColumns(conn.getCatalog(), conn.getSchema(), tableName, columnName)) { - return rs.next(); - - } catch (SQLException e) { - log.error(e.getMessage(), e); - throw new RuntimeException(e.getMessage(), e); - } - } - -} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ProcessDefinitionDao.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ProcessDefinitionDao.java similarity index 98% rename from dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ProcessDefinitionDao.java rename to dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ProcessDefinitionDao.java index 2c66b67414..338a9d591d 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ProcessDefinitionDao.java +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ProcessDefinitionDao.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.dao.upgrade; +package org.apache.dolphinscheduler.tools.datasource.dao; import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.enums.Flag; @@ -33,6 +33,7 @@ import java.util.Map; import lombok.extern.slf4j.Slf4j; +@Deprecated @Slf4j public class ProcessDefinitionDao { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ProjectDao.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ProjectDao.java similarity index 97% rename from dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ProjectDao.java rename to dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ProjectDao.java index 3b267f8b5f..65466fe99a 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ProjectDao.java +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ProjectDao.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.dao.upgrade; +package org.apache.dolphinscheduler.tools.datasource.dao; import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils; @@ -27,6 +27,7 @@ import java.util.Map; import lombok.extern.slf4j.Slf4j; +@Deprecated @Slf4j public class ProjectDao { diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ResourceDao.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ResourceDao.java index b6ace04f17..4a9880f66a 100644 --- a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ResourceDao.java +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ResourceDao.java @@ -34,6 +34,7 @@ import com.google.common.base.Strings; /** * resource dao */ +@Deprecated @Slf4j public class ResourceDao { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ScheduleDao.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ScheduleDao.java similarity index 98% rename from dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ScheduleDao.java rename to dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ScheduleDao.java index b79e40b0ce..e3f6772ea2 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ScheduleDao.java +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ScheduleDao.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.dao.upgrade; +package org.apache.dolphinscheduler.tools.datasource.dao; import java.sql.Connection; import java.sql.PreparedStatement; @@ -26,6 +26,7 @@ import java.util.Map; import lombok.extern.slf4j.Slf4j; +@Deprecated @Slf4j public class ScheduleDao { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/WorkerGroupDao.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/WorkerGroupDao.java similarity index 96% rename from dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/WorkerGroupDao.java rename to dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/WorkerGroupDao.java index 44b5d85ad2..4b4b2394c5 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/WorkerGroupDao.java +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/WorkerGroupDao.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.dao.upgrade; +package org.apache.dolphinscheduler.tools.datasource.dao; import java.sql.Connection; import java.sql.PreparedStatement; @@ -25,6 +25,7 @@ import java.util.Map; import lombok.extern.slf4j.Slf4j; +@Deprecated @Slf4j public class WorkerGroupDao { diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/UpgradeDao.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/upgrader/UpgradeDao.java similarity index 88% rename from dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/UpgradeDao.java rename to dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/upgrader/UpgradeDao.java index 0154718780..89704bc736 100644 --- a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/UpgradeDao.java +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/upgrader/UpgradeDao.java @@ -15,11 +15,12 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.tools.datasource.dao; +package org.apache.dolphinscheduler.tools.datasource.upgrader; import org.apache.dolphinscheduler.common.sql.SqlScriptRunner; -import org.apache.dolphinscheduler.dao.upgrade.SchemaUtils; -import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.dolphinscheduler.dao.plugin.api.dialect.DatabaseDialect; +import org.apache.dolphinscheduler.tools.datasource.dao.ResourceDao; +import org.apache.dolphinscheduler.tools.datasource.utils.SchemaUtils; import java.io.FileNotFoundException; import java.sql.Connection; @@ -31,34 +32,33 @@ import javax.sql.DataSource; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import com.baomidou.mybatisplus.annotation.DbType; + @Slf4j -public abstract class UpgradeDao { +@Service +public class UpgradeDao { private static final String T_VERSION_NAME = "t_escheduler_version"; private static final String T_NEW_VERSION_NAME = "t_ds_version"; - protected final DataSource dataSource; - - protected UpgradeDao(DataSource dataSource) { - this.dataSource = dataSource; - } + @Autowired + private DataSource dataSource; - protected abstract String initSqlPath(); + @Autowired + private DbType dbType; - public abstract DbType getDbType(); - - public void initSchema() { - // Execute the dolphinscheduler full sql - runInitSql(getDbType()); - } + @Autowired + private DatabaseDialect databaseDialect; /** * run init sql to init db schema - * - * @param dbType db type */ - private void runInitSql(DbType dbType) { - String sqlFilePath = String.format("sql/dolphinscheduler_%s.sql", dbType.getDescp()); + public void initSchema() { + // Execute the dolphinscheduler full sql + String sqlFilePath = String.format("sql/dolphinscheduler_%s.sql", dbType.getDb()); SqlScriptRunner sqlScriptRunner = new SqlScriptRunner(dataSource, sqlFilePath); try { sqlScriptRunner.execute(); @@ -68,10 +68,6 @@ public abstract class UpgradeDao { } } - public abstract boolean isExistsTable(String tableName); - - public abstract boolean isExistsColumn(String tableName, String columnName); - public String getCurrentVersion(String versionName) { String sql = String.format("select version from %s", versionName); String version = null; @@ -112,17 +108,17 @@ public abstract class UpgradeDao { private void upgradeDolphinSchedulerDML(String schemaDir, String scriptFile) { String schemaVersion = schemaDir.split("_")[0]; String sqlFilePath = - String.format("sql/upgrade/%s/%s/%s", schemaDir, getDbType().name().toLowerCase(), scriptFile); + String.format("sql/upgrade/%s/%s/%s", schemaDir, dbType.getDb(), scriptFile); try { // Execute the upgraded dolphinscheduler dml SqlScriptRunner sqlScriptRunner = new SqlScriptRunner(dataSource, sqlFilePath); sqlScriptRunner.execute(); try (Connection connection = dataSource.getConnection()) { String upgradeSQL; - if (isExistsTable(T_VERSION_NAME)) { + if (databaseDialect.tableExists(T_VERSION_NAME)) { // Change version in the version table to the new version upgradeSQL = String.format("update %s set version = ?", T_VERSION_NAME); - } else if (isExistsTable(T_NEW_VERSION_NAME)) { + } else if (databaseDialect.tableExists(T_NEW_VERSION_NAME)) { // Change version in the version table to the new version upgradeSQL = String.format("update %s set version = ?", T_NEW_VERSION_NAME); } else { @@ -151,7 +147,7 @@ public abstract class UpgradeDao { */ public void upgradeDolphinSchedulerDDL(String schemaDir, String scriptFile) { String sqlFilePath = - String.format("sql/upgrade/%s/%s/%s", schemaDir, getDbType().name().toLowerCase(), scriptFile); + String.format("sql/upgrade/%s/%s/%s", schemaDir, dbType.getDb(), scriptFile); SqlScriptRunner sqlScriptRunner = new SqlScriptRunner(dataSource, sqlFilePath); try { // Execute the dolphinscheduler ddl.sql for the upgrade diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/upgrader/v130/V130DolphinSchedulerUpgrader.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/upgrader/v130/V130DolphinSchedulerUpgrader.java index 27871277bf..37b1cc0a33 100644 --- a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/upgrader/v130/V130DolphinSchedulerUpgrader.java +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/upgrader/v130/V130DolphinSchedulerUpgrader.java @@ -18,8 +18,8 @@ package org.apache.dolphinscheduler.tools.datasource.upgrader.v130; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.dao.upgrade.ProcessDefinitionDao; -import org.apache.dolphinscheduler.dao.upgrade.WorkerGroupDao; +import org.apache.dolphinscheduler.tools.datasource.dao.ProcessDefinitionDao; +import org.apache.dolphinscheduler.tools.datasource.dao.WorkerGroupDao; import org.apache.dolphinscheduler.tools.datasource.upgrader.DolphinSchedulerUpgrader; import org.apache.dolphinscheduler.tools.datasource.upgrader.DolphinSchedulerVersion; diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/upgrader/v132/V132DolphinSchedulerUpgrader.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/upgrader/v132/V132DolphinSchedulerUpgrader.java index abb7045e34..5164194121 100644 --- a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/upgrader/v132/V132DolphinSchedulerUpgrader.java +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/upgrader/v132/V132DolphinSchedulerUpgrader.java @@ -18,8 +18,8 @@ package org.apache.dolphinscheduler.tools.datasource.upgrader.v132; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.dao.upgrade.ProcessDefinitionDao; import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; +import org.apache.dolphinscheduler.tools.datasource.dao.ProcessDefinitionDao; import org.apache.dolphinscheduler.tools.datasource.upgrader.DolphinSchedulerUpgrader; import org.apache.dolphinscheduler.tools.datasource.upgrader.DolphinSchedulerVersion; diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/upgrader/v200/V200DolphinSchedulerUpgrader.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/upgrader/v200/V200DolphinSchedulerUpgrader.java index f14ed15d22..35a9be75dc 100644 --- a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/upgrader/v200/V200DolphinSchedulerUpgrader.java +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/upgrader/v200/V200DolphinSchedulerUpgrader.java @@ -32,15 +32,15 @@ import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog; import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelationLog; import org.apache.dolphinscheduler.dao.entity.TaskDefinitionLog; -import org.apache.dolphinscheduler.dao.upgrade.JsonSplitDao; -import org.apache.dolphinscheduler.dao.upgrade.ProcessDefinitionDao; -import org.apache.dolphinscheduler.dao.upgrade.ProjectDao; -import org.apache.dolphinscheduler.dao.upgrade.ScheduleDao; import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; import org.apache.dolphinscheduler.plugin.task.api.parameters.TaskTimeoutParameter; -import org.apache.dolphinscheduler.tools.datasource.dao.UpgradeDao; +import org.apache.dolphinscheduler.tools.datasource.dao.JsonSplitDao; +import org.apache.dolphinscheduler.tools.datasource.dao.ProcessDefinitionDao; +import org.apache.dolphinscheduler.tools.datasource.dao.ProjectDao; +import org.apache.dolphinscheduler.tools.datasource.dao.ScheduleDao; import org.apache.dolphinscheduler.tools.datasource.upgrader.DolphinSchedulerUpgrader; import org.apache.dolphinscheduler.tools.datasource.upgrader.DolphinSchedulerVersion; +import org.apache.dolphinscheduler.tools.datasource.upgrader.UpgradeDao; import org.apache.commons.collections4.CollectionUtils; diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/upgrader/v320/V320DolphinSchedulerUpgrader.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/upgrader/v320/V320DolphinSchedulerUpgrader.java index 61405b7ffc..5c8378a70e 100644 --- a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/upgrader/v320/V320DolphinSchedulerUpgrader.java +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/upgrader/v320/V320DolphinSchedulerUpgrader.java @@ -18,59 +18,41 @@ package org.apache.dolphinscheduler.tools.datasource.upgrader.v320; import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog; -import org.apache.dolphinscheduler.dao.entity.ProcessInstance; -import org.apache.dolphinscheduler.dao.entity.Schedule; -import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.apache.dolphinscheduler.dao.entity.User; -import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionLogMapper; -import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; -import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper; -import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; -import org.apache.dolphinscheduler.dao.mapper.UserMapper; -import org.apache.dolphinscheduler.tools.datasource.dao.UpgradeDao; import org.apache.dolphinscheduler.tools.datasource.upgrader.DolphinSchedulerUpgrader; import org.apache.dolphinscheduler.tools.datasource.upgrader.DolphinSchedulerVersion; +import org.apache.dolphinscheduler.tools.datasource.upgrader.UpgradeDao; import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; +import org.apache.commons.collections4.MapUtils; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; +import javax.sql.DataSource; + +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; -import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; -import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; - @Slf4j @Component public class V320DolphinSchedulerUpgrader implements DolphinSchedulerUpgrader { @Autowired - private ProcessInstanceMapper processInstanceMapper; - - @Autowired - private ProcessDefinitionLogMapper processDefinitionLogMapper; + private DataSource dataSource; - @Autowired - private ScheduleMapper scheduleMapper; - - @Autowired - private UserMapper userMapper; - - @Autowired - private TaskInstanceMapper taskInstanceMapper; - - @Lazy() @Autowired private UpgradeDao upgradeDao; + @SneakyThrows @Override public void doUpgrade() { upgradeWorkflowInstance(); @@ -80,68 +62,219 @@ public class V320DolphinSchedulerUpgrader implements DolphinSchedulerUpgrader { } private void upgradeWorkflowInstance() { - Map userMap = userMapper.selectList(new QueryWrapper<>()) - .stream() - .collect(Collectors.toMap(User::getId, User::getUserName)); - + Map userMap = getUserMap(); while (true) { - LambdaQueryWrapper wrapper = new QueryWrapper() - .lambda() - .eq(ProcessInstance::getProjectCode, null) - .last("limit 1000"); - List needUpdateWorkflowInstance = processInstanceMapper.selectList(wrapper); - if (CollectionUtils.isEmpty(needUpdateWorkflowInstance)) { + List> needUpdateWorkflowInstances = getProcessInstanceWhichProjectCodeIsNull(); + if (CollectionUtils.isEmpty(needUpdateWorkflowInstances)) { return; } - needUpdateWorkflowInstance.parallelStream() + needUpdateWorkflowInstances.parallelStream() .forEach(processInstance -> { - ProcessDefinitionLog processDefinitionLog = - processDefinitionLogMapper.queryByDefinitionCodeAndVersion( - processInstance.getProcessDefinitionCode(), - processInstance.getProcessDefinitionVersion()); - Schedule schedule = - scheduleMapper.queryByProcessDefinitionCode(processInstance.getProcessDefinitionCode()); - if (processDefinitionLog != null) { - processInstance.setProjectCode(processDefinitionLog.getProjectCode()); - processInstance.setTenantCode( - StringUtils.defaultIfEmpty(schedule.getTenantCode(), Constants.DEFAULT)); - processInstance.setExecutorName(userMap.get(processInstance.getExecutorId())); - } else { - processInstance.setProjectCode(-1L); + Integer id = (Integer) processInstance.get("id"); + Long processDefinitionCode = (Long) processInstance.get("process_definition_code"); + Integer processDefinitionVersion = (Integer) processInstance.get("process_definition_version"); + + Map processDefinitionLog = + getProcessDefinitionLogByCode(processDefinitionCode, processDefinitionVersion); + + Long projectCode = -1L; + String tenantCode = null; + String executorName = null; + if (MapUtils.isNotEmpty(processDefinitionLog)) { + Map scheduler = getSchedulerByProcessDefinitionCode(processDefinitionCode); + projectCode = processDefinitionLog.get("project_code") == null ? -1L + : (Long) processDefinitionLog.get("project_code"); + tenantCode = scheduler.get("tenant_code") == null ? Constants.DEFAULT + : (String) scheduler.get("tenant_code"); + executorName = userMap.get((Integer) processInstance.get("executor_id")); } - processInstanceMapper.updateById(processInstance); + updateProjectCodeInProcessInstance(id, projectCode, tenantCode, executorName); }); - log.info("Success upgrade workflow instance, current batch size: {}", needUpdateWorkflowInstance.size()); + log.info("Success upgrade workflow instance, current batch size: {}", needUpdateWorkflowInstances.size()); } } private void upgradeTaskInstance() { while (true) { - LambdaQueryWrapper wrapper = new QueryWrapper() - .lambda() - .eq(TaskInstance::getProjectCode, null) - .last("limit 1000"); - List taskInstances = taskInstanceMapper.selectList(wrapper); + List> taskInstances = getTaskInstanceWhichProjectCodeIsNull(); if (CollectionUtils.isEmpty(taskInstances)) { return; } + taskInstances.parallelStream() .forEach(taskInstance -> { - ProcessInstance processInstance = - processInstanceMapper.selectById(taskInstance.getProcessInstanceId()); - if (processInstance == null) { - taskInstance.setProjectCode(-1L); - } else { - taskInstance.setProjectCode(processInstance.getProjectCode()); - taskInstance.setProcessInstanceName(processInstance.getName()); - taskInstance.setExecutorName(processInstance.getExecutorName()); + Integer id = (Integer) taskInstance.get("id"); + Integer processInstanceId = (Integer) taskInstance.get("process_instance_id"); + Map processInstance = getProcessInstanceById(processInstanceId); + + Long projectCode = -1L; + String processInstanceName = null; + String executorName = null; + + if (MapUtils.isNotEmpty(processInstance)) { + projectCode = processInstance.get("project_code") == null ? -1L + : (Long) processInstance.get("project_code"); + processInstanceName = (String) processInstance.get("name"); + executorName = (String) processInstance.get("executor_name"); } - taskInstanceMapper.updateById(taskInstance); + updateProjectCodeInTaskInstance(id, projectCode, processInstanceName, executorName); }); log.info("Success upgrade task instance, current batch size: {}", taskInstances.size()); } } + private List> getTaskInstanceWhichProjectCodeIsNull() { + List> processInstanceList = new ArrayList<>(); + try ( + Connection connection = dataSource.getConnection(); + PreparedStatement preparedStatement = connection.prepareStatement( + "select id, process_instance_id from t_ds_task_instance where project_code is null limit 1000"); + ResultSet resultSet = preparedStatement.executeQuery()) { + while (resultSet.next()) { + Map row = new HashMap<>(); + row.put("id", resultSet.getInt("id")); + row.put("process_instance_id", resultSet.getInt("process_instance_id")); + processInstanceList.add(row); + } + return processInstanceList; + } catch (Exception ex) { + throw new RuntimeException("Query t_ds_process_instance error", ex); + } + + } + + private Map getUserMap() { + Map userMap = new HashMap<>(); + try ( + Connection connection = dataSource.getConnection(); + PreparedStatement preparedStatement = + connection.prepareStatement("select id, user_name from t_ds_user"); + ResultSet resultSet = preparedStatement.executeQuery()) { + while (resultSet.next()) { + userMap.put(resultSet.getInt("id"), resultSet.getString("user_name")); + } + } catch (Exception ex) { + throw new RuntimeException("Query t_ds_user error", ex); + } + return userMap; + } + + private List> getProcessInstanceWhichProjectCodeIsNull() { + List> processInstanceList = new ArrayList<>(); + try ( + Connection connection = dataSource.getConnection(); + PreparedStatement preparedStatement = connection.prepareStatement( + "select id, process_definition_code, process_definition_version, executor_id from t_ds_process_instance where project_code is null limit 1000"); + ResultSet resultSet = preparedStatement.executeQuery()) { + while (resultSet.next()) { + Map row = new HashMap<>(); + row.put("id", resultSet.getInt("id")); + row.put("process_definition_code", resultSet.getLong("process_definition_code")); + row.put("process_definition_version", resultSet.getInt("process_definition_version")); + row.put("executor_id", resultSet.getInt("executor_id")); + processInstanceList.add(row); + } + return processInstanceList; + } catch (Exception ex) { + throw new RuntimeException("Query t_ds_process_instance error", ex); + } + } + + private Map getProcessInstanceById(Integer processInstanceId) { + try ( + Connection connection = dataSource.getConnection(); + PreparedStatement preparedStatement = connection.prepareStatement( + "select project_code, name, executor_name from t_ds_process_instance where id = ?");) { + preparedStatement.setInt(1, processInstanceId); + try (ResultSet resultSet = preparedStatement.executeQuery()) { + while (resultSet.next()) { + Map row = new HashMap<>(); + row.put("project_code", resultSet.getLong("project_code")); + row.put("name", resultSet.getString("name")); + row.put("executor_name", resultSet.getString("executor_name")); + return row; + } + } + return Collections.emptyMap(); + } catch (Exception ex) { + throw new RuntimeException("Query t_ds_process_instance error", ex); + } + } + + private Map getProcessDefinitionLogByCode(Long processDefinitionCode, + Integer processDefinitionVersion) { + try ( + Connection connection = dataSource.getConnection(); + PreparedStatement preparedStatement = connection.prepareStatement( + "select project_code from t_ds_process_definition_log where code = ? and version = ?")) { + preparedStatement.setLong(1, processDefinitionCode); + preparedStatement.setInt(2, processDefinitionVersion); + + try (ResultSet resultSet = preparedStatement.executeQuery()) { + while (resultSet.next()) { + Map row = new HashMap<>(); + row.put("project_code", resultSet.getLong("project_code")); + return row; + } + } + return Collections.emptyMap(); + } catch (Exception ex) { + throw new RuntimeException("Query t_ds_process_definition_log error", ex); + } + } + + private Map getSchedulerByProcessDefinitionCode(Long processDefinitionCode) { + try ( + Connection connection = dataSource.getConnection(); + PreparedStatement preparedStatement = connection + .prepareStatement("select * from t_ds_schedules where process_definition_code = ?")) { + preparedStatement.setLong(1, processDefinitionCode); + + try (ResultSet resultSet = preparedStatement.executeQuery()) { + while (resultSet.next()) { + Map row = new HashMap<>(); + row.put("tenant_code", resultSet.getString("tenant_code")); + return row; + } + } + return Collections.emptyMap(); + } catch (Exception ex) { + throw new RuntimeException("Query t_ds_schedules error", ex); + } + } + + private void updateProjectCodeInProcessInstance(Integer processInstanceId, Long projectCode, String tenantCode, + String executorName) { + try ( + Connection connection = dataSource.getConnection(); + PreparedStatement preparedStatement = connection.prepareStatement( + "update t_ds_process_instance set project_code = ?, tenant_code = ?, executor_name = ? where id = ?")) { + preparedStatement.setLong(1, projectCode); + preparedStatement.setString(2, tenantCode); + preparedStatement.setString(3, executorName); + preparedStatement.setInt(4, processInstanceId); + preparedStatement.executeUpdate(); + } catch (Exception ex) { + throw new RuntimeException("Update t_ds_process_instance error", ex); + } + } + + private void updateProjectCodeInTaskInstance(Integer id, Long projectCode, String processInstanceName, + String executorName) { + try ( + Connection connection = dataSource.getConnection(); + PreparedStatement preparedStatement = connection.prepareStatement( + "update t_ds_task_instance set project_code = ?, process_instance_name = ?, executor_name = ? where id = ?")) { + preparedStatement.setLong(1, projectCode); + preparedStatement.setString(2, processInstanceName); + preparedStatement.setString(3, executorName); + preparedStatement.setInt(4, id); + preparedStatement.executeUpdate(); + } catch (Exception ex) { + throw new RuntimeException("Update t_ds_process_instance error", ex); + } + } + @Override public DolphinSchedulerVersion getCurrentVersion() { return DolphinSchedulerVersion.V3_2_0; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/SchemaUtils.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/utils/SchemaUtils.java similarity index 98% rename from dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/SchemaUtils.java rename to dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/utils/SchemaUtils.java index 25164a6059..bafe9e2017 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/SchemaUtils.java +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/utils/SchemaUtils.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.dao.upgrade; +package org.apache.dolphinscheduler.tools.datasource.utils; import org.apache.dolphinscheduler.common.utils.FileUtils; diff --git a/dolphinscheduler-tools/src/main/resources/application.yaml b/dolphinscheduler-tools/src/main/resources/application.yaml index a6688770b4..38752021dc 100644 --- a/dolphinscheduler-tools/src/main/resources/application.yaml +++ b/dolphinscheduler-tools/src/main/resources/application.yaml @@ -34,6 +34,20 @@ spring: leak-detection-threshold: 0 initialization-fail-timeout: 1 +# Mybatis-plus configuration, you don't need to change it +mybatis-plus: + mapper-locations: classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml + type-aliases-package: org.apache.dolphinscheduler.dao.entity + configuration: + cache-enabled: false + call-setters-on-nulls: true + map-underscore-to-camel-case: true + jdbc-type-for-null: NULL + global-config: + db-config: + id-type: auto + banner: false + demo: tenant-code: default domain-name: localhost diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/SchemaUtilsTest.java b/dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/utils/SchemaUtilsTest.java similarity index 66% rename from dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/SchemaUtilsTest.java rename to dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/utils/SchemaUtilsTest.java index b157203def..151c813438 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/SchemaUtilsTest.java +++ b/dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/utils/SchemaUtilsTest.java @@ -1,21 +1,24 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. */ -package org.apache.dolphinscheduler.dao.upgrade; +package org.apache.dolphinscheduler.tools.datasource.utils; import org.apache.commons.collections4.CollectionUtils; @@ -25,10 +28,10 @@ import java.util.List; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -public class SchemaUtilsTest { +class SchemaUtilsTest { @Test - public void testIsAGreatVersion() { + void testIsAGreatVersion() { // param is null try { SchemaUtils.isAGreatVersion(null, null); @@ -64,7 +67,7 @@ public class SchemaUtilsTest { } @Test - public void testGetAllSchemaList() { + void testGetAllSchemaList() { List list = null; try { list = SchemaUtils.getAllSchemaList(); @@ -73,4 +76,5 @@ public class SchemaUtilsTest { } Assertions.assertFalse(CollectionUtils.isEmpty(list), "Can not find any schema files"); } + } diff --git a/dolphinscheduler-tools/src/test/resources/application-mysql.yaml b/dolphinscheduler-tools/src/test/resources/application-mysql.yaml index 48cdd2b2c5..0b553c2bda 100644 --- a/dolphinscheduler-tools/src/test/resources/application-mysql.yaml +++ b/dolphinscheduler-tools/src/test/resources/application-mysql.yaml @@ -34,6 +34,19 @@ spring: leak-detection-threshold: 0 initialization-fail-timeout: 1 +mybatis-plus: + mapper-locations: classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml + type-aliases-package: org.apache.dolphinscheduler.dao.entity + configuration: + cache-enabled: false + call-setters-on-nulls: true + map-underscore-to-camel-case: true + jdbc-type-for-null: NULL + global-config: + db-config: + id-type: auto + banner: false + demo: tenant-code: default domain-name: localhost diff --git a/dolphinscheduler-tools/src/test/resources/application-postgresql.yaml b/dolphinscheduler-tools/src/test/resources/application-postgresql.yaml index 46043a7e76..21daaa0606 100644 --- a/dolphinscheduler-tools/src/test/resources/application-postgresql.yaml +++ b/dolphinscheduler-tools/src/test/resources/application-postgresql.yaml @@ -34,6 +34,19 @@ spring: leak-detection-threshold: 0 initialization-fail-timeout: 1 +mybatis-plus: + mapper-locations: classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml + type-aliases-package: org.apache.dolphinscheduler.dao.entity + configuration: + cache-enabled: false + call-setters-on-nulls: true + map-underscore-to-camel-case: true + jdbc-type-for-null: NULL + global-config: + db-config: + id-type: auto + banner: false + demo: tenant-code: default domain-name: localhost diff --git a/dolphinscheduler-tools/src/test/resources/logback.xml b/dolphinscheduler-tools/src/test/resources/logback.xml index 6c17361120..d820a8d013 100644 --- a/dolphinscheduler-tools/src/test/resources/logback.xml +++ b/dolphinscheduler-tools/src/test/resources/logback.xml @@ -22,7 +22,7 @@ - [%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{96}:[%line] - [WorkflowInstance-%X{workflowInstanceId:-0}][TaskInstance-%X{taskInstanceId:-0}] - %msg%n + [%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{96}:[%line] - %msg%n UTF-8 diff --git a/dolphinscheduler-tools/src/test/resources/sql/upgrade/3.3.0_schema/mysql/dolphinscheduler_ddl.sql b/dolphinscheduler-tools/src/test/resources/sql/upgrade/3.3.0_schema/mysql/dolphinscheduler_ddl.sql new file mode 100644 index 0000000000..4a14f326b9 --- /dev/null +++ b/dolphinscheduler-tools/src/test/resources/sql/upgrade/3.3.0_schema/mysql/dolphinscheduler_ddl.sql @@ -0,0 +1,16 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ diff --git a/dolphinscheduler-tools/src/test/resources/sql/upgrade/3.3.0_schema/mysql/dolphinscheduler_ddl_post.sql b/dolphinscheduler-tools/src/test/resources/sql/upgrade/3.3.0_schema/mysql/dolphinscheduler_ddl_post.sql new file mode 100644 index 0000000000..4a14f326b9 --- /dev/null +++ b/dolphinscheduler-tools/src/test/resources/sql/upgrade/3.3.0_schema/mysql/dolphinscheduler_ddl_post.sql @@ -0,0 +1,16 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ diff --git a/dolphinscheduler-tools/src/test/resources/sql/upgrade/3.3.0_schema/mysql/dolphinscheduler_dml.sql b/dolphinscheduler-tools/src/test/resources/sql/upgrade/3.3.0_schema/mysql/dolphinscheduler_dml.sql new file mode 100644 index 0000000000..4a14f326b9 --- /dev/null +++ b/dolphinscheduler-tools/src/test/resources/sql/upgrade/3.3.0_schema/mysql/dolphinscheduler_dml.sql @@ -0,0 +1,16 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ diff --git a/dolphinscheduler-tools/src/test/resources/sql/upgrade/3.3.0_schema/postgresql/dolphinscheduler_ddl.sql b/dolphinscheduler-tools/src/test/resources/sql/upgrade/3.3.0_schema/postgresql/dolphinscheduler_ddl.sql new file mode 100644 index 0000000000..4a14f326b9 --- /dev/null +++ b/dolphinscheduler-tools/src/test/resources/sql/upgrade/3.3.0_schema/postgresql/dolphinscheduler_ddl.sql @@ -0,0 +1,16 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ diff --git a/dolphinscheduler-tools/src/test/resources/sql/upgrade/3.3.0_schema/postgresql/dolphinscheduler_ddl_post.sql b/dolphinscheduler-tools/src/test/resources/sql/upgrade/3.3.0_schema/postgresql/dolphinscheduler_ddl_post.sql new file mode 100644 index 0000000000..4a14f326b9 --- /dev/null +++ b/dolphinscheduler-tools/src/test/resources/sql/upgrade/3.3.0_schema/postgresql/dolphinscheduler_ddl_post.sql @@ -0,0 +1,16 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ diff --git a/dolphinscheduler-tools/src/test/resources/sql/upgrade/3.3.0_schema/postgresql/dolphinscheduler_dml.sql b/dolphinscheduler-tools/src/test/resources/sql/upgrade/3.3.0_schema/postgresql/dolphinscheduler_dml.sql new file mode 100644 index 0000000000..4a14f326b9 --- /dev/null +++ b/dolphinscheduler-tools/src/test/resources/sql/upgrade/3.3.0_schema/postgresql/dolphinscheduler_dml.sql @@ -0,0 +1,16 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/