From 6b8461e901de65000f4f8f34ebfe011946e787d3 Mon Sep 17 00:00:00 2001 From: Shiwen Cheng Date: Sun, 25 Apr 2021 23:29:00 +0800 Subject: [PATCH] Revert "[Improvement][DAO] CreateDolphinScheduler (#5357) (#5358)" (#5383) This reverts commit b108ac43b8489ad1d744afd0321d4ec150880548. --- dolphinscheduler-dao/pom.xml | 17 --- .../dao/upgrade/DolphinSchedulerManager.java | 11 +- .../dao/upgrade/UpgradeDao.java | 137 +++++++++++++----- .../upgrade/shell/CreateDolphinScheduler.java | 34 ++--- .../upgrade/shell/InitDolphinScheduler.java | 23 ++- .../dao/upgrade/UpgradeDaoTest.java | 69 +-------- .../shell/CreateDolphinSchedulerTest.java | 40 ----- .../shell/InitDolphinSchedulerTest.java | 40 ----- pom.xml | 2 - 9 files changed, 137 insertions(+), 236 deletions(-) delete mode 100644 dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/shell/CreateDolphinSchedulerTest.java delete mode 100644 dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/shell/InitDolphinSchedulerTest.java diff --git a/dolphinscheduler-dao/pom.xml b/dolphinscheduler-dao/pom.xml index 3adb3077fd..4fff6575dc 100644 --- a/dolphinscheduler-dao/pom.xml +++ b/dolphinscheduler-dao/pom.xml @@ -151,22 +151,5 @@ org.yaml snakeyaml - - org.powermock - powermock-module-junit4 - test - - - - org.powermock - powermock-api-mockito2 - test - - - org.mockito - mockito-core - - - diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/DolphinSchedulerManager.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/DolphinSchedulerManager.java index aa6bf62299..b2daae28cb 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/DolphinSchedulerManager.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/DolphinSchedulerManager.java @@ -14,19 +14,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.dolphinscheduler.dao.upgrade; import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.utils.SchemaUtils; - -import java.io.IOException; -import java.sql.SQLException; -import java.util.List; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.List; /** * upgrade manager @@ -65,7 +60,7 @@ public class DolphinSchedulerManager { /** * init DolphinScheduler */ - public void initDolphinScheduler() throws SQLException, IOException { + public void initDolphinScheduler() { // Determines whether the dolphinscheduler table structure has been init if (upgradeDao.isExistsTable("t_escheduler_version") || upgradeDao.isExistsTable("t_ds_version") || @@ -79,7 +74,7 @@ public class DolphinSchedulerManager { /** * init DolphinScheduler Schema */ - public void initDolphinSchedulerSchema() throws SQLException, IOException { + public void initDolphinSchedulerSchema() { logger.info("Start initializing the DolphinScheduler manager table structure"); upgradeDao.initSchema(); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java index 67193df532..f0ffc52f68 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java @@ -14,25 +14,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.dolphinscheduler.dao.upgrade; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.process.ResourceInfo; -import org.apache.dolphinscheduler.common.utils.CollectionUtils; -import org.apache.dolphinscheduler.common.utils.ConnectionUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.common.utils.SchemaUtils; -import org.apache.dolphinscheduler.common.utils.ScriptRunner; -import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.dao.AbstractBaseDao; import org.apache.dolphinscheduler.dao.datasource.ConnectionFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileReader; -import java.io.IOException; -import java.io.Reader; +import javax.sql.DataSource; +import java.io.*; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -43,14 +38,6 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import javax.sql.DataSource; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; - public abstract class UpgradeDao extends AbstractBaseDao { public static final Logger logger = LoggerFactory.getLogger(UpgradeDao.class); @@ -60,9 +47,6 @@ public abstract class UpgradeDao extends AbstractBaseDao { protected static final DataSource dataSource = getDataSource(); private static final DbType dbType = getCurrentDbType(); - private static final String MYSQL_CREATE_SCRIPT = rootDir + "/sql/dolphinscheduler_mysql.sql"; - private static final String POSTGRE_CREATE_SCRIPT = rootDir + "/sql/dolphinscheduler_postgre.sql"; - @Override protected void init() { @@ -106,40 +90,119 @@ public abstract class UpgradeDao extends AbstractBaseDao { /** * init schema */ - public void initSchema() throws SQLException, IOException { + public void initSchema() { DbType dbType = getDbType(); String initSqlPath = ""; if (dbType != null) { switch (dbType) { case MYSQL: - initSqlPath = MYSQL_CREATE_SCRIPT; + initSqlPath = "/sql/create/release-1.0.0_schema/mysql/"; + initSchema(initSqlPath); break; case POSTGRESQL: - initSqlPath = POSTGRE_CREATE_SCRIPT; + initSqlPath = "/sql/create/release-1.2.0_schema/postgresql/"; + initSchema(initSqlPath); break; default: logger.error("not support sql type: {},can't upgrade", dbType); throw new IllegalArgumentException("not support sql type,can't upgrade"); } } + } + + + /** + * init scheam + * + * @param initSqlPath initSqlPath + */ + public void initSchema(String initSqlPath) { + + // Execute the dolphinscheduler DDL, it cannot be rolled back + runInitDDL(initSqlPath); + + // Execute the dolphinscheduler DML, it can be rolled back + runInitDML(initSqlPath); + + + } + + /** + * run DML + * + * @param initSqlPath initSqlPath + */ + private void runInitDML(String initSqlPath) { + Connection conn = null; if (StringUtils.isEmpty(rootDir)) { throw new RuntimeException("Environment variable user.dir not found"); } - logger.info("Init sql filePath: {}", initSqlPath); - try (Connection conn = dataSource.getConnection()) { + String mysqlSQLFilePath = rootDir + initSqlPath + "dolphinscheduler_dml.sql"; + try { + conn = dataSource.getConnection(); + conn.setAutoCommit(false); + + // Execute the dolphinscheduler_dml.sql script to import related data of dolphinscheduler + ScriptRunner initScriptRunner = new ScriptRunner(conn, false, true); + Reader initSqlReader = new FileReader(new File(mysqlSQLFilePath)); + initScriptRunner.runScript(initSqlReader); + + conn.commit(); + } catch (IOException e) { try { - conn.setAutoCommit(false); - ScriptRunner initScriptRunner = new ScriptRunner(conn, false, true); - Reader initSqlReader = new FileReader(initSqlPath); - initScriptRunner.runScript(initSqlReader); - conn.commit(); - } catch (IOException | SQLException e) { conn.rollback(); - logger.error("execute init script error.", e); - throw e; + } catch (SQLException e1) { + logger.error(e1.getMessage(), e1); } + logger.error(e.getMessage(), e); + throw new RuntimeException(e.getMessage(), e); + } catch (Exception e) { + try { + if (null != conn) { + conn.rollback(); + } + } catch (SQLException e1) { + logger.error(e1.getMessage(), e1); + } + logger.error(e.getMessage(), e); + throw new RuntimeException(e.getMessage(), e); } finally { - // ignore + ConnectionUtils.releaseResource(conn); + + } + + } + + /** + * run DDL + * + * @param initSqlPath initSqlPath + */ + private void runInitDDL(String initSqlPath) { + Connection conn = null; + if (StringUtils.isEmpty(rootDir)) { + throw new RuntimeException("Environment variable user.dir not found"); + } + //String mysqlSQLFilePath = rootDir + "/sql/create/release-1.0.0_schema/mysql/dolphinscheduler_ddl.sql"; + String mysqlSQLFilePath = rootDir + initSqlPath + "dolphinscheduler_ddl.sql"; + try { + conn = dataSource.getConnection(); + // Execute the dolphinscheduler_ddl.sql script to create the table structure of dolphinscheduler + ScriptRunner initScriptRunner = new ScriptRunner(conn, true, true); + Reader initSqlReader = new FileReader(new File(mysqlSQLFilePath)); + initScriptRunner.runScript(initSqlReader); + + } catch (IOException e) { + + logger.error(e.getMessage(), e); + throw new RuntimeException(e.getMessage(), e); + } catch (Exception e) { + + logger.error(e.getMessage(), e); + throw new RuntimeException(e.getMessage(), e); + } finally { + ConnectionUtils.releaseResource(conn); + } } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/shell/CreateDolphinScheduler.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/shell/CreateDolphinScheduler.java index edfe3bfefe..1c0f002567 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/shell/CreateDolphinScheduler.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/shell/CreateDolphinScheduler.java @@ -14,35 +14,35 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.dolphinscheduler.dao.upgrade.shell; import org.apache.dolphinscheduler.dao.upgrade.DolphinSchedulerManager; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * create DolphinScheduler + * */ public class CreateDolphinScheduler { - private static final Logger logger = LoggerFactory.getLogger(CreateDolphinScheduler.class); + private static final Logger logger = LoggerFactory.getLogger(CreateDolphinScheduler.class); - /** + /** * create dolphin scheduler db - * - * @param args args - */ - public static void main(String[] args) { - DolphinSchedulerManager dolphinSchedulerManager = new DolphinSchedulerManager(); - try { - logger.info("create DolphinScheduler begin"); - dolphinSchedulerManager.initDolphinScheduler(); - logger.info("create DolphinScheduler success"); - } catch (Exception e) { - logger.error("create DolphinScheduler failed", e); - } + * @param args args + */ + public static void main(String[] args) { + DolphinSchedulerManager dolphinSchedulerManager = new DolphinSchedulerManager(); + try { + dolphinSchedulerManager.initDolphinScheduler(); + logger.info("init DolphinScheduler finished"); + dolphinSchedulerManager.upgradeDolphinScheduler(); + logger.info("upgrade DolphinScheduler finished"); + logger.info("create DolphinScheduler success"); + } catch (Exception e) { + logger.error("create DolphinScheduler failed",e); + } - } + } } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/shell/InitDolphinScheduler.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/shell/InitDolphinScheduler.java index 6abe184d2a..3bcb3abc03 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/shell/InitDolphinScheduler.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/shell/InitDolphinScheduler.java @@ -14,34 +14,29 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.dolphinscheduler.dao.upgrade.shell; import org.apache.dolphinscheduler.dao.upgrade.DolphinSchedulerManager; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * init DolphinScheduler + * */ public class InitDolphinScheduler { - private static final Logger logger = LoggerFactory.getLogger(InitDolphinScheduler.class); + private static final Logger logger = LoggerFactory.getLogger(InitDolphinScheduler.class); /** * init dolphin scheduler db * @param args args */ - public static void main(String[] args) { - Thread.currentThread().setName("manager-InitDolphinScheduler"); - DolphinSchedulerManager dolphinSchedulerManager = new DolphinSchedulerManager(); - try { - dolphinSchedulerManager.initDolphinScheduler(); - logger.info("init DolphinScheduler finished"); - } catch (Exception ex) { - logger.error("init DolphinScheduler error", ex); - } - - } + public static void main(String[] args) { + Thread.currentThread().setName("manager-InitDolphinScheduler"); + DolphinSchedulerManager dolphinSchedulerManager = new DolphinSchedulerManager(); + dolphinSchedulerManager.initDolphinScheduler(); + logger.info("init DolphinScheduler finished"); + + } } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDaoTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDaoTest.java index 986149f98e..ed96e920f5 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDaoTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDaoTest.java @@ -14,76 +14,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.dolphinscheduler.dao.upgrade; -import org.apache.dolphinscheduler.common.utils.ScriptRunner; -import org.apache.dolphinscheduler.dao.datasource.ConnectionFactory; - -import java.io.FileReader; -import java.io.IOException; -import java.sql.Connection; -import java.sql.DatabaseMetaData; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; +import org.junit.Test; import javax.sql.DataSource; +import java.util.Map; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mockito; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; +import static org.apache.dolphinscheduler.dao.upgrade.UpgradeDao.getDataSource; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.junit.Assert.assertThat; -@RunWith(PowerMockRunner.class) -@PrepareForTest({ConnectionFactory.class, ScriptRunner.class, FileReader.class}) public class UpgradeDaoTest { + PostgresqlUpgradeDao postgresqlUpgradeDao = PostgresqlUpgradeDao.getInstance(); @Test - public void testGetCurrentVersion() throws SQLException { - PowerMockito.mockStatic(ConnectionFactory.class); - ConnectionFactory mockConnectionFactory = PowerMockito.mock(ConnectionFactory.class); - PowerMockito.when(ConnectionFactory.getInstance()).thenReturn(mockConnectionFactory); - DataSource mockDatasource = PowerMockito.mock(DataSource.class); - PowerMockito.when(mockConnectionFactory.getDataSource()).thenReturn(mockDatasource); - Connection mockConnection = PowerMockito.mock(Connection.class); - PowerMockito.when(mockDatasource.getConnection()).thenReturn(mockConnection); - PreparedStatement mockPrepareStatement = PowerMockito.mock(PreparedStatement.class); - PowerMockito.when(mockConnection.prepareStatement(Mockito.any())).thenReturn(mockPrepareStatement); - ResultSet mockResultSet = PowerMockito.mock(ResultSet.class); - PowerMockito.when(mockPrepareStatement.executeQuery()).thenReturn(mockResultSet); - - DatabaseMetaData mockMetaData = PowerMockito.mock(DatabaseMetaData.class); - PowerMockito.when(mockConnection.getMetaData()).thenReturn(mockMetaData); - PowerMockito.when(mockMetaData.getDatabaseProductName()).thenReturn("mysql"); - - UpgradeDao upgradeDao = MysqlUpgradeDao.getInstance(); - upgradeDao.getCurrentVersion("xx"); - Assert.assertTrue(true); + public void testQueryQueryAllOldWorkerGroup() throws Exception{ + postgresqlUpgradeDao.updateProcessDefinitionJsonWorkerGroup(); } - @Test(expected = IOException.class) - public void testInitSchema() throws Exception { - - PowerMockito.mockStatic(ConnectionFactory.class); - ConnectionFactory mockConnectionFactory = PowerMockito.mock(ConnectionFactory.class); - PowerMockito.when(ConnectionFactory.getInstance()).thenReturn(mockConnectionFactory); - - DataSource mockDatasource = PowerMockito.mock(DataSource.class); - PowerMockito.when(mockConnectionFactory.getDataSource()).thenReturn(mockDatasource); - Connection mockConnection = PowerMockito.mock(Connection.class); - PowerMockito.when(mockDatasource.getConnection()).thenReturn(mockConnection); - - DatabaseMetaData mockMetaData = PowerMockito.mock(DatabaseMetaData.class); - PowerMockito.when(mockConnection.getMetaData()).thenReturn(mockMetaData); - PowerMockito.when(mockMetaData.getDatabaseProductName()).thenReturn("mysql"); - - UpgradeDao upgradeDao = MysqlUpgradeDao.getInstance(); - upgradeDao.initSchema(); - Assert.assertTrue(true); - - } } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/shell/CreateDolphinSchedulerTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/shell/CreateDolphinSchedulerTest.java deleted file mode 100644 index 4c18760b70..0000000000 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/shell/CreateDolphinSchedulerTest.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.dao.upgrade.shell; - -import org.apache.dolphinscheduler.dao.upgrade.DolphinSchedulerManager; - -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; - -@RunWith(PowerMockRunner.class) -@PrepareForTest({DolphinSchedulerManager.class, CreateDolphinScheduler.class}) -public class CreateDolphinSchedulerTest { - - @Test - public void mainTest() throws Exception { - DolphinSchedulerManager mockManager = PowerMockito.mock(DolphinSchedulerManager.class); - PowerMockito.whenNew(DolphinSchedulerManager.class).withNoArguments().thenReturn(mockManager); - CreateDolphinScheduler.main(null); - Assert.assertTrue(true); - } -} \ No newline at end of file diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/shell/InitDolphinSchedulerTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/shell/InitDolphinSchedulerTest.java deleted file mode 100644 index e3103c4a01..0000000000 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/shell/InitDolphinSchedulerTest.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.dao.upgrade.shell; - -import org.apache.dolphinscheduler.dao.upgrade.DolphinSchedulerManager; - -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; - -@RunWith(PowerMockRunner.class) -@PrepareForTest({DolphinSchedulerManager.class, InitDolphinScheduler.class}) -public class InitDolphinSchedulerTest { - - @Test - public void main() throws Exception { - DolphinSchedulerManager mockManager = PowerMockito.mock(DolphinSchedulerManager.class); - PowerMockito.whenNew(DolphinSchedulerManager.class).withNoArguments().thenReturn(mockManager); - InitDolphinScheduler.main(null); - Assert.assertTrue(true); - } -} \ No newline at end of file diff --git a/pom.xml b/pom.xml index 1c054c414c..e698304dd8 100644 --- a/pom.xml +++ b/pom.xml @@ -910,8 +910,6 @@ **/dao/datasource/MySQLDataSourceTest.java **/dao/entity/TaskInstanceTest.java **/dao/entity/UdfFuncTest.java - **/dao/upgrade/shell/CreateDolphinSchedulerTest.java - **/dao/upgrade/shell/InitDolphinSchedulerTest.java **/remote/command/alert/AlertSendRequestCommandTest.java **/remote/command/alert/AlertSendResponseCommandTest.java **/remote/command/future/ResponseFutureTest.java