From 5bfe3fca673d9f0a0622ef5c3f089e52a9599641 Mon Sep 17 00:00:00 2001 From: zhuangchong <37063904+zhuangchong@users.noreply.github.com> Date: Wed, 3 Mar 2021 09:15:37 +0800 Subject: [PATCH] [Fix][SqlTask] fix hive sqltask invalid connection parameters. (#4917) * fix hive sqltask invalid connection parameters. * add setConnParams test method. * update base datasource test class code style. * add BaseDataSourceTest in the pom xml. * update hive conf list jdbc url stitching. * update hive datasource test class. --- .../dao/datasource/BaseDataSource.java | 26 ++++++++++++++ .../dao/datasource/HiveDataSource.java | 17 ++++++++++ .../dao/datasource/BaseDataSourceTest.java | 34 +++++++++++++++++++ .../dao/datasource/HiveDataSourceTest.java | 2 +- .../server/worker/task/sql/SqlTask.java | 3 ++ pom.xml | 1 + 6 files changed, 82 insertions(+), 1 deletion(-) diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java index 9bbbf32a6f..e26a3751fb 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java @@ -17,12 +17,16 @@ package org.apache.dolphinscheduler.dao.datasource; +import static org.apache.dolphinscheduler.common.Constants.PASSWORD; +import static org.apache.dolphinscheduler.common.Constants.USER; + import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; import java.sql.Connection; import java.sql.DriverManager; +import java.util.Properties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -157,6 +161,11 @@ public abstract class BaseDataSource { separator = ":"; break; case HIVE: + if ("?".equals(otherParams.substring(0, 1))) { + break; + } + separator = ";"; + break; case SPARK: case SQLSERVER: separator = ";"; @@ -178,6 +187,19 @@ public abstract class BaseDataSource { return DriverManager.getConnection(getJdbcUrl(), getUser(), getPassword()); } + /** + * the data source test connection + * @param info Properties + * @return Connection Connection + * @throws Exception Exception + */ + public Connection getConnection(Properties info) throws Exception { + Class.forName(driverClassSelector()); + info.setProperty(USER, getUser()); + info.setProperty(PASSWORD, getPassword()); + return DriverManager.getConnection(getJdbcUrl(), info); + } + protected String filterOther(String otherParams) { return otherParams; } @@ -226,6 +248,10 @@ public abstract class BaseDataSource { this.other = other; } + public void setConnParams(String connParams) { + + } + public String getJavaSecurityKrb5Conf() { return javaSecurityKrb5Conf; } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java index bcf1cdf3d2..8fccb08bb7 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java @@ -17,13 +17,17 @@ package org.apache.dolphinscheduler.dao.datasource; +import static org.apache.dolphinscheduler.common.Constants.SEMICOLON; + import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.dolphinscheduler.common.utils.HiveConfUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; import java.sql.Connection; +import java.util.Map; /** * data source of hive @@ -100,4 +104,17 @@ public class HiveDataSource extends BaseDataSource { return super.getConnection(); } + @Override + public void setConnParams(String connParams) { + // Verification parameters + Map connParamMap = CollectionUtils.stringToMap(connParams, SEMICOLON); + if (connParamMap.isEmpty()) { + return; + } + + StringBuilder otherSb = new StringBuilder(); + connParamMap.forEach((k, v) -> otherSb.append(String.format("%s=%s%s", k, v, SEMICOLON))); + StringBuilder otherAppend = StringUtils.isNotBlank(getOther()) ? otherSb.append(getOther()) : otherSb.deleteCharAt(otherSb.length() - 1); + super.setOther(otherAppend.toString()); + } } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSourceTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSourceTest.java index 1d02fd4fe3..31c963d5d7 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSourceTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSourceTest.java @@ -158,4 +158,38 @@ public class BaseDataSourceTest { } + @Test + public void testSetConnParams() { + + BaseDataSource hiveDataSource = new HiveDataSource(); + hiveDataSource.setAddress("jdbc:hive2://127.0.0.1:10000"); + hiveDataSource.setDatabase("test"); + hiveDataSource.setPassword("123456"); + hiveDataSource.setUser("test"); + hiveDataSource.setConnParams(""); + Assert.assertEquals("jdbc:hive2://127.0.0.1:10000/test", hiveDataSource.getJdbcUrl()); + + //set fake other + hiveDataSource.setConnParams("hive.tez.container.size=20000;"); + Assert.assertEquals("jdbc:hive2://127.0.0.1:10000/test?hive.tez.container.size=20000", hiveDataSource.getJdbcUrl()); + + hiveDataSource.setOther(null); + hiveDataSource.setConnParams("hive.tez.container.size=20000"); + Assert.assertEquals("jdbc:hive2://127.0.0.1:10000/test?hive.tez.container.size=20000", hiveDataSource.getJdbcUrl()); + + hiveDataSource.setOther(null); + hiveDataSource.setConnParams("hive.tez.container.size=20000;hive.zzz=100"); + Assert.assertEquals("jdbc:hive2://127.0.0.1:10000/test;hive.zzz=100?hive.tez.container.size=20000", hiveDataSource.getJdbcUrl()); + + hiveDataSource.setOther("charset=UTF-8"); + Assert.assertEquals("jdbc:hive2://127.0.0.1:10000/test;charset=UTF-8", hiveDataSource.getJdbcUrl()); + + hiveDataSource.setConnParams("hive.tez.container.size=20000;hive.zzz=100"); + Assert.assertEquals("jdbc:hive2://127.0.0.1:10000/test;hive.zzz=100;charset=UTF-8?hive.tez.container.size=20000", hiveDataSource.getJdbcUrl()); + + hiveDataSource.setOther("charset=UTF-8;hive.exec.stagingdir=/tmp"); + hiveDataSource.setConnParams("hive.tez.container.size=20000;hive.zzz=100"); + Assert.assertEquals("jdbc:hive2://127.0.0.1:10000/test;hive.zzz=100;charset=UTF-8?hive.tez.container.size=20000;hive.exec.stagingdir=/tmp", hiveDataSource.getJdbcUrl()); + } + } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSourceTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSourceTest.java index 3920fd39b2..145b7f38d2 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSourceTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSourceTest.java @@ -81,7 +81,7 @@ public class HiveDataSourceTest { hiveDataSource.setOther("hive.mapred.mode=strict;hive.server2.thrift.http.path=hs2"); Assert.assertEquals( - "jdbc:hive2://127.0.0.1:10000/test;?hive.mapred.mode=strict;hive.server2.thrift.http.path=hs2", + "jdbc:hive2://127.0.0.1:10000/test?hive.mapred.mode=strict;hive.server2.thrift.http.path=hs2", hiveDataSource.getJdbcUrl()); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java index 5a16194b3b..8f21d5e95c 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java @@ -244,6 +244,9 @@ public class SqlTask extends AbstractTask { PreparedStatement stmt = null; ResultSet resultSet = null; try { + + baseDataSource.setConnParams(sqlParameters.getConnParams()); + // create connection connection = baseDataSource.getConnection(); // create temp function diff --git a/pom.xml b/pom.xml index a07fde2fd1..2829dca7ac 100644 --- a/pom.xml +++ b/pom.xml @@ -990,6 +990,7 @@ **/dao/AlertDaoTest.java **/dao/datasource/OracleDataSourceTest.java **/dao/datasource/HiveDataSourceTest.java + **/dao/datasource/BaseDataSourceTest.java **/dao/upgrade/ProcessDefinitionDaoTest.java **/dao/upgrade/WokrerGrouopDaoTest.java **/dao/upgrade/UpgradeDaoTest.java