Browse Source

[Bug 2923] Hive JDBC connection parameter ignored (#3194)

* fix the bug #2923 (Hive JDBC connection parameter ignored) , the hive jdbc connection url ;

* fix the bug index out of bound and add some test case

* add the Licensed

* fix the checkstyle and import StringBuilder

* put HiveDataSourceTest.java to root pom maven-surefire-plugin to avoiding the sonar 0.0% Coverage.

* Update HiveDataSource.java

Co-authored-by: dailidong <dailidong66@gmail.com>
pull/3/MERGE
LEI SHENG 4 years ago committed by GitHub
parent
commit
98fdba6740
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 53
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java
  2. 85
      dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSourceTest.java
  3. 1
      pom.xml

53
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java

@ -14,10 +14,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.datasource;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
/**
* data source of hive
@ -40,4 +46,51 @@ public class HiveDataSource extends BaseDataSource {
public DbType dbTypeSelector() {
return DbType.HIVE;
}
/**
* build hive jdbc params,append : ?hive_conf_list
*
* hive jdbc url template:
*
* jdbc:hive2://<host1>:<port1>,<host2>:<port2>/dbName;initFile=<file>;sess_var_list?hive_conf_list#hive_var_list
*
* @param otherParams otherParams
* @return filter otherParams
*/
@Override
protected String filterOther(String otherParams) {
if (StringUtils.isBlank(otherParams)) {
return "";
}
StringBuilder hiveConfListSb = new StringBuilder();
hiveConfListSb.append("?");
StringBuilder sessionVarListSb = new StringBuilder();
String[] otherArray = otherParams.split(";", -1);
// get the default hive conf var name
Set<String> hiveConfSet = Stream.of(ConfVars.values()).map(confVars -> confVars.varname)
.collect(Collectors.toSet());
for (String conf : otherArray) {
if (hiveConfSet.contains(conf.split("=")[0])) {
hiveConfListSb.append(conf).append(";");
} else {
sessionVarListSb.append(conf).append(";");
}
}
// remove the last ";"
if (sessionVarListSb.length() > 0) {
sessionVarListSb.deleteCharAt(sessionVarListSb.length() - 1);
}
if (hiveConfListSb.length() > 0) {
hiveConfListSb.deleteCharAt(hiveConfListSb.length() - 1);
}
return sessionVarListSb.toString() + hiveConfListSb.toString();
}
}

85
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSourceTest.java

@ -0,0 +1,85 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.datasource;
import org.junit.Assert;
import org.junit.Test;
/**
* test data source of hive
*/
public class HiveDataSourceTest {
@Test
public void testfilterOther() {
BaseDataSource hiveDataSource = new HiveDataSource();
// not contain hive_site_conf
String other = hiveDataSource.filterOther("charset=UTF-8");
Assert.assertEquals("charset=UTF-8", other);
// not contain
other = hiveDataSource.filterOther("");
Assert.assertEquals("", other);
// only contain hive_site_conf
other = hiveDataSource.filterOther("hive.mapred.mode=strict");
Assert.assertEquals("?hive.mapred.mode=strict", other);
// contain hive_site_conf at the first
other = hiveDataSource.filterOther("hive.mapred.mode=strict;charset=UTF-8");
Assert.assertEquals("charset=UTF-8?hive.mapred.mode=strict", other);
// contain hive_site_conf in the middle
other = hiveDataSource.filterOther("charset=UTF-8;hive.mapred.mode=strict;foo=bar");
Assert.assertEquals("charset=UTF-8;foo=bar?hive.mapred.mode=strict", other);
// contain hive_site_conf at the end
other = hiveDataSource.filterOther("charset=UTF-8;foo=bar;hive.mapred.mode=strict");
Assert.assertEquals("charset=UTF-8;foo=bar?hive.mapred.mode=strict", other);
// contain multi hive_site_conf
other = hiveDataSource.filterOther("charset=UTF-8;foo=bar;hive.mapred.mode=strict;hive.exec.parallel=true");
Assert.assertEquals("charset=UTF-8;foo=bar?hive.mapred.mode=strict;hive.exec.parallel=true", other);
}
@Test
public void testGetHiveJdbcUrlOther() {
BaseDataSource hiveDataSource = new HiveDataSource();
hiveDataSource.setAddress("jdbc:hive2://127.0.0.1:10000");
hiveDataSource.setDatabase("test");
hiveDataSource.setPassword("123456");
hiveDataSource.setUser("test");
Assert.assertEquals("jdbc:hive2://127.0.0.1:10000/test", hiveDataSource.getJdbcUrl());
hiveDataSource.setOther("charset=UTF-8;hive.mapred.mode=strict;hive.server2.thrift.http.path=hs2");
Assert.assertEquals(
"jdbc:hive2://127.0.0.1:10000/test;charset=UTF-8?hive.mapred.mode=strict;hive.server2.thrift.http.path=hs2",
hiveDataSource.getJdbcUrl());
hiveDataSource.setOther("hive.mapred.mode=strict;hive.server2.thrift.http.path=hs2");
Assert.assertEquals(
"jdbc:hive2://127.0.0.1:10000/test;?hive.mapred.mode=strict;hive.server2.thrift.http.path=hs2",
hiveDataSource.getJdbcUrl());
}
}

1
pom.xml

@ -852,6 +852,7 @@
<include>**/dao/utils/DagHelperTest.java</include>
<include>**/dao/AlertDaoTest.java</include>
<include>**/dao/datasource/OracleDataSourceTest.java</include>
<include>**/dao/datasource/HiveDataSourceTest.java</include>
<include>**/dao/upgrade/ProcessDefinitionDaoTest.java</include>
<include>**/dao/upgrade/WokrerGrouopDaoTest.java</include>
<include>**/dao/upgrade/UpgradeDaoTest.java</include>

Loading…
Cancel
Save