Browse Source
# Conflicts: # dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java # dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionVersionServiceImpl.java # dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java # dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionControllerTest.java # dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionVersionServiceTest.java # dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java # dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java # dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java # dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.javapull/3/MERGE
baoliang
3 years ago
188 changed files with 5314 additions and 3712 deletions
@ -0,0 +1 @@
|
||||
Subproject commit 966bbbd2113de1f8008322fd1cc7674981a6b1ab |
||||
@ -0,0 +1 @@
|
||||
Subproject commit 978f846c4ca6299fd136f465b42c5e87aca28cac |
||||
@ -0,0 +1 @@
|
||||
Subproject commit 4b5062133773227695d089319f0374ff0868c9fb |
||||
@ -0,0 +1,26 @@
|
||||
# Licensed to Apache Software Foundation (ASF) under one or more contributor |
||||
# license agreements. See the NOTICE file distributed with |
||||
# this work for additional information regarding copyright |
||||
# ownership. Apache Software Foundation (ASF) licenses this file to you under |
||||
# the Apache License, Version 2.0 (the "License"); you may |
||||
# not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
[submodule ".github/actions/comment-on-issue"] |
||||
path = .github/actions/comment-on-issue |
||||
url = https://github.com/xingchun-chen/actions-comment-on-issue |
||||
[submodule ".github/actions/lable-on-issue"] |
||||
path = .github/actions/lable-on-issue |
||||
url = https://github.com/xingchun-chen/labeler |
||||
[submodule ".github/actions/translation-on-issue"] |
||||
path = .github/actions/translation-on-issue |
||||
url = https://github.com/xingchun-chen/translation-helper |
||||
@ -1,5 +0,0 @@
|
||||
Apache DolphinScheduler is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator. |
||||
Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, |
||||
communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. |
||||
While incubation status is not necessarily a reflection of the completeness or stability of the code, |
||||
it does indicate that the project has yet to be fully endorsed by the ASF. |
||||
@ -0,0 +1,78 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.api.configuration; |
||||
|
||||
import java.util.Map; |
||||
|
||||
import org.springframework.beans.factory.annotation.Value; |
||||
import org.springframework.context.annotation.Configuration; |
||||
|
||||
@Configuration |
||||
public class TrafficConfiguration { |
||||
|
||||
@Value("${traffic.control.global.switch:false}") |
||||
private boolean trafficGlobalControlSwitch; |
||||
@Value("${traffic.control.max.global.qps.rate:300}") |
||||
private Integer maxGlobalQpsRate; |
||||
@Value("${traffic.control.tenant.switch:false}") |
||||
private boolean trafficTenantControlSwitch; |
||||
@Value("${traffic.control.default.tenant.qps.rate:10}") |
||||
private Integer defaultTenantQpsRate; |
||||
@Value("#{'${traffic.control.customize.tenant.qps.rate:}'.empty?null:'${traffic.control.customize.tenant.qps.rate:}'}") |
||||
private Map<String, Integer> customizeTenantQpsRate; |
||||
|
||||
public boolean isTrafficGlobalControlSwitch() { |
||||
return trafficGlobalControlSwitch; |
||||
} |
||||
|
||||
public void setTrafficGlobalControlSwitch(boolean trafficGlobalControlSwitch) { |
||||
this.trafficGlobalControlSwitch = trafficGlobalControlSwitch; |
||||
} |
||||
|
||||
public Integer getMaxGlobalQpsRate() { |
||||
return maxGlobalQpsRate; |
||||
} |
||||
|
||||
public void setMaxGlobalQpsRate(Integer maxGlobalQpsRate) { |
||||
this.maxGlobalQpsRate = maxGlobalQpsRate; |
||||
} |
||||
|
||||
public boolean isTrafficTenantControlSwitch() { |
||||
return trafficTenantControlSwitch; |
||||
} |
||||
|
||||
public void setTrafficTenantControlSwitch(boolean trafficTenantControlSwitch) { |
||||
this.trafficTenantControlSwitch = trafficTenantControlSwitch; |
||||
} |
||||
|
||||
public Integer getDefaultTenantQpsRate() { |
||||
return defaultTenantQpsRate; |
||||
} |
||||
|
||||
public void setDefaultTenantQpsRate(Integer defaultTenantQpsRate) { |
||||
this.defaultTenantQpsRate = defaultTenantQpsRate; |
||||
} |
||||
|
||||
public Map<String, Integer> getCustomizeTenantQpsRate() { |
||||
return customizeTenantQpsRate; |
||||
} |
||||
|
||||
public void setCustomizeTenantQpsRate(Map<String, Integer> customizeTenantQpsRate) { |
||||
this.customizeTenantQpsRate = customizeTenantQpsRate; |
||||
} |
||||
} |
||||
@ -0,0 +1,104 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.api.interceptor; |
||||
|
||||
import org.apache.dolphinscheduler.api.configuration.TrafficConfiguration; |
||||
import org.apache.dolphinscheduler.common.utils.StringUtils; |
||||
|
||||
import org.apache.commons.collections.MapUtils; |
||||
|
||||
import java.util.Map; |
||||
import java.util.concurrent.ExecutionException; |
||||
import java.util.concurrent.TimeUnit; |
||||
|
||||
import javax.servlet.http.HttpServletRequest; |
||||
import javax.servlet.http.HttpServletResponse; |
||||
|
||||
import org.slf4j.Logger; |
||||
import org.slf4j.LoggerFactory; |
||||
import org.springframework.http.HttpStatus; |
||||
import org.springframework.web.servlet.HandlerInterceptor; |
||||
|
||||
import com.google.common.cache.CacheBuilder; |
||||
import com.google.common.cache.CacheLoader; |
||||
import com.google.common.cache.LoadingCache; |
||||
import com.google.common.util.concurrent.RateLimiter; |
||||
|
||||
/** |
||||
* This interceptor is used to control the traffic, consists with global traffic control and tenant-leve traffic control. |
||||
* If the current coming tenant reaches his tenant-level request quota, his request will be reject fast. |
||||
* If the current system request number reaches the global request quota, all coming request will be reject fast. |
||||
*/ |
||||
public class RateLimitInterceptor implements HandlerInterceptor { |
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(RateLimitInterceptor.class); |
||||
|
||||
private TrafficConfiguration trafficConfiguration; |
||||
|
||||
private RateLimiter globalRateLimiter; |
||||
|
||||
private LoadingCache<String, RateLimiter> tenantRateLimiterCache = CacheBuilder.newBuilder() |
||||
.maximumSize(100) |
||||
.expireAfterAccess(10, TimeUnit.MINUTES) |
||||
.build(new CacheLoader<String, RateLimiter>() { |
||||
@Override |
||||
public RateLimiter load(String token) { |
||||
// use tenant customize rate limit
|
||||
Map<String, Integer> customizeTenantQpsRate = trafficConfiguration.getCustomizeTenantQpsRate(); |
||||
int tenantQuota = trafficConfiguration.getDefaultTenantQpsRate(); |
||||
if (MapUtils.isNotEmpty(customizeTenantQpsRate)) { |
||||
tenantQuota = customizeTenantQpsRate.getOrDefault(token, trafficConfiguration.getDefaultTenantQpsRate()); |
||||
} |
||||
// use tenant default rate limit
|
||||
return RateLimiter.create(tenantQuota, 1, TimeUnit.SECONDS); |
||||
} |
||||
}); |
||||
|
||||
@Override |
||||
public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) throws ExecutionException { |
||||
// tenant-level rate limit
|
||||
if (trafficConfiguration.isTrafficTenantControlSwitch()) { |
||||
String token = request.getHeader("token"); |
||||
if (StringUtils.isNotEmpty(token)) { |
||||
RateLimiter tenantRateLimiter = tenantRateLimiterCache.get(token); |
||||
if (!tenantRateLimiter.tryAcquire()) { |
||||
response.setStatus(HttpStatus.TOO_MANY_REQUESTS.value()); |
||||
logger.warn("Too many request, reach tenant rate limit, current tenant:{} qps is {}", token, tenantRateLimiter.getRate()); |
||||
return false; |
||||
} |
||||
} |
||||
} |
||||
// global rate limit
|
||||
if (trafficConfiguration.isTrafficGlobalControlSwitch()) { |
||||
if (!globalRateLimiter.tryAcquire()) { |
||||
response.setStatus(HttpStatus.TOO_MANY_REQUESTS.value()); |
||||
logger.warn("Too many request, reach global rate limit, current qps is {}", globalRateLimiter.getRate()); |
||||
return false; |
||||
} |
||||
} |
||||
return true; |
||||
} |
||||
|
||||
public RateLimitInterceptor(TrafficConfiguration trafficConfiguration) { |
||||
this.trafficConfiguration = trafficConfiguration; |
||||
if (trafficConfiguration.isTrafficGlobalControlSwitch()) { |
||||
this.globalRateLimiter = RateLimiter.create(trafficConfiguration.getMaxGlobalQpsRate(), 1, TimeUnit.SECONDS); |
||||
} |
||||
} |
||||
|
||||
} |
||||
@ -0,0 +1,60 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.api.configuration; |
||||
|
||||
import org.apache.commons.collections.MapUtils; |
||||
|
||||
import org.junit.Assert; |
||||
import org.junit.Test; |
||||
import org.junit.runner.RunWith; |
||||
import org.springframework.beans.factory.annotation.Autowired; |
||||
import org.springframework.boot.test.context.SpringBootTest; |
||||
import org.springframework.test.context.junit4.SpringRunner; |
||||
|
||||
@RunWith(SpringRunner.class) |
||||
@SpringBootTest |
||||
public class TrafficConfigurationTest { |
||||
|
||||
@Autowired |
||||
private TrafficConfiguration trafficConfiguration; |
||||
|
||||
@Test |
||||
public void isTrafficGlobalControlSwitch() { |
||||
Assert.assertFalse(trafficConfiguration.isTrafficGlobalControlSwitch()); |
||||
} |
||||
|
||||
@Test |
||||
public void getMaxGlobalQpsLimit() { |
||||
Assert.assertEquals(300, (int) trafficConfiguration.getMaxGlobalQpsRate()); |
||||
} |
||||
|
||||
@Test |
||||
public void isTrafficTenantControlSwitch() { |
||||
Assert.assertFalse(trafficConfiguration.isTrafficTenantControlSwitch()); |
||||
} |
||||
|
||||
@Test |
||||
public void getDefaultTenantQpsLimit() { |
||||
Assert.assertEquals(10, (int) trafficConfiguration.getDefaultTenantQpsRate()); |
||||
} |
||||
|
||||
@Test |
||||
public void getCustomizeTenantQpsRate() { |
||||
Assert.assertTrue(MapUtils.isEmpty(trafficConfiguration.getCustomizeTenantQpsRate())); |
||||
} |
||||
} |
||||
@ -0,0 +1,89 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.api.interceptor; |
||||
|
||||
import org.apache.dolphinscheduler.api.configuration.TrafficConfiguration; |
||||
|
||||
import java.util.HashMap; |
||||
import java.util.Map; |
||||
import java.util.concurrent.ExecutionException; |
||||
|
||||
import javax.servlet.http.HttpServletRequest; |
||||
import javax.servlet.http.HttpServletResponse; |
||||
|
||||
import org.junit.Assert; |
||||
import org.junit.Test; |
||||
import org.junit.runner.RunWith; |
||||
import org.mockito.Mockito; |
||||
import org.powermock.api.mockito.PowerMockito; |
||||
import org.powermock.modules.junit4.PowerMockRunner; |
||||
|
||||
@RunWith(PowerMockRunner.class) |
||||
public class RateLimitInterceptorTest { |
||||
|
||||
@Test |
||||
public void testPreHandleWithoutControl() throws ExecutionException { |
||||
HttpServletRequest request = Mockito.mock(HttpServletRequest.class); |
||||
HttpServletResponse response = Mockito.mock(HttpServletResponse.class); |
||||
RateLimitInterceptor rateLimitInterceptor = new RateLimitInterceptor(new TrafficConfiguration()); |
||||
Assert.assertTrue(rateLimitInterceptor.preHandle(request, response, null)); |
||||
Assert.assertTrue(rateLimitInterceptor.preHandle(request, response, null)); |
||||
} |
||||
|
||||
@Test |
||||
public void testPreHandleWithTenantLevenControl() throws ExecutionException { |
||||
TrafficConfiguration trafficConfiguration = new TrafficConfiguration(); |
||||
trafficConfiguration.setTrafficTenantControlSwitch(true); |
||||
Map<String, Integer> map = new HashMap<>(); |
||||
map.put("tenant1", 2); |
||||
map.put("tenant2", 2); |
||||
trafficConfiguration.setCustomizeTenantQpsRate(map); |
||||
trafficConfiguration.setDefaultTenantQpsRate(4); |
||||
RateLimitInterceptor rateLimitInterceptor = new RateLimitInterceptor(trafficConfiguration); |
||||
|
||||
HttpServletRequest tenant1Request = Mockito.mock(HttpServletRequest.class); |
||||
HttpServletRequest tenant2Request = Mockito.mock(HttpServletRequest.class); |
||||
PowerMockito.when(tenant1Request.getHeader(Mockito.any())).thenReturn("tenant1"); |
||||
PowerMockito.when(tenant2Request.getHeader(Mockito.any())).thenReturn("tenant2"); |
||||
HttpServletResponse response = Mockito.mock(HttpServletResponse.class); |
||||
|
||||
for (int i = 0; i < 2; i++) { |
||||
rateLimitInterceptor.preHandle(tenant1Request, response, null); |
||||
} |
||||
Assert.assertFalse(rateLimitInterceptor.preHandle(tenant1Request, response, null)); |
||||
Assert.assertTrue(rateLimitInterceptor.preHandle(tenant2Request, response, null)); |
||||
} |
||||
|
||||
@Test |
||||
public void testPreHandleWithGlobalControl() throws ExecutionException { |
||||
TrafficConfiguration trafficConfiguration = new TrafficConfiguration(); |
||||
trafficConfiguration.setTrafficTenantControlSwitch(true); |
||||
trafficConfiguration.setTrafficGlobalControlSwitch(true); |
||||
trafficConfiguration.setMaxGlobalQpsRate(3); |
||||
|
||||
RateLimitInterceptor rateLimitInterceptor = new RateLimitInterceptor(trafficConfiguration); |
||||
HttpServletRequest request = Mockito.mock(HttpServletRequest.class); |
||||
HttpServletResponse response = Mockito.mock(HttpServletResponse.class); |
||||
|
||||
for (int i = 0; i < 2; i++) { |
||||
rateLimitInterceptor.preHandle(request, response, null); |
||||
} |
||||
Assert.assertFalse(rateLimitInterceptor.preHandle(request, response, null)); |
||||
} |
||||
|
||||
} |
||||
@ -0,0 +1,79 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.datasource; |
||||
|
||||
import org.apache.commons.collections4.MapUtils; |
||||
|
||||
import java.util.Map; |
||||
import java.util.regex.Pattern; |
||||
|
||||
public abstract class AbstractDatasourceProcessor implements DatasourceProcessor { |
||||
|
||||
private static final Pattern IPV4_PATTERN = Pattern.compile("^[a-zA-Z0-9\\_\\-\\.]+$"); |
||||
|
||||
private static final Pattern IPV6_PATTERN = Pattern.compile("^[a-zA-Z0-9\\_\\-\\.\\:\\[\\]]+$"); |
||||
|
||||
private static final Pattern DATABASE_PATTER = Pattern.compile("^[a-zA-Z0-9\\_\\-\\.]+$"); |
||||
|
||||
private static final Pattern PARAMS_PATTER = Pattern.compile("^[a-zA-Z0-9]+$"); |
||||
|
||||
@Override |
||||
public void checkDatasourceParam(BaseDataSourceParamDTO baseDataSourceParamDTO) { |
||||
checkHost(baseDataSourceParamDTO.getHost()); |
||||
checkDatasourcePatter(baseDataSourceParamDTO.getDatabase()); |
||||
checkOther(baseDataSourceParamDTO.getOther()); |
||||
} |
||||
|
||||
/** |
||||
* Check the host is valid |
||||
* |
||||
* @param host datasource host |
||||
*/ |
||||
protected void checkHost(String host) { |
||||
if (!IPV4_PATTERN.matcher(host).matches() || !IPV6_PATTERN.matcher(host).matches()) { |
||||
throw new IllegalArgumentException("datasource host illegal"); |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* check database name is valid |
||||
* |
||||
* @param database database name |
||||
*/ |
||||
protected void checkDatasourcePatter(String database) { |
||||
if (!DATABASE_PATTER.matcher(database).matches()) { |
||||
throw new IllegalArgumentException("datasource name illegal"); |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* check other is valid |
||||
* |
||||
* @param other other |
||||
*/ |
||||
protected void checkOther(Map<String, String> other) { |
||||
if (MapUtils.isEmpty(other)) { |
||||
return; |
||||
} |
||||
boolean paramsCheck = other.entrySet().stream().allMatch(p -> PARAMS_PATTER.matcher(p.getValue()).matches()); |
||||
if (!paramsCheck) { |
||||
throw new IllegalArgumentException("datasource other params illegal"); |
||||
} |
||||
} |
||||
|
||||
} |
||||
@ -0,0 +1,98 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.datasource; |
||||
|
||||
import com.fasterxml.jackson.annotation.JsonInclude; |
||||
import com.fasterxml.jackson.annotation.JsonInclude.Include; |
||||
|
||||
/** |
||||
* The base model of connection param |
||||
* <p> |
||||
* {@link org.apache.dolphinscheduler.common.datasource.clickhouse.ClickhouseConnectionParam} |
||||
* {@link org.apache.dolphinscheduler.common.datasource.db2.Db2ConnectionParam} |
||||
* {@link org.apache.dolphinscheduler.common.datasource.hive.HiveConnectionParam} |
||||
* {@link org.apache.dolphinscheduler.common.datasource.mysql.MysqlConnectionParam} |
||||
* {@link org.apache.dolphinscheduler.common.datasource.oracle.OracleConnectionParam} |
||||
* {@link org.apache.dolphinscheduler.common.datasource.postgresql.PostgreSqlConnectionParam} |
||||
* {@link org.apache.dolphinscheduler.common.datasource.presto.PrestoConnectionParam} |
||||
* {@link org.apache.dolphinscheduler.common.datasource.spark.SparkConnectionParam} |
||||
* {@link org.apache.dolphinscheduler.common.datasource.sqlserver.SqlServerConnectionParam} |
||||
*/ |
||||
@JsonInclude(Include.NON_NULL) |
||||
public abstract class BaseConnectionParam implements ConnectionParam { |
||||
|
||||
protected String user; |
||||
|
||||
protected String password; |
||||
|
||||
protected String address; |
||||
|
||||
protected String database; |
||||
|
||||
protected String jdbcUrl; |
||||
|
||||
protected String other; |
||||
|
||||
public String getUser() { |
||||
return user; |
||||
} |
||||
|
||||
public void setUser(String user) { |
||||
this.user = user; |
||||
} |
||||
|
||||
public String getPassword() { |
||||
return password; |
||||
} |
||||
|
||||
public void setPassword(String password) { |
||||
this.password = password; |
||||
} |
||||
|
||||
public String getAddress() { |
||||
return address; |
||||
} |
||||
|
||||
public void setAddress(String address) { |
||||
this.address = address; |
||||
} |
||||
|
||||
public String getDatabase() { |
||||
return database; |
||||
} |
||||
|
||||
public void setDatabase(String database) { |
||||
this.database = database; |
||||
} |
||||
|
||||
public String getJdbcUrl() { |
||||
return jdbcUrl; |
||||
} |
||||
|
||||
public void setJdbcUrl(String jdbcUrl) { |
||||
this.jdbcUrl = jdbcUrl; |
||||
} |
||||
|
||||
public String getOther() { |
||||
return other; |
||||
} |
||||
|
||||
public void setOther(String other) { |
||||
this.other = other; |
||||
} |
||||
} |
||||
@ -0,0 +1,161 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.datasource; |
||||
|
||||
import org.apache.dolphinscheduler.common.datasource.clickhouse.ClickHouseDatasourceParamDTO; |
||||
import org.apache.dolphinscheduler.common.datasource.db2.Db2DatasourceParamDTO; |
||||
import org.apache.dolphinscheduler.common.datasource.hive.HiveDataSourceParamDTO; |
||||
import org.apache.dolphinscheduler.common.datasource.mysql.MysqlDatasourceParamDTO; |
||||
import org.apache.dolphinscheduler.common.datasource.oracle.OracleDatasourceParamDTO; |
||||
import org.apache.dolphinscheduler.common.datasource.postgresql.PostgreSqlDatasourceParamDTO; |
||||
import org.apache.dolphinscheduler.common.datasource.presto.PrestoDatasourceParamDTO; |
||||
import org.apache.dolphinscheduler.common.datasource.spark.SparkDatasourceParamDTO; |
||||
import org.apache.dolphinscheduler.common.datasource.sqlserver.SqlServerDatasourceParamDTO; |
||||
import org.apache.dolphinscheduler.common.enums.DbType; |
||||
|
||||
import java.io.Serializable; |
||||
import java.util.Map; |
||||
|
||||
import com.fasterxml.jackson.annotation.JsonSubTypes; |
||||
import com.fasterxml.jackson.annotation.JsonTypeInfo; |
||||
|
||||
/** |
||||
* Basic datasource params submitted to api. |
||||
* <p> |
||||
* see {@link MysqlDatasourceParamDTO} |
||||
* see {@link PostgreSqlDatasourceParamDTO} |
||||
* see {@link HiveDataSourceParamDTO} |
||||
* see {@link SparkDatasourceParamDTO} |
||||
* see {@link ClickHouseDatasourceParamDTO} |
||||
* see {@link OracleDatasourceParamDTO} |
||||
* see {@link SqlServerDatasourceParamDTO} |
||||
* see {@link Db2DatasourceParamDTO} |
||||
* see {@link PrestoDatasourceParamDTO} |
||||
*/ |
||||
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") |
||||
@JsonSubTypes(value = { |
||||
@JsonSubTypes.Type(value = MysqlDatasourceParamDTO.class, name = "MYSQL"), |
||||
@JsonSubTypes.Type(value = PostgreSqlDatasourceParamDTO.class, name = "POSTGRESQL"), |
||||
@JsonSubTypes.Type(value = HiveDataSourceParamDTO.class, name = "HIVE"), |
||||
@JsonSubTypes.Type(value = SparkDatasourceParamDTO.class, name = "SPARK"), |
||||
@JsonSubTypes.Type(value = ClickHouseDatasourceParamDTO.class, name = "CLICKHOUSE"), |
||||
@JsonSubTypes.Type(value = OracleDatasourceParamDTO.class, name = "ORACLE"), |
||||
@JsonSubTypes.Type(value = SqlServerDatasourceParamDTO.class, name = "SQLSERVER"), |
||||
@JsonSubTypes.Type(value = Db2DatasourceParamDTO.class, name = "DB2"), |
||||
@JsonSubTypes.Type(value = PrestoDatasourceParamDTO.class, name = "PRESTO"), |
||||
}) |
||||
public abstract class BaseDataSourceParamDTO implements Serializable { |
||||
|
||||
protected Integer id; |
||||
|
||||
protected String name; |
||||
|
||||
protected String note; |
||||
|
||||
protected String host; |
||||
|
||||
protected Integer port; |
||||
|
||||
protected String database; |
||||
|
||||
protected String userName; |
||||
|
||||
protected String password; |
||||
|
||||
protected Map<String, String> other; |
||||
|
||||
public Integer getId() { |
||||
return id; |
||||
} |
||||
|
||||
public void setId(Integer id) { |
||||
this.id = id; |
||||
} |
||||
|
||||
public String getName() { |
||||
return name; |
||||
} |
||||
|
||||
public void setName(String name) { |
||||
this.name = name; |
||||
} |
||||
|
||||
public String getNote() { |
||||
return note; |
||||
} |
||||
|
||||
public void setNote(String note) { |
||||
this.note = note; |
||||
} |
||||
|
||||
public String getHost() { |
||||
return host; |
||||
} |
||||
|
||||
public void setHost(String host) { |
||||
this.host = host; |
||||
} |
||||
|
||||
public Integer getPort() { |
||||
return port; |
||||
} |
||||
|
||||
public void setPort(Integer port) { |
||||
this.port = port; |
||||
} |
||||
|
||||
public String getDatabase() { |
||||
return database; |
||||
} |
||||
|
||||
public void setDatabase(String database) { |
||||
this.database = database; |
||||
} |
||||
|
||||
public String getUserName() { |
||||
return userName; |
||||
} |
||||
|
||||
public void setUserName(String userName) { |
||||
this.userName = userName; |
||||
} |
||||
|
||||
public String getPassword() { |
||||
return password; |
||||
} |
||||
|
||||
public void setPassword(String password) { |
||||
this.password = password; |
||||
} |
||||
|
||||
public Map<String, String> getOther() { |
||||
return other; |
||||
} |
||||
|
||||
public void setOther(Map<String, String> other) { |
||||
this.other = other; |
||||
} |
||||
|
||||
/** |
||||
* Get the datasource type |
||||
* see{@link DbType} |
||||
* |
||||
* @return datasource type code |
||||
*/ |
||||
public abstract DbType getType(); |
||||
} |
||||
@ -0,0 +1,57 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.datasource; |
||||
|
||||
public class BaseHdfsConnectionParam extends BaseConnectionParam { |
||||
protected String principal; |
||||
protected String javaSecurityKrb5Conf; |
||||
protected String loginUserKeytabUsername; |
||||
protected String loginUserKeytabPath; |
||||
|
||||
public String getPrincipal() { |
||||
return principal; |
||||
} |
||||
|
||||
public void setPrincipal(String principal) { |
||||
this.principal = principal; |
||||
} |
||||
|
||||
public String getJavaSecurityKrb5Conf() { |
||||
return javaSecurityKrb5Conf; |
||||
} |
||||
|
||||
public void setJavaSecurityKrb5Conf(String javaSecurityKrb5Conf) { |
||||
this.javaSecurityKrb5Conf = javaSecurityKrb5Conf; |
||||
} |
||||
|
||||
public String getLoginUserKeytabUsername() { |
||||
return loginUserKeytabUsername; |
||||
} |
||||
|
||||
public void setLoginUserKeytabUsername(String loginUserKeytabUsername) { |
||||
this.loginUserKeytabUsername = loginUserKeytabUsername; |
||||
} |
||||
|
||||
public String getLoginUserKeytabPath() { |
||||
return loginUserKeytabPath; |
||||
} |
||||
|
||||
public void setLoginUserKeytabPath(String loginUserKeytabPath) { |
||||
this.loginUserKeytabPath = loginUserKeytabPath; |
||||
} |
||||
} |
||||
@ -0,0 +1,61 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.datasource; |
||||
|
||||
public abstract class BaseHdfsDatasourceParamDTO extends BaseDataSourceParamDTO { |
||||
|
||||
protected String principal; |
||||
|
||||
protected String javaSecurityKrb5Conf; |
||||
|
||||
protected String loginUserKeytabUsername; |
||||
|
||||
protected String loginUserKeytabPath; |
||||
|
||||
public String getPrincipal() { |
||||
return principal; |
||||
} |
||||
|
||||
public void setPrincipal(String principal) { |
||||
this.principal = principal; |
||||
} |
||||
|
||||
public String getLoginUserKeytabUsername() { |
||||
return loginUserKeytabUsername; |
||||
} |
||||
|
||||
public void setLoginUserKeytabUsername(String loginUserKeytabUsername) { |
||||
this.loginUserKeytabUsername = loginUserKeytabUsername; |
||||
} |
||||
|
||||
public String getLoginUserKeytabPath() { |
||||
return loginUserKeytabPath; |
||||
} |
||||
|
||||
public void setLoginUserKeytabPath(String loginUserKeytabPath) { |
||||
this.loginUserKeytabPath = loginUserKeytabPath; |
||||
} |
||||
|
||||
public String getJavaSecurityKrb5Conf() { |
||||
return javaSecurityKrb5Conf; |
||||
} |
||||
|
||||
public void setJavaSecurityKrb5Conf(String javaSecurityKrb5Conf) { |
||||
this.javaSecurityKrb5Conf = javaSecurityKrb5Conf; |
||||
} |
||||
} |
||||
@ -0,0 +1,81 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.datasource; |
||||
|
||||
import org.apache.dolphinscheduler.common.enums.DbType; |
||||
|
||||
import java.io.IOException; |
||||
import java.sql.Connection; |
||||
import java.sql.SQLException; |
||||
|
||||
public interface DatasourceProcessor { |
||||
|
||||
/** |
||||
* check datasource param is valid |
||||
*/ |
||||
void checkDatasourceParam(BaseDataSourceParamDTO datasourceParam); |
||||
|
||||
/** |
||||
* create BaseDataSourceParamDTO by connectionJson |
||||
* |
||||
* @param connectionJson see{@link org.apache.dolphinscheduler.dao.entity.Datasource} |
||||
* @return {@link BaseDataSourceParamDTO} |
||||
*/ |
||||
BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson); |
||||
|
||||
/** |
||||
* create datasource connection parameter which will be stored at DataSource |
||||
* <p> |
||||
* see {@code org.apache.dolphinscheduler.dao.entity.DataSource.connectionParams} |
||||
*/ |
||||
ConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam); |
||||
|
||||
/** |
||||
* deserialize json to datasource connection param |
||||
* |
||||
* @param connectionJson {@code org.apache.dolphinscheduler.dao.entity.DataSource.connectionParams} |
||||
* @return {@link BaseConnectionParam} |
||||
*/ |
||||
ConnectionParam createConnectionParams(String connectionJson); |
||||
|
||||
/** |
||||
* get datasource Driver |
||||
*/ |
||||
String getDatasourceDriver(); |
||||
|
||||
/** |
||||
* get jdbcUrl by connection param, the jdbcUrl is different with ConnectionParam.jdbcUrl, this method will inject |
||||
* other to jdbcUrl |
||||
* |
||||
* @param connectionParam connection param |
||||
*/ |
||||
String getJdbcUrl(ConnectionParam connectionParam); |
||||
|
||||
/** |
||||
* get connection by connectionParam |
||||
* |
||||
* @param connectionParam connectionParam |
||||
* @return {@link Connection} |
||||
*/ |
||||
Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException, IOException; |
||||
|
||||
/** |
||||
* @return {@link DbType} |
||||
*/ |
||||
DbType getDbType(); |
||||
} |
||||
@ -0,0 +1,121 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.datasource; |
||||
|
||||
import org.apache.dolphinscheduler.common.datasource.clickhouse.ClickHouseDatasourceProcessor; |
||||
import org.apache.dolphinscheduler.common.datasource.db2.Db2DatasourceProcessor; |
||||
import org.apache.dolphinscheduler.common.datasource.hive.HiveDatasourceProcessor; |
||||
import org.apache.dolphinscheduler.common.datasource.mysql.MysqlDatasourceProcessor; |
||||
import org.apache.dolphinscheduler.common.datasource.oracle.OracleDatasourceProcessor; |
||||
import org.apache.dolphinscheduler.common.datasource.postgresql.PostgreSqlDatasourceProcessor; |
||||
import org.apache.dolphinscheduler.common.datasource.presto.PrestoDatasourceProcessor; |
||||
import org.apache.dolphinscheduler.common.datasource.spark.SparkDatasourceProcessor; |
||||
import org.apache.dolphinscheduler.common.datasource.sqlserver.SqlServerDatasourceProcessor; |
||||
import org.apache.dolphinscheduler.common.enums.DbType; |
||||
|
||||
import java.sql.Connection; |
||||
|
||||
import org.slf4j.Logger; |
||||
import org.slf4j.LoggerFactory; |
||||
|
||||
public class DatasourceUtil { |
||||
|
||||
private DatasourceUtil() { |
||||
} |
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(DatasourceUtil.class); |
||||
|
||||
private static final DatasourceProcessor mysqlProcessor = new MysqlDatasourceProcessor(); |
||||
private static final DatasourceProcessor postgreSqlProcessor = new PostgreSqlDatasourceProcessor(); |
||||
private static final DatasourceProcessor hiveProcessor = new HiveDatasourceProcessor(); |
||||
private static final DatasourceProcessor sparkProcessor = new SparkDatasourceProcessor(); |
||||
private static final DatasourceProcessor clickhouseProcessor = new ClickHouseDatasourceProcessor(); |
||||
private static final DatasourceProcessor oracleProcessor = new OracleDatasourceProcessor(); |
||||
private static final DatasourceProcessor sqlServerProcessor = new SqlServerDatasourceProcessor(); |
||||
private static final DatasourceProcessor db2PROCESSOR = new Db2DatasourceProcessor(); |
||||
private static final DatasourceProcessor prestoPROCESSOR = new PrestoDatasourceProcessor(); |
||||
|
||||
/** |
||||
* check datasource param |
||||
* |
||||
* @param baseDataSourceParamDTO datasource param |
||||
*/ |
||||
public static void checkDatasourceParam(BaseDataSourceParamDTO baseDataSourceParamDTO) { |
||||
getDatasourceProcessor(baseDataSourceParamDTO.getType()).checkDatasourceParam(baseDataSourceParamDTO); |
||||
} |
||||
|
||||
/** |
||||
* build connection url |
||||
* |
||||
* @param baseDataSourceParamDTO datasourceParam |
||||
*/ |
||||
public static ConnectionParam buildConnectionParams(BaseDataSourceParamDTO baseDataSourceParamDTO) { |
||||
ConnectionParam connectionParams = getDatasourceProcessor(baseDataSourceParamDTO.getType()) |
||||
.createConnectionParams(baseDataSourceParamDTO); |
||||
if (logger.isDebugEnabled()) { |
||||
logger.info("parameters map:{}", connectionParams); |
||||
} |
||||
return connectionParams; |
||||
} |
||||
|
||||
public static ConnectionParam buildConnectionParams(DbType dbType, String connectionJson) { |
||||
return getDatasourceProcessor(dbType).createConnectionParams(connectionJson); |
||||
} |
||||
|
||||
public static Connection getConnection(DbType dbType, ConnectionParam connectionParam) { |
||||
try { |
||||
return getDatasourceProcessor(dbType).getConnection(connectionParam); |
||||
} catch (Exception e) { |
||||
throw new RuntimeException(e); |
||||
} |
||||
} |
||||
|
||||
public static String getJdbcUrl(DbType dbType, ConnectionParam baseConnectionParam) { |
||||
return getDatasourceProcessor(dbType).getJdbcUrl(baseConnectionParam); |
||||
} |
||||
|
||||
public static BaseDataSourceParamDTO buildDatasourceParamDTO(DbType dbType, String connectionParams) { |
||||
return getDatasourceProcessor(dbType).createDatasourceParamDTO(connectionParams); |
||||
} |
||||
|
||||
public static DatasourceProcessor getDatasourceProcessor(DbType dbType) { |
||||
switch (dbType) { |
||||
case MYSQL: |
||||
return mysqlProcessor; |
||||
case POSTGRESQL: |
||||
return postgreSqlProcessor; |
||||
case HIVE: |
||||
return hiveProcessor; |
||||
case SPARK: |
||||
return sparkProcessor; |
||||
case CLICKHOUSE: |
||||
return clickhouseProcessor; |
||||
case ORACLE: |
||||
return oracleProcessor; |
||||
case SQLSERVER: |
||||
return sqlServerProcessor; |
||||
case DB2: |
||||
return db2PROCESSOR; |
||||
case PRESTO: |
||||
return prestoPROCESSOR; |
||||
default: |
||||
throw new IllegalArgumentException("datasource type illegal:" + dbType); |
||||
} |
||||
} |
||||
|
||||
} |
||||
@ -0,0 +1,125 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.datasource.clickhouse; |
||||
|
||||
import org.apache.dolphinscheduler.common.Constants; |
||||
import org.apache.dolphinscheduler.common.datasource.AbstractDatasourceProcessor; |
||||
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; |
||||
import org.apache.dolphinscheduler.common.datasource.ConnectionParam; |
||||
import org.apache.dolphinscheduler.common.enums.DbType; |
||||
import org.apache.dolphinscheduler.common.utils.CommonUtils; |
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils; |
||||
import org.apache.dolphinscheduler.common.utils.StringUtils; |
||||
|
||||
import org.apache.commons.collections4.MapUtils; |
||||
|
||||
import java.sql.Connection; |
||||
import java.sql.DriverManager; |
||||
import java.sql.SQLException; |
||||
import java.util.LinkedHashMap; |
||||
import java.util.Map; |
||||
|
||||
public class ClickHouseDatasourceProcessor extends AbstractDatasourceProcessor { |
||||
|
||||
@Override |
||||
public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { |
||||
ClickhouseConnectionParam connectionParams = (ClickhouseConnectionParam) createConnectionParams(connectionJson); |
||||
|
||||
ClickHouseDatasourceParamDTO clickHouseDatasourceParamDTO = new ClickHouseDatasourceParamDTO(); |
||||
clickHouseDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); |
||||
clickHouseDatasourceParamDTO.setUserName(connectionParams.getUser()); |
||||
clickHouseDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); |
||||
|
||||
String[] hostSeperator = connectionParams.getAddress().split(Constants.DOUBLE_SLASH); |
||||
String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(Constants.COMMA); |
||||
clickHouseDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(Constants.COLON)[1])); |
||||
clickHouseDatasourceParamDTO.setHost(hostPortArray[0].split(Constants.COLON)[0]); |
||||
|
||||
return clickHouseDatasourceParamDTO; |
||||
} |
||||
|
||||
@Override |
||||
public ConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { |
||||
ClickHouseDatasourceParamDTO clickHouseParam = (ClickHouseDatasourceParamDTO) datasourceParam; |
||||
String address = String.format("%s%s:%s", Constants.JDBC_CLICKHOUSE, clickHouseParam.getHost(), clickHouseParam.getPort()); |
||||
String jdbcUrl = address + "/" + clickHouseParam.getDatabase(); |
||||
|
||||
ClickhouseConnectionParam clickhouseConnectionParam = new ClickhouseConnectionParam(); |
||||
clickhouseConnectionParam.setDatabase(clickHouseParam.getDatabase()); |
||||
clickhouseConnectionParam.setAddress(address); |
||||
clickhouseConnectionParam.setJdbcUrl(jdbcUrl); |
||||
clickhouseConnectionParam.setUser(clickHouseParam.getUserName()); |
||||
clickhouseConnectionParam.setPassword(CommonUtils.encodePassword(clickHouseParam.getPassword())); |
||||
clickhouseConnectionParam.setOther(transformOther(clickHouseParam.getOther())); |
||||
return clickhouseConnectionParam; |
||||
} |
||||
|
||||
@Override |
||||
public ConnectionParam createConnectionParams(String connectionJson) { |
||||
return JSONUtils.parseObject(connectionJson, ClickhouseConnectionParam.class); |
||||
} |
||||
|
||||
@Override |
||||
public String getDatasourceDriver() { |
||||
return Constants.COM_CLICKHOUSE_JDBC_DRIVER; |
||||
} |
||||
|
||||
@Override |
||||
public String getJdbcUrl(ConnectionParam connectionParam) { |
||||
ClickhouseConnectionParam clickhouseConnectionParam = (ClickhouseConnectionParam) connectionParam; |
||||
String jdbcUrl = clickhouseConnectionParam.getJdbcUrl(); |
||||
if (StringUtils.isNotEmpty(clickhouseConnectionParam.getOther())) { |
||||
jdbcUrl = String.format("%s?%s", jdbcUrl, clickhouseConnectionParam.getOther()); |
||||
} |
||||
return jdbcUrl; |
||||
} |
||||
|
||||
@Override |
||||
public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException { |
||||
ClickhouseConnectionParam clickhouseConnectionParam = (ClickhouseConnectionParam) connectionParam; |
||||
Class.forName(getDatasourceDriver()); |
||||
return DriverManager.getConnection(getJdbcUrl(clickhouseConnectionParam), |
||||
clickhouseConnectionParam.getUser(), CommonUtils.decodePassword(clickhouseConnectionParam.getPassword())); |
||||
} |
||||
|
||||
@Override |
||||
public DbType getDbType() { |
||||
return DbType.CLICKHOUSE; |
||||
} |
||||
|
||||
private String transformOther(Map<String, String> otherMap) { |
||||
if (MapUtils.isEmpty(otherMap)) { |
||||
return null; |
||||
} |
||||
StringBuilder stringBuilder = new StringBuilder(); |
||||
otherMap.forEach((key, value) -> stringBuilder.append(String.format("%s=%s%s", key, value, "&"))); |
||||
return stringBuilder.toString(); |
||||
} |
||||
|
||||
private Map<String, String> parseOther(String other) { |
||||
if (other == null) { |
||||
return null; |
||||
} |
||||
Map<String, String> otherMap = new LinkedHashMap<>(); |
||||
String[] configs = other.split("&"); |
||||
for (String config : configs) { |
||||
otherMap.put(config.split("=")[0], config.split("=")[1]); |
||||
} |
||||
return otherMap; |
||||
} |
||||
} |
||||
@ -0,0 +1,34 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.datasource.clickhouse; |
||||
|
||||
import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam; |
||||
|
||||
public class ClickhouseConnectionParam extends BaseConnectionParam { |
||||
@Override |
||||
public String toString() { |
||||
return "ClickhouseConnectionParam{" |
||||
+ "user='" + user + '\'' |
||||
+ ", password='" + password + '\'' |
||||
+ ", address='" + address + '\'' |
||||
+ ", database='" + database + '\'' |
||||
+ ", jdbcUrl='" + jdbcUrl + '\'' |
||||
+ ", other='" + other + '\'' |
||||
+ '}'; |
||||
} |
||||
} |
||||
@ -0,0 +1,43 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.datasource.db2; |
||||
|
||||
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; |
||||
import org.apache.dolphinscheduler.common.enums.DbType; |
||||
|
||||
public class Db2DatasourceParamDTO extends BaseDataSourceParamDTO { |
||||
|
||||
@Override |
||||
public String toString() { |
||||
return "Db2DatasourceParamDTO{" |
||||
+ "name='" + name + '\'' |
||||
+ ", note='" + note + '\'' |
||||
+ ", host='" + host + '\'' |
||||
+ ", port=" + port |
||||
+ ", database='" + database + '\'' |
||||
+ ", userName='" + userName + '\'' |
||||
+ ", password='" + password + '\'' |
||||
+ ", other='" + other + '\'' |
||||
+ '}'; |
||||
} |
||||
|
||||
@Override |
||||
public DbType getType() { |
||||
return DbType.DB2; |
||||
} |
||||
} |
||||
@ -0,0 +1,126 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.datasource.db2; |
||||
|
||||
import org.apache.dolphinscheduler.common.Constants; |
||||
import org.apache.dolphinscheduler.common.datasource.AbstractDatasourceProcessor; |
||||
import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam; |
||||
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; |
||||
import org.apache.dolphinscheduler.common.datasource.ConnectionParam; |
||||
import org.apache.dolphinscheduler.common.enums.DbType; |
||||
import org.apache.dolphinscheduler.common.utils.CommonUtils; |
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils; |
||||
import org.apache.dolphinscheduler.common.utils.StringUtils; |
||||
|
||||
import org.apache.commons.collections4.MapUtils; |
||||
|
||||
import java.sql.Connection; |
||||
import java.sql.DriverManager; |
||||
import java.sql.SQLException; |
||||
import java.util.LinkedHashMap; |
||||
import java.util.Map; |
||||
|
||||
public class Db2DatasourceProcessor extends AbstractDatasourceProcessor { |
||||
|
||||
@Override |
||||
public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { |
||||
Db2ConnectionParam connectionParams = (Db2ConnectionParam) createConnectionParams(connectionJson); |
||||
|
||||
Db2DatasourceParamDTO db2DatasourceParamDTO = new Db2DatasourceParamDTO(); |
||||
db2DatasourceParamDTO.setDatabase(connectionParams.getDatabase()); |
||||
db2DatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); |
||||
db2DatasourceParamDTO.setUserName(db2DatasourceParamDTO.getUserName()); |
||||
|
||||
String[] hostSeperator = connectionParams.getAddress().split(Constants.DOUBLE_SLASH); |
||||
String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(Constants.COMMA); |
||||
db2DatasourceParamDTO.setHost(hostPortArray[0].split(Constants.COLON)[0]); |
||||
db2DatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(Constants.COLON)[1])); |
||||
|
||||
return db2DatasourceParamDTO; |
||||
} |
||||
|
||||
@Override |
||||
public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { |
||||
Db2DatasourceParamDTO db2Param = (Db2DatasourceParamDTO) datasourceParam; |
||||
String address = String.format("%s%s:%s", Constants.JDBC_DB2, db2Param.getHost(), db2Param.getPort()); |
||||
String jdbcUrl = String.format("%s/%s", address, db2Param.getDatabase()); |
||||
|
||||
Db2ConnectionParam db2ConnectionParam = new Db2ConnectionParam(); |
||||
db2ConnectionParam.setAddress(address); |
||||
db2ConnectionParam.setDatabase(db2Param.getDatabase()); |
||||
db2ConnectionParam.setJdbcUrl(jdbcUrl); |
||||
db2ConnectionParam.setUser(db2Param.getUserName()); |
||||
db2ConnectionParam.setPassword(CommonUtils.encodePassword(db2Param.getPassword())); |
||||
db2ConnectionParam.setOther(transformOther(db2Param.getOther())); |
||||
|
||||
return db2ConnectionParam; |
||||
} |
||||
|
||||
@Override |
||||
public ConnectionParam createConnectionParams(String connectionJson) { |
||||
return JSONUtils.parseObject(connectionJson, Db2ConnectionParam.class); |
||||
} |
||||
|
||||
@Override |
||||
public String getDatasourceDriver() { |
||||
return Constants.COM_DB2_JDBC_DRIVER; |
||||
} |
||||
|
||||
@Override |
||||
public String getJdbcUrl(ConnectionParam connectionParam) { |
||||
Db2ConnectionParam db2ConnectionParam = (Db2ConnectionParam) connectionParam; |
||||
if (StringUtils.isNotEmpty(db2ConnectionParam.getOther())) { |
||||
return String.format("%s;%s", db2ConnectionParam.getJdbcUrl(), db2ConnectionParam.getOther()); |
||||
} |
||||
return db2ConnectionParam.getJdbcUrl(); |
||||
} |
||||
|
||||
@Override |
||||
public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException { |
||||
Db2ConnectionParam db2ConnectionParam = (Db2ConnectionParam) connectionParam; |
||||
Class.forName(getDatasourceDriver()); |
||||
return DriverManager.getConnection(getJdbcUrl(db2ConnectionParam), |
||||
db2ConnectionParam.getUser(), CommonUtils.decodePassword(db2ConnectionParam.getPassword())); |
||||
} |
||||
|
||||
@Override |
||||
public DbType getDbType() { |
||||
return DbType.DB2; |
||||
} |
||||
|
||||
private String transformOther(Map<String, String> otherMap) { |
||||
if (MapUtils.isEmpty(otherMap)) { |
||||
return null; |
||||
} |
||||
StringBuilder stringBuilder = new StringBuilder(); |
||||
otherMap.forEach((key, value) -> stringBuilder.append(String.format("%s=%s%s", key, value, ";"))); |
||||
stringBuilder.deleteCharAt(stringBuilder.length() - 1); |
||||
return stringBuilder.toString(); |
||||
} |
||||
|
||||
private Map<String, String> parseOther(String other) { |
||||
if (other == null) { |
||||
return null; |
||||
} |
||||
Map<String, String> otherMap = new LinkedHashMap<>(); |
||||
for (String config : other.split("&")) { |
||||
otherMap.put(config.split("=")[0], config.split("=")[1]); |
||||
} |
||||
return otherMap; |
||||
} |
||||
} |
||||
@ -0,0 +1,38 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.datasource.hive; |
||||
|
||||
import org.apache.dolphinscheduler.common.datasource.BaseHdfsConnectionParam; |
||||
|
||||
public class HiveConnectionParam extends BaseHdfsConnectionParam { |
||||
@Override |
||||
public String toString() { |
||||
return "HiveConnectionParam{" |
||||
+ "user='" + user + '\'' |
||||
+ ", password='" + password + '\'' |
||||
+ ", address='" + address + '\'' |
||||
+ ", database='" + database + '\'' |
||||
+ ", jdbcUrl='" + jdbcUrl + '\'' |
||||
+ ", other='" + other + '\'' |
||||
+ ", principal='" + principal + '\'' |
||||
+ ", javaSecurityKrb5Conf='" + javaSecurityKrb5Conf + '\'' |
||||
+ ", loginUserKeytabUsername='" + loginUserKeytabUsername + '\'' |
||||
+ ", loginUserKeytabPath='" + loginUserKeytabPath + '\'' |
||||
+ '}'; |
||||
} |
||||
} |
||||
@ -0,0 +1,45 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.datasource.hive; |
||||
|
||||
import org.apache.dolphinscheduler.common.datasource.BaseHdfsDatasourceParamDTO; |
||||
import org.apache.dolphinscheduler.common.enums.DbType; |
||||
|
||||
public class HiveDataSourceParamDTO extends BaseHdfsDatasourceParamDTO { |
||||
|
||||
@Override |
||||
public String toString() { |
||||
return "HiveDataSourceParamDTO{" |
||||
+ "host='" + host + '\'' |
||||
+ ", port=" + port |
||||
+ ", database='" + database + '\'' |
||||
+ ", principal='" + principal + '\'' |
||||
+ ", userName='" + userName + '\'' |
||||
+ ", password='" + password + '\'' |
||||
+ ", other='" + other + '\'' |
||||
+ ", javaSecurityKrb5Conf='" + javaSecurityKrb5Conf + '\'' |
||||
+ ", loginUserKeytabUsername='" + loginUserKeytabUsername + '\'' |
||||
+ ", loginUserKeytabPath='" + loginUserKeytabPath + '\'' |
||||
+ '}'; |
||||
} |
||||
|
||||
@Override |
||||
public DbType getType() { |
||||
return DbType.HIVE; |
||||
} |
||||
} |
||||
@ -0,0 +1,185 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.datasource.hive; |
||||
|
||||
import org.apache.dolphinscheduler.common.Constants; |
||||
import org.apache.dolphinscheduler.common.datasource.AbstractDatasourceProcessor; |
||||
import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam; |
||||
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; |
||||
import org.apache.dolphinscheduler.common.datasource.ConnectionParam; |
||||
import org.apache.dolphinscheduler.common.enums.DbType; |
||||
import org.apache.dolphinscheduler.common.utils.CommonUtils; |
||||
import org.apache.dolphinscheduler.common.utils.HiveConfUtils; |
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils; |
||||
import org.apache.dolphinscheduler.common.utils.StringUtils; |
||||
|
||||
import org.apache.commons.collections4.MapUtils; |
||||
|
||||
import java.io.IOException; |
||||
import java.sql.Connection; |
||||
import java.sql.DriverManager; |
||||
import java.sql.SQLException; |
||||
import java.util.LinkedHashMap; |
||||
import java.util.Map; |
||||
|
||||
public class HiveDatasourceProcessor extends AbstractDatasourceProcessor { |
||||
|
||||
@Override |
||||
public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { |
||||
HiveDataSourceParamDTO hiveDataSourceParamDTO = new HiveDataSourceParamDTO(); |
||||
HiveConnectionParam hiveConnectionParam = (HiveConnectionParam) createConnectionParams(connectionJson); |
||||
|
||||
hiveDataSourceParamDTO.setDatabase(hiveConnectionParam.getDatabase()); |
||||
hiveDataSourceParamDTO.setUserName(hiveConnectionParam.getUser()); |
||||
hiveDataSourceParamDTO.setOther(parseOther(hiveConnectionParam.getOther())); |
||||
hiveDataSourceParamDTO.setLoginUserKeytabUsername(hiveConnectionParam.getLoginUserKeytabUsername()); |
||||
hiveDataSourceParamDTO.setLoginUserKeytabPath(hiveConnectionParam.getLoginUserKeytabPath()); |
||||
hiveDataSourceParamDTO.setJavaSecurityKrb5Conf(hiveConnectionParam.getJavaSecurityKrb5Conf()); |
||||
|
||||
String[] tmpArray = hiveConnectionParam.getAddress().split(Constants.DOUBLE_SLASH); |
||||
StringBuilder hosts = new StringBuilder(); |
||||
String[] hostPortArray = tmpArray[tmpArray.length - 1].split(Constants.COMMA); |
||||
for (String hostPort : hostPortArray) { |
||||
hosts.append(hostPort.split(Constants.COLON)[0]).append(Constants.COMMA); |
||||
} |
||||
hosts.deleteCharAt(hosts.length() - 1); |
||||
hiveDataSourceParamDTO.setHost(hosts.toString()); |
||||
hiveDataSourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(Constants.COLON)[1])); |
||||
|
||||
return hiveDataSourceParamDTO; |
||||
} |
||||
|
||||
@Override |
||||
public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { |
||||
HiveDataSourceParamDTO hiveParam = (HiveDataSourceParamDTO) datasourceParam; |
||||
StringBuilder address = new StringBuilder(); |
||||
address.append(Constants.JDBC_HIVE_2); |
||||
for (String zkHost : hiveParam.getHost().split(",")) { |
||||
address.append(String.format("%s:%s,", zkHost, hiveParam.getPort())); |
||||
} |
||||
address.deleteCharAt(address.length() - 1); |
||||
String jdbcUrl = address.toString() + "/" + hiveParam.getDatabase(); |
||||
if (CommonUtils.getKerberosStartupState()) { |
||||
jdbcUrl += ";principal=" + hiveParam.getPrincipal(); |
||||
} |
||||
|
||||
HiveConnectionParam hiveConnectionParam = new HiveConnectionParam(); |
||||
hiveConnectionParam.setDatabase(hiveParam.getDatabase()); |
||||
hiveConnectionParam.setAddress(address.toString()); |
||||
hiveConnectionParam.setJdbcUrl(jdbcUrl); |
||||
hiveConnectionParam.setUser(hiveParam.getUserName()); |
||||
hiveConnectionParam.setPassword(CommonUtils.encodePassword(hiveParam.getPassword())); |
||||
|
||||
if (CommonUtils.getKerberosStartupState()) { |
||||
hiveConnectionParam.setPrincipal(hiveParam.getPrincipal()); |
||||
hiveConnectionParam.setJavaSecurityKrb5Conf(hiveParam.getJavaSecurityKrb5Conf()); |
||||
hiveConnectionParam.setLoginUserKeytabPath(hiveParam.getLoginUserKeytabPath()); |
||||
hiveConnectionParam.setLoginUserKeytabUsername(hiveParam.getLoginUserKeytabUsername()); |
||||
} |
||||
hiveConnectionParam.setOther(transformOther(hiveParam.getOther())); |
||||
return hiveConnectionParam; |
||||
} |
||||
|
||||
@Override |
||||
public ConnectionParam createConnectionParams(String connectionJson) { |
||||
return JSONUtils.parseObject(connectionJson, HiveConnectionParam.class); |
||||
} |
||||
|
||||
@Override |
||||
public String getDatasourceDriver() { |
||||
return Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER; |
||||
} |
||||
|
||||
@Override |
||||
public String getJdbcUrl(ConnectionParam connectionParam) { |
||||
HiveConnectionParam hiveConnectionParam = (HiveConnectionParam) connectionParam; |
||||
String jdbcUrl = hiveConnectionParam.getJdbcUrl(); |
||||
String otherParams = filterOther(hiveConnectionParam.getOther()); |
||||
if (StringUtils.isNotEmpty(otherParams) && !"?".equals(otherParams.substring(0, 1))) { |
||||
jdbcUrl += ";"; |
||||
} |
||||
return jdbcUrl + otherParams; |
||||
} |
||||
|
||||
@Override |
||||
public Connection getConnection(ConnectionParam connectionParam) throws IOException, ClassNotFoundException, SQLException { |
||||
HiveConnectionParam hiveConnectionParam = (HiveConnectionParam) connectionParam; |
||||
CommonUtils.loadKerberosConf(hiveConnectionParam.getJavaSecurityKrb5Conf(), |
||||
hiveConnectionParam.getLoginUserKeytabUsername(), hiveConnectionParam.getLoginUserKeytabPath()); |
||||
Class.forName(getDatasourceDriver()); |
||||
return DriverManager.getConnection(getJdbcUrl(connectionParam), |
||||
hiveConnectionParam.getUser(), CommonUtils.decodePassword(hiveConnectionParam.getPassword())); |
||||
} |
||||
|
||||
@Override |
||||
public DbType getDbType() { |
||||
return DbType.HIVE; |
||||
} |
||||
|
||||
private String transformOther(Map<String, String> otherMap) { |
||||
if (MapUtils.isEmpty(otherMap)) { |
||||
return null; |
||||
} |
||||
StringBuilder stringBuilder = new StringBuilder(); |
||||
otherMap.forEach((key, value) -> stringBuilder.append(String.format("%s=%s;", key, value))); |
||||
return stringBuilder.toString(); |
||||
} |
||||
|
||||
private String filterOther(String otherParams) { |
||||
if (StringUtils.isBlank(otherParams)) { |
||||
return ""; |
||||
} |
||||
|
||||
StringBuilder hiveConfListSb = new StringBuilder(); |
||||
hiveConfListSb.append("?"); |
||||
StringBuilder sessionVarListSb = new StringBuilder(); |
||||
|
||||
String[] otherArray = otherParams.split(";", -1); |
||||
|
||||
for (String conf : otherArray) { |
||||
if (HiveConfUtils.isHiveConfVar(conf)) { |
||||
hiveConfListSb.append(conf).append(";"); |
||||
} else { |
||||
sessionVarListSb.append(conf).append(";"); |
||||
} |
||||
} |
||||
|
||||
// remove the last ";"
|
||||
if (sessionVarListSb.length() > 0) { |
||||
sessionVarListSb.deleteCharAt(sessionVarListSb.length() - 1); |
||||
} |
||||
|
||||
if (hiveConfListSb.length() > 0) { |
||||
hiveConfListSb.deleteCharAt(hiveConfListSb.length() - 1); |
||||
} |
||||
|
||||
return sessionVarListSb.toString() + hiveConfListSb.toString(); |
||||
} |
||||
|
||||
private Map<String, String> parseOther(String other) { |
||||
if (other == null) { |
||||
return null; |
||||
} |
||||
Map<String, String> otherMap = new LinkedHashMap<>(); |
||||
String[] configs = other.split(";"); |
||||
for (String config : configs) { |
||||
otherMap.put(config.split("=")[0], config.split("=")[1]); |
||||
} |
||||
return otherMap; |
||||
} |
||||
} |
||||
@ -0,0 +1,43 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.datasource.mysql; |
||||
|
||||
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; |
||||
import org.apache.dolphinscheduler.common.enums.DbType; |
||||
|
||||
public class MysqlDatasourceParamDTO extends BaseDataSourceParamDTO { |
||||
|
||||
@Override |
||||
public String toString() { |
||||
return "MysqlDatasourceParamDTO{" |
||||
+ "name='" + name + '\'' |
||||
+ ", note='" + note + '\'' |
||||
+ ", host='" + host + '\'' |
||||
+ ", port=" + port |
||||
+ ", database='" + database + '\'' |
||||
+ ", userName='" + userName + '\'' |
||||
+ ", password='" + password + '\'' |
||||
+ ", other='" + other + '\'' |
||||
+ '}'; |
||||
} |
||||
|
||||
@Override |
||||
public DbType getType() { |
||||
return DbType.MYSQL; |
||||
} |
||||
} |
||||
@ -0,0 +1,170 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.datasource.mysql; |
||||
|
||||
import org.apache.dolphinscheduler.common.Constants; |
||||
import org.apache.dolphinscheduler.common.datasource.AbstractDatasourceProcessor; |
||||
import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam; |
||||
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; |
||||
import org.apache.dolphinscheduler.common.datasource.ConnectionParam; |
||||
import org.apache.dolphinscheduler.common.enums.DbType; |
||||
import org.apache.dolphinscheduler.common.utils.CommonUtils; |
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils; |
||||
import org.apache.dolphinscheduler.common.utils.StringUtils; |
||||
|
||||
import org.apache.commons.collections4.MapUtils; |
||||
|
||||
import java.sql.Connection; |
||||
import java.sql.DriverManager; |
||||
import java.sql.SQLException; |
||||
import java.util.HashMap; |
||||
import java.util.LinkedHashMap; |
||||
import java.util.Map; |
||||
|
||||
import org.slf4j.Logger; |
||||
import org.slf4j.LoggerFactory; |
||||
|
||||
public class MysqlDatasourceProcessor extends AbstractDatasourceProcessor { |
||||
|
||||
private final Logger logger = LoggerFactory.getLogger(MysqlDatasourceProcessor.class); |
||||
|
||||
private static final String ALLOW_LOAD_LOCAL_IN_FILE_NAME = "allowLoadLocalInfile"; |
||||
|
||||
private static final String AUTO_DESERIALIZE = "autoDeserialize"; |
||||
|
||||
private static final String ALLOW_LOCAL_IN_FILE_NAME = "allowLocalInfile"; |
||||
|
||||
private static final String ALLOW_URL_IN_LOCAL_IN_FILE_NAME = "allowUrlInLocalInfile"; |
||||
|
||||
private static final String APPEND_PARAMS = "allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false"; |
||||
|
||||
@Override |
||||
public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { |
||||
MysqlConnectionParam connectionParams = (MysqlConnectionParam) createConnectionParams(connectionJson); |
||||
MysqlDatasourceParamDTO mysqlDatasourceParamDTO = new MysqlDatasourceParamDTO(); |
||||
|
||||
mysqlDatasourceParamDTO.setUserName(connectionParams.getUser()); |
||||
mysqlDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); |
||||
mysqlDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); |
||||
|
||||
String address = connectionParams.getAddress(); |
||||
String[] hostSeperator = address.split(Constants.DOUBLE_SLASH); |
||||
String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(Constants.COMMA); |
||||
mysqlDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(Constants.COLON)[1])); |
||||
mysqlDatasourceParamDTO.setHost(hostPortArray[0].split(Constants.COLON)[0]); |
||||
|
||||
return mysqlDatasourceParamDTO; |
||||
} |
||||
|
||||
@Override |
||||
public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO dataSourceParam) { |
||||
MysqlDatasourceParamDTO mysqlDatasourceParam = (MysqlDatasourceParamDTO) dataSourceParam; |
||||
String address = String.format("%s%s:%s", Constants.JDBC_MYSQL, mysqlDatasourceParam.getHost(), mysqlDatasourceParam.getPort()); |
||||
String jdbcUrl = String.format("%s/%s", address, mysqlDatasourceParam.getDatabase()); |
||||
|
||||
MysqlConnectionParam mysqlConnectionParam = new MysqlConnectionParam(); |
||||
mysqlConnectionParam.setJdbcUrl(jdbcUrl); |
||||
mysqlConnectionParam.setDatabase(mysqlDatasourceParam.getDatabase()); |
||||
mysqlConnectionParam.setAddress(address); |
||||
mysqlConnectionParam.setUser(mysqlDatasourceParam.getUserName()); |
||||
mysqlConnectionParam.setPassword(CommonUtils.encodePassword(mysqlDatasourceParam.getPassword())); |
||||
mysqlConnectionParam.setOther(transformOther(mysqlDatasourceParam.getOther())); |
||||
|
||||
return mysqlConnectionParam; |
||||
} |
||||
|
||||
@Override |
||||
public ConnectionParam createConnectionParams(String connectionJson) { |
||||
return JSONUtils.parseObject(connectionJson, MysqlConnectionParam.class); |
||||
} |
||||
|
||||
@Override |
||||
public String getDatasourceDriver() { |
||||
return Constants.COM_MYSQL_JDBC_DRIVER; |
||||
} |
||||
|
||||
@Override |
||||
public String getJdbcUrl(ConnectionParam connectionParam) { |
||||
MysqlConnectionParam mysqlConnectionParam = (MysqlConnectionParam) connectionParam; |
||||
String jdbcUrl = mysqlConnectionParam.getJdbcUrl(); |
||||
if (StringUtils.isNotEmpty(mysqlConnectionParam.getOther())) { |
||||
return String.format("%s?%s&%s", jdbcUrl, mysqlConnectionParam.getOther(), APPEND_PARAMS); |
||||
} |
||||
return String.format("%s?%s", jdbcUrl, APPEND_PARAMS); |
||||
} |
||||
|
||||
@Override |
||||
public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException { |
||||
MysqlConnectionParam mysqlConnectionParam = (MysqlConnectionParam) connectionParam; |
||||
Class.forName(getDatasourceDriver()); |
||||
String user = mysqlConnectionParam.getUser(); |
||||
if (user.contains(AUTO_DESERIALIZE)) { |
||||
logger.warn("sensitive param : {} in username field is filtered", AUTO_DESERIALIZE); |
||||
user = user.replace(AUTO_DESERIALIZE, ""); |
||||
} |
||||
String password = CommonUtils.decodePassword(mysqlConnectionParam.getPassword()); |
||||
if (password.contains(AUTO_DESERIALIZE)) { |
||||
logger.warn("sensitive param : {} in password field is filtered", AUTO_DESERIALIZE); |
||||
password = password.replace(AUTO_DESERIALIZE, ""); |
||||
} |
||||
return DriverManager.getConnection(getJdbcUrl(connectionParam), user, password); |
||||
} |
||||
|
||||
@Override |
||||
public DbType getDbType() { |
||||
return DbType.MYSQL; |
||||
} |
||||
|
||||
private String transformOther(Map<String, String> paramMap) { |
||||
if (MapUtils.isEmpty(paramMap)) { |
||||
return null; |
||||
} |
||||
Map<String, String> otherMap = new HashMap<>(); |
||||
paramMap.forEach((k, v) -> { |
||||
if (!checkKeyIsLegitimate(k)) { |
||||
return; |
||||
} |
||||
otherMap.put(k, v); |
||||
}); |
||||
if (MapUtils.isEmpty(otherMap)) { |
||||
return null; |
||||
} |
||||
StringBuilder stringBuilder = new StringBuilder(); |
||||
otherMap.forEach((key, value) -> stringBuilder.append(String.format("%s=%s&", key, value))); |
||||
return stringBuilder.toString(); |
||||
} |
||||
|
||||
private static boolean checkKeyIsLegitimate(String key) { |
||||
return !key.contains(ALLOW_LOAD_LOCAL_IN_FILE_NAME) |
||||
&& !key.contains(AUTO_DESERIALIZE) |
||||
&& !key.contains(ALLOW_LOCAL_IN_FILE_NAME) |
||||
&& !key.contains(ALLOW_URL_IN_LOCAL_IN_FILE_NAME); |
||||
} |
||||
|
||||
private Map<String, String> parseOther(String other) { |
||||
if (StringUtils.isEmpty(other)) { |
||||
return null; |
||||
} |
||||
Map<String, String> otherMap = new LinkedHashMap<>(); |
||||
for (String config : other.split("&")) { |
||||
otherMap.put(config.split("=")[0], config.split("=")[1]); |
||||
} |
||||
return otherMap; |
||||
} |
||||
|
||||
} |
||||
@ -0,0 +1,46 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.datasource.oracle; |
||||
|
||||
import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam; |
||||
import org.apache.dolphinscheduler.common.enums.DbConnectType; |
||||
|
||||
public class OracleConnectionParam extends BaseConnectionParam { |
||||
protected DbConnectType connectType; |
||||
|
||||
public DbConnectType getConnectType() { |
||||
return connectType; |
||||
} |
||||
|
||||
public void setConnectType(DbConnectType connectType) { |
||||
this.connectType = connectType; |
||||
} |
||||
|
||||
@Override |
||||
public String toString() { |
||||
return "OracleConnectionParam{" |
||||
+ "user='" + user + '\'' |
||||
+ ", password='" + password + '\'' |
||||
+ ", address='" + address + '\'' |
||||
+ ", database='" + database + '\'' |
||||
+ ", jdbcUrl='" + jdbcUrl + '\'' |
||||
+ ", other='" + other + '\'' |
||||
+ ", connectType=" + connectType |
||||
+ '}'; |
||||
} |
||||
} |
||||
@ -0,0 +1,141 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.datasource.oracle; |
||||
|
||||
import org.apache.dolphinscheduler.common.Constants; |
||||
import org.apache.dolphinscheduler.common.datasource.AbstractDatasourceProcessor; |
||||
import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam; |
||||
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; |
||||
import org.apache.dolphinscheduler.common.datasource.ConnectionParam; |
||||
import org.apache.dolphinscheduler.common.enums.DbConnectType; |
||||
import org.apache.dolphinscheduler.common.enums.DbType; |
||||
import org.apache.dolphinscheduler.common.utils.CommonUtils; |
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils; |
||||
import org.apache.dolphinscheduler.common.utils.StringUtils; |
||||
|
||||
import org.apache.commons.collections4.MapUtils; |
||||
|
||||
import java.sql.Connection; |
||||
import java.sql.DriverManager; |
||||
import java.sql.SQLException; |
||||
import java.util.ArrayList; |
||||
import java.util.LinkedHashMap; |
||||
import java.util.List; |
||||
import java.util.Map; |
||||
|
||||
public class OracleDatasourceProcessor extends AbstractDatasourceProcessor { |
||||
|
||||
@Override |
||||
public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { |
||||
OracleConnectionParam connectionParams = (OracleConnectionParam) createConnectionParams(connectionJson); |
||||
OracleDatasourceParamDTO oracleDatasourceParamDTO = new OracleDatasourceParamDTO(); |
||||
|
||||
oracleDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); |
||||
oracleDatasourceParamDTO.setUserName(connectionParams.getUser()); |
||||
oracleDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); |
||||
|
||||
String hostSeperator = Constants.DOUBLE_SLASH; |
||||
if (DbConnectType.ORACLE_SID.equals(connectionParams.connectType)) { |
||||
hostSeperator = Constants.AT_SIGN; |
||||
} |
||||
String[] hostPort = connectionParams.getAddress().split(hostSeperator); |
||||
String[] hostPortArray = hostPort[hostPort.length - 1].split(Constants.COMMA); |
||||
oracleDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(Constants.COLON)[1])); |
||||
oracleDatasourceParamDTO.setHost(hostPortArray[0].split(Constants.COLON)[0]); |
||||
|
||||
return oracleDatasourceParamDTO; |
||||
} |
||||
|
||||
@Override |
||||
public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { |
||||
OracleDatasourceParamDTO oracleParam = (OracleDatasourceParamDTO) datasourceParam; |
||||
String address; |
||||
if (DbConnectType.ORACLE_SID.equals(oracleParam.getConnectType())) { |
||||
address = String.format("%s%s:%s", |
||||
Constants.JDBC_ORACLE_SID, oracleParam.getHost(), oracleParam.getPort()); |
||||
} else { |
||||
address = String.format("%s%s:%s", |
||||
Constants.JDBC_ORACLE_SERVICE_NAME, oracleParam.getHost(), oracleParam.getPort()); |
||||
} |
||||
String jdbcUrl = address + "/" + oracleParam.getDatabase(); |
||||
|
||||
OracleConnectionParam oracleConnectionParam = new OracleConnectionParam(); |
||||
oracleConnectionParam.setUser(oracleParam.getUserName()); |
||||
oracleConnectionParam.setPassword(CommonUtils.encodePassword(oracleParam.getPassword())); |
||||
oracleConnectionParam.setAddress(address); |
||||
oracleConnectionParam.setJdbcUrl(jdbcUrl); |
||||
oracleConnectionParam.setDatabase(oracleParam.getDatabase()); |
||||
oracleConnectionParam.setConnectType(oracleParam.getConnectType()); |
||||
oracleConnectionParam.setOther(transformOther(oracleParam.getOther())); |
||||
|
||||
return oracleConnectionParam; |
||||
} |
||||
|
||||
@Override |
||||
public ConnectionParam createConnectionParams(String connectionJson) { |
||||
return JSONUtils.parseObject(connectionJson, OracleConnectionParam.class); |
||||
} |
||||
|
||||
@Override |
||||
public String getDatasourceDriver() { |
||||
return Constants.COM_ORACLE_JDBC_DRIVER; |
||||
} |
||||
|
||||
@Override |
||||
public String getJdbcUrl(ConnectionParam connectionParam) { |
||||
OracleConnectionParam oracleConnectionParam = (OracleConnectionParam) connectionParam; |
||||
if (StringUtils.isNotEmpty(oracleConnectionParam.getOther())) { |
||||
return String.format("%s?%s", oracleConnectionParam.getJdbcUrl(), oracleConnectionParam.getOther()); |
||||
} |
||||
return oracleConnectionParam.getJdbcUrl(); |
||||
} |
||||
|
||||
@Override |
||||
public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException { |
||||
OracleConnectionParam oracleConnectionParam = (OracleConnectionParam) connectionParam; |
||||
Class.forName(getDatasourceDriver()); |
||||
return DriverManager.getConnection(getJdbcUrl(connectionParam), |
||||
oracleConnectionParam.getUser(), CommonUtils.decodePassword(oracleConnectionParam.getPassword())); |
||||
} |
||||
|
||||
@Override |
||||
public DbType getDbType() { |
||||
return DbType.ORACLE; |
||||
} |
||||
|
||||
private String transformOther(Map<String, String> otherMap) { |
||||
if (MapUtils.isEmpty(otherMap)) { |
||||
return null; |
||||
} |
||||
List<String> list = new ArrayList<>(); |
||||
otherMap.forEach((key, value) -> list.add(String.format("%s=%s", key, value))); |
||||
return String.join("&", list); |
||||
} |
||||
|
||||
private Map<String, String> parseOther(String other) { |
||||
if (StringUtils.isEmpty(other)) { |
||||
return null; |
||||
} |
||||
Map<String, String> otherMap = new LinkedHashMap<>(); |
||||
String[] configs = other.split("&"); |
||||
for (String config : configs) { |
||||
otherMap.put(config.split("=")[0], config.split("=")[1]); |
||||
} |
||||
return otherMap; |
||||
} |
||||
} |
||||
@ -0,0 +1,34 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.datasource.postgresql; |
||||
|
||||
import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam; |
||||
|
||||
public class PostgreSqlConnectionParam extends BaseConnectionParam { |
||||
@Override |
||||
public String toString() { |
||||
return "PostgreSqlConnectionParam{" |
||||
+ "user='" + user + '\'' |
||||
+ ", password='" + password + '\'' |
||||
+ ", address='" + address + '\'' |
||||
+ ", database='" + database + '\'' |
||||
+ ", jdbcUrl='" + jdbcUrl + '\'' |
||||
+ ", other='" + other + '\'' |
||||
+ '}'; |
||||
} |
||||
} |
||||
@ -0,0 +1,41 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.datasource.postgresql; |
||||
|
||||
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; |
||||
import org.apache.dolphinscheduler.common.enums.DbType; |
||||
|
||||
public class PostgreSqlDatasourceParamDTO extends BaseDataSourceParamDTO { |
||||
|
||||
@Override |
||||
public String toString() { |
||||
return "PostgreSqlDatasourceParamDTO{" |
||||
+ "host='" + host + '\'' |
||||
+ ", port=" + port |
||||
+ ", database='" + database + '\'' |
||||
+ ", userName='" + userName + '\'' |
||||
+ ", password='" + password + '\'' |
||||
+ ", other='" + other + '\'' |
||||
+ '}'; |
||||
} |
||||
|
||||
@Override |
||||
public DbType getType() { |
||||
return DbType.POSTGRESQL; |
||||
} |
||||
} |
||||
@ -0,0 +1,126 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.datasource.postgresql; |
||||
|
||||
import org.apache.dolphinscheduler.common.Constants; |
||||
import org.apache.dolphinscheduler.common.datasource.AbstractDatasourceProcessor; |
||||
import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam; |
||||
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; |
||||
import org.apache.dolphinscheduler.common.datasource.ConnectionParam; |
||||
import org.apache.dolphinscheduler.common.enums.DbType; |
||||
import org.apache.dolphinscheduler.common.utils.CommonUtils; |
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils; |
||||
import org.apache.dolphinscheduler.common.utils.StringUtils; |
||||
|
||||
import org.apache.commons.collections4.MapUtils; |
||||
|
||||
import java.sql.Connection; |
||||
import java.sql.DriverManager; |
||||
import java.sql.SQLException; |
||||
import java.util.LinkedHashMap; |
||||
import java.util.Map; |
||||
|
||||
public class PostgreSqlDatasourceProcessor extends AbstractDatasourceProcessor { |
||||
|
||||
@Override |
||||
public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { |
||||
PostgreSqlConnectionParam connectionParams = (PostgreSqlConnectionParam) createConnectionParams(connectionJson); |
||||
PostgreSqlDatasourceParamDTO postgreSqlDatasourceParamDTO = new PostgreSqlDatasourceParamDTO(); |
||||
postgreSqlDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); |
||||
postgreSqlDatasourceParamDTO.setUserName(connectionParams.getUser()); |
||||
postgreSqlDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); |
||||
|
||||
String address = connectionParams.getAddress(); |
||||
String[] hostSeperator = address.split(Constants.DOUBLE_SLASH); |
||||
String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(Constants.COMMA); |
||||
postgreSqlDatasourceParamDTO.setHost(hostPortArray[0].split(Constants.COLON)[0]); |
||||
postgreSqlDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(Constants.COLON)[1])); |
||||
|
||||
return postgreSqlDatasourceParamDTO; |
||||
} |
||||
|
||||
@Override |
||||
public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { |
||||
PostgreSqlDatasourceParamDTO postgreSqlParam = (PostgreSqlDatasourceParamDTO) datasourceParam; |
||||
String address = String.format("%s%s:%s", Constants.JDBC_POSTGRESQL, postgreSqlParam.getHost(), postgreSqlParam.getPort()); |
||||
String jdbcUrl = String.format("%s/%s", address, postgreSqlParam.getDatabase()); |
||||
|
||||
PostgreSqlConnectionParam postgreSqlConnectionParam = new PostgreSqlConnectionParam(); |
||||
postgreSqlConnectionParam.setJdbcUrl(jdbcUrl); |
||||
postgreSqlConnectionParam.setAddress(address); |
||||
postgreSqlConnectionParam.setDatabase(postgreSqlParam.getDatabase()); |
||||
postgreSqlConnectionParam.setUser(postgreSqlParam.getUserName()); |
||||
postgreSqlConnectionParam.setPassword(CommonUtils.encodePassword(postgreSqlParam.getPassword())); |
||||
postgreSqlConnectionParam.setOther(transformOther(postgreSqlParam.getOther())); |
||||
|
||||
return postgreSqlConnectionParam; |
||||
} |
||||
|
||||
@Override |
||||
public ConnectionParam createConnectionParams(String connectionJson) { |
||||
return JSONUtils.parseObject(connectionJson, PostgreSqlConnectionParam.class); |
||||
} |
||||
|
||||
@Override |
||||
public String getDatasourceDriver() { |
||||
return Constants.ORG_POSTGRESQL_DRIVER; |
||||
} |
||||
|
||||
@Override |
||||
public String getJdbcUrl(ConnectionParam connectionParam) { |
||||
PostgreSqlConnectionParam postgreSqlConnectionParam = (PostgreSqlConnectionParam) connectionParam; |
||||
if (StringUtils.isNotEmpty(postgreSqlConnectionParam.getOther())) { |
||||
return String.format("%s?%s", postgreSqlConnectionParam.getJdbcUrl(), postgreSqlConnectionParam.getOther()); |
||||
} |
||||
return postgreSqlConnectionParam.getJdbcUrl(); |
||||
} |
||||
|
||||
@Override |
||||
public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException { |
||||
PostgreSqlConnectionParam postgreSqlConnectionParam = (PostgreSqlConnectionParam) connectionParam; |
||||
Class.forName(getDatasourceDriver()); |
||||
return DriverManager.getConnection(getJdbcUrl(postgreSqlConnectionParam), |
||||
postgreSqlConnectionParam.getUser(), CommonUtils.decodePassword(postgreSqlConnectionParam.getPassword())); |
||||
} |
||||
|
||||
@Override |
||||
public DbType getDbType() { |
||||
return DbType.POSTGRESQL; |
||||
} |
||||
|
||||
private String transformOther(Map<String, String> otherMap) { |
||||
if (MapUtils.isEmpty(otherMap)) { |
||||
return null; |
||||
} |
||||
StringBuilder stringBuilder = new StringBuilder(); |
||||
otherMap.forEach((key, value) -> stringBuilder.append(String.format("%s=%s&", key, value))); |
||||
return stringBuilder.toString(); |
||||
} |
||||
|
||||
private Map<String, String> parseOther(String other) { |
||||
if (StringUtils.isEmpty(other)) { |
||||
return null; |
||||
} |
||||
Map<String, String> otherMap = new LinkedHashMap<>(); |
||||
for (String config : other.split("&")) { |
||||
String[] split = config.split("="); |
||||
otherMap.put(split[0], split[1]); |
||||
} |
||||
return otherMap; |
||||
} |
||||
} |
||||
@ -0,0 +1,34 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.datasource.presto; |
||||
|
||||
import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam; |
||||
|
||||
public class PrestoConnectionParam extends BaseConnectionParam { |
||||
@Override |
||||
public String toString() { |
||||
return "PrestoConnectionParam{" |
||||
+ "user='" + user + '\'' |
||||
+ ", password='" + password + '\'' |
||||
+ ", address='" + address + '\'' |
||||
+ ", database='" + database + '\'' |
||||
+ ", jdbcUrl='" + jdbcUrl + '\'' |
||||
+ ", other='" + other + '\'' |
||||
+ '}'; |
||||
} |
||||
} |
||||
@ -0,0 +1,43 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.datasource.presto; |
||||
|
||||
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; |
||||
import org.apache.dolphinscheduler.common.enums.DbType; |
||||
|
||||
public class PrestoDatasourceParamDTO extends BaseDataSourceParamDTO { |
||||
|
||||
@Override |
||||
public String toString() { |
||||
return "PrestoDatasourceParamDTO{" |
||||
+ "name='" + name + '\'' |
||||
+ ", note='" + note + '\'' |
||||
+ ", host='" + host + '\'' |
||||
+ ", port=" + port |
||||
+ ", database='" + database + '\'' |
||||
+ ", userName='" + userName + '\'' |
||||
+ ", password='" + password + '\'' |
||||
+ ", other='" + other + '\'' |
||||
+ '}'; |
||||
} |
||||
|
||||
@Override |
||||
public DbType getType() { |
||||
return DbType.PRESTO; |
||||
} |
||||
} |
||||
@ -0,0 +1,128 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.datasource.presto; |
||||
|
||||
import org.apache.dolphinscheduler.common.Constants; |
||||
import org.apache.dolphinscheduler.common.datasource.AbstractDatasourceProcessor; |
||||
import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam; |
||||
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; |
||||
import org.apache.dolphinscheduler.common.datasource.ConnectionParam; |
||||
import org.apache.dolphinscheduler.common.enums.DbType; |
||||
import org.apache.dolphinscheduler.common.utils.CommonUtils; |
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils; |
||||
import org.apache.dolphinscheduler.common.utils.StringUtils; |
||||
|
||||
import org.apache.commons.collections4.MapUtils; |
||||
|
||||
import java.sql.Connection; |
||||
import java.sql.DriverManager; |
||||
import java.sql.SQLException; |
||||
import java.util.ArrayList; |
||||
import java.util.LinkedHashMap; |
||||
import java.util.List; |
||||
import java.util.Map; |
||||
|
||||
public class PrestoDatasourceProcessor extends AbstractDatasourceProcessor { |
||||
|
||||
@Override |
||||
public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { |
||||
PrestoConnectionParam connectionParams = (PrestoConnectionParam) createConnectionParams(connectionJson); |
||||
|
||||
String[] hostSeperator = connectionParams.getAddress().split(Constants.DOUBLE_SLASH); |
||||
String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(Constants.COMMA); |
||||
|
||||
PrestoDatasourceParamDTO prestoDatasourceParamDTO = new PrestoDatasourceParamDTO(); |
||||
prestoDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(Constants.COLON)[1])); |
||||
prestoDatasourceParamDTO.setHost(hostPortArray[0].split(Constants.COLON)[0]); |
||||
prestoDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); |
||||
prestoDatasourceParamDTO.setUserName(connectionParams.getUser()); |
||||
prestoDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); |
||||
|
||||
return prestoDatasourceParamDTO; |
||||
} |
||||
|
||||
@Override |
||||
public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { |
||||
PrestoDatasourceParamDTO prestoParam = (PrestoDatasourceParamDTO) datasourceParam; |
||||
String address = String.format("%s%s:%s", Constants.JDBC_PRESTO, prestoParam.getHost(), prestoParam.getPort()); |
||||
String jdbcUrl = address + "/" + prestoParam.getDatabase(); |
||||
|
||||
PrestoConnectionParam prestoConnectionParam = new PrestoConnectionParam(); |
||||
prestoConnectionParam.setUser(prestoParam.getUserName()); |
||||
prestoConnectionParam.setPassword(CommonUtils.encodePassword(prestoParam.getPassword())); |
||||
prestoConnectionParam.setOther(transformOther(prestoParam.getOther())); |
||||
prestoConnectionParam.setAddress(address); |
||||
prestoConnectionParam.setJdbcUrl(jdbcUrl); |
||||
prestoConnectionParam.setDatabase(prestoParam.getDatabase()); |
||||
|
||||
return prestoConnectionParam; |
||||
} |
||||
|
||||
@Override |
||||
public ConnectionParam createConnectionParams(String connectionJson) { |
||||
return JSONUtils.parseObject(connectionJson, PrestoConnectionParam.class); |
||||
} |
||||
|
||||
@Override |
||||
public String getDatasourceDriver() { |
||||
return Constants.COM_PRESTO_JDBC_DRIVER; |
||||
} |
||||
|
||||
@Override |
||||
public String getJdbcUrl(ConnectionParam connectionParam) { |
||||
PrestoConnectionParam prestoConnectionParam = (PrestoConnectionParam) connectionParam; |
||||
if (StringUtils.isNotEmpty(prestoConnectionParam.getOther())) { |
||||
return String.format("%s?%s", prestoConnectionParam.getJdbcUrl(), prestoConnectionParam.getOther()); |
||||
} |
||||
return prestoConnectionParam.getJdbcUrl(); |
||||
} |
||||
|
||||
@Override |
||||
public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException { |
||||
PrestoConnectionParam prestoConnectionParam = (PrestoConnectionParam) connectionParam; |
||||
Class.forName(getDatasourceDriver()); |
||||
return DriverManager.getConnection(getJdbcUrl(connectionParam), |
||||
prestoConnectionParam.getUser(), CommonUtils.decodePassword(prestoConnectionParam.getPassword())); |
||||
} |
||||
|
||||
@Override |
||||
public DbType getDbType() { |
||||
return DbType.PRESTO; |
||||
} |
||||
|
||||
private String transformOther(Map<String, String> otherMap) { |
||||
if (MapUtils.isNotEmpty(otherMap)) { |
||||
List<String> list = new ArrayList<>(); |
||||
otherMap.forEach((key, value) -> list.add(String.format("%s=%s", key, value))); |
||||
return String.join("&", list); |
||||
} |
||||
return null; |
||||
} |
||||
|
||||
private Map<String, String> parseOther(String other) { |
||||
if (StringUtils.isEmpty(other)) { |
||||
return null; |
||||
} |
||||
Map<String, String> otherMap = new LinkedHashMap<>(); |
||||
String[] configs = other.split("&"); |
||||
for (String config : configs) { |
||||
otherMap.put(config.split("=")[0], config.split("=")[1]); |
||||
} |
||||
return otherMap; |
||||
} |
||||
} |
||||
@ -0,0 +1,38 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.datasource.spark; |
||||
|
||||
import org.apache.dolphinscheduler.common.datasource.BaseHdfsConnectionParam; |
||||
|
||||
public class SparkConnectionParam extends BaseHdfsConnectionParam { |
||||
@Override |
||||
public String toString() { |
||||
return "SparkConnectionParam{" |
||||
+ "user='" + user + '\'' |
||||
+ ", password='" + password + '\'' |
||||
+ ", address='" + address + '\'' |
||||
+ ", database='" + database + '\'' |
||||
+ ", jdbcUrl='" + jdbcUrl + '\'' |
||||
+ ", other='" + other + '\'' |
||||
+ ", principal='" + principal + '\'' |
||||
+ ", javaSecurityKrb5Conf='" + javaSecurityKrb5Conf + '\'' |
||||
+ ", loginUserKeytabUsername='" + loginUserKeytabUsername + '\'' |
||||
+ ", loginUserKeytabPath='" + loginUserKeytabPath + '\'' |
||||
+ '}'; |
||||
} |
||||
} |
||||
@ -0,0 +1,154 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.datasource.spark; |
||||
|
||||
import org.apache.dolphinscheduler.common.Constants; |
||||
import org.apache.dolphinscheduler.common.datasource.AbstractDatasourceProcessor; |
||||
import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam; |
||||
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; |
||||
import org.apache.dolphinscheduler.common.datasource.ConnectionParam; |
||||
import org.apache.dolphinscheduler.common.enums.DbType; |
||||
import org.apache.dolphinscheduler.common.utils.CommonUtils; |
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils; |
||||
import org.apache.dolphinscheduler.common.utils.StringUtils; |
||||
|
||||
import org.apache.commons.collections4.MapUtils; |
||||
|
||||
import java.io.IOException; |
||||
import java.sql.Connection; |
||||
import java.sql.DriverManager; |
||||
import java.sql.SQLException; |
||||
import java.util.Arrays; |
||||
import java.util.LinkedHashMap; |
||||
import java.util.List; |
||||
import java.util.Map; |
||||
import java.util.stream.Collectors; |
||||
|
||||
public class SparkDatasourceProcessor extends AbstractDatasourceProcessor { |
||||
|
||||
@Override |
||||
public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { |
||||
SparkConnectionParam connectionParams = (SparkConnectionParam) createConnectionParams(connectionJson); |
||||
|
||||
SparkDatasourceParamDTO sparkDatasourceParamDTO = new SparkDatasourceParamDTO(); |
||||
sparkDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); |
||||
sparkDatasourceParamDTO.setUserName(connectionParams.getUser()); |
||||
sparkDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); |
||||
sparkDatasourceParamDTO.setJavaSecurityKrb5Conf(connectionParams.getJavaSecurityKrb5Conf()); |
||||
sparkDatasourceParamDTO.setLoginUserKeytabPath(connectionParams.getLoginUserKeytabPath()); |
||||
sparkDatasourceParamDTO.setLoginUserKeytabUsername(connectionParams.getLoginUserKeytabUsername()); |
||||
|
||||
StringBuilder hosts = new StringBuilder(); |
||||
String[] tmpArray = connectionParams.getAddress().split(Constants.DOUBLE_SLASH); |
||||
String[] hostPortArray = tmpArray[tmpArray.length - 1].split(Constants.COMMA); |
||||
Arrays.stream(hostPortArray).forEach(hostPort -> hosts.append(hostPort.split(Constants.COLON)[0]).append(Constants.COMMA)); |
||||
hosts.deleteCharAt(hosts.length() - 1); |
||||
|
||||
sparkDatasourceParamDTO.setHost(hosts.toString()); |
||||
sparkDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(Constants.COLON)[1])); |
||||
|
||||
return sparkDatasourceParamDTO; |
||||
} |
||||
|
||||
@Override |
||||
public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO dataSourceParam) { |
||||
StringBuilder address = new StringBuilder(); |
||||
SparkDatasourceParamDTO sparkDatasourceParam = (SparkDatasourceParamDTO) dataSourceParam; |
||||
address.append(Constants.JDBC_HIVE_2); |
||||
for (String zkHost : sparkDatasourceParam.getHost().split(",")) { |
||||
address.append(String.format("%s:%s,", zkHost, sparkDatasourceParam.getPort())); |
||||
} |
||||
address.deleteCharAt(address.length() - 1); |
||||
|
||||
String jdbcUrl = address + "/" + sparkDatasourceParam.getDatabase(); |
||||
if (CommonUtils.getKerberosStartupState()) { |
||||
jdbcUrl += ";principal=" + sparkDatasourceParam.getPrincipal(); |
||||
} |
||||
|
||||
SparkConnectionParam sparkConnectionParam = new SparkConnectionParam(); |
||||
sparkConnectionParam.setPassword(CommonUtils.encodePassword(sparkDatasourceParam.getPassword())); |
||||
sparkConnectionParam.setUser(sparkDatasourceParam.getUserName()); |
||||
sparkConnectionParam.setOther(transformOther(sparkDatasourceParam.getOther())); |
||||
sparkConnectionParam.setDatabase(sparkDatasourceParam.getDatabase()); |
||||
sparkConnectionParam.setAddress(address.toString()); |
||||
sparkConnectionParam.setJdbcUrl(jdbcUrl); |
||||
if (CommonUtils.getKerberosStartupState()) { |
||||
sparkConnectionParam.setPrincipal(sparkDatasourceParam.getPrincipal()); |
||||
sparkConnectionParam.setJavaSecurityKrb5Conf(sparkDatasourceParam.getJavaSecurityKrb5Conf()); |
||||
sparkConnectionParam.setLoginUserKeytabPath(sparkDatasourceParam.getLoginUserKeytabPath()); |
||||
sparkConnectionParam.setLoginUserKeytabUsername(sparkDatasourceParam.getLoginUserKeytabUsername()); |
||||
} |
||||
|
||||
return sparkConnectionParam; |
||||
} |
||||
|
||||
@Override |
||||
public ConnectionParam createConnectionParams(String connectionJson) { |
||||
return JSONUtils.parseObject(connectionJson, SparkConnectionParam.class); |
||||
} |
||||
|
||||
@Override |
||||
public String getDatasourceDriver() { |
||||
return Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER; |
||||
} |
||||
|
||||
@Override |
||||
public String getJdbcUrl(ConnectionParam connectionParam) { |
||||
SparkConnectionParam sparkConnectionParam = (SparkConnectionParam) connectionParam; |
||||
if (StringUtils.isNotEmpty(sparkConnectionParam.getOther())) { |
||||
return String.format("%s;%s", sparkConnectionParam.getJdbcUrl(), sparkConnectionParam.getOther()); |
||||
} |
||||
return sparkConnectionParam.getJdbcUrl(); |
||||
} |
||||
|
||||
@Override |
||||
public Connection getConnection(ConnectionParam connectionParam) throws IOException, ClassNotFoundException, SQLException { |
||||
SparkConnectionParam sparkConnectionParam = (SparkConnectionParam) connectionParam; |
||||
CommonUtils.loadKerberosConf(sparkConnectionParam.getJavaSecurityKrb5Conf(), |
||||
sparkConnectionParam.getLoginUserKeytabUsername(), sparkConnectionParam.getLoginUserKeytabPath()); |
||||
Class.forName(getDatasourceDriver()); |
||||
return DriverManager.getConnection(getJdbcUrl(sparkConnectionParam), |
||||
sparkConnectionParam.getUser(), CommonUtils.decodePassword(sparkConnectionParam.getPassword())); |
||||
} |
||||
|
||||
@Override |
||||
public DbType getDbType() { |
||||
return DbType.SPARK; |
||||
} |
||||
|
||||
private String transformOther(Map<String, String> otherMap) { |
||||
if (MapUtils.isEmpty(otherMap)) { |
||||
return null; |
||||
} |
||||
List<String> stringBuilder = otherMap.entrySet().stream() |
||||
.map(entry -> String.format("%s=%s", entry.getKey(), entry.getValue())).collect(Collectors.toList()); |
||||
return String.join(";", stringBuilder); |
||||
} |
||||
|
||||
private Map<String, String> parseOther(String other) { |
||||
if (StringUtils.isEmpty(other)) { |
||||
return null; |
||||
} |
||||
Map<String, String> otherMap = new LinkedHashMap<>(); |
||||
String[] configs = other.split(";"); |
||||
for (String config : configs) { |
||||
otherMap.put(config.split("=")[0], config.split("=")[1]); |
||||
} |
||||
return otherMap; |
||||
} |
||||
} |
||||
@ -0,0 +1,34 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.datasource.sqlserver; |
||||
|
||||
import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam; |
||||
|
||||
public class SqlServerConnectionParam extends BaseConnectionParam { |
||||
@Override |
||||
public String toString() { |
||||
return "SqlServerConnectionParam{" |
||||
+ "user='" + user + '\'' |
||||
+ ", password='" + password + '\'' |
||||
+ ", address='" + address + '\'' |
||||
+ ", database='" + database + '\'' |
||||
+ ", jdbcUrl='" + jdbcUrl + '\'' |
||||
+ ", other='" + other + '\'' |
||||
+ '}'; |
||||
} |
||||
} |
||||
@ -0,0 +1,123 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.datasource.sqlserver; |
||||
|
||||
import org.apache.dolphinscheduler.common.Constants; |
||||
import org.apache.dolphinscheduler.common.datasource.AbstractDatasourceProcessor; |
||||
import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam; |
||||
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; |
||||
import org.apache.dolphinscheduler.common.datasource.ConnectionParam; |
||||
import org.apache.dolphinscheduler.common.enums.DbType; |
||||
import org.apache.dolphinscheduler.common.utils.CommonUtils; |
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils; |
||||
import org.apache.dolphinscheduler.common.utils.StringUtils; |
||||
|
||||
import org.apache.commons.collections4.MapUtils; |
||||
|
||||
import java.sql.Connection; |
||||
import java.sql.DriverManager; |
||||
import java.sql.SQLException; |
||||
import java.util.LinkedHashMap; |
||||
import java.util.Map; |
||||
|
||||
public class SqlServerDatasourceProcessor extends AbstractDatasourceProcessor { |
||||
|
||||
@Override |
||||
public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { |
||||
SqlServerConnectionParam connectionParams = (SqlServerConnectionParam) createConnectionParams(connectionJson); |
||||
String[] hostSeperator = connectionParams.getAddress().split(Constants.DOUBLE_SLASH); |
||||
String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(Constants.COMMA); |
||||
|
||||
SqlServerDatasourceParamDTO sqlServerDatasourceParamDTO = new SqlServerDatasourceParamDTO(); |
||||
sqlServerDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); |
||||
sqlServerDatasourceParamDTO.setUserName(connectionParams.getUser()); |
||||
sqlServerDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); |
||||
sqlServerDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(Constants.COLON)[1])); |
||||
sqlServerDatasourceParamDTO.setHost(hostPortArray[0].split(Constants.COLON)[0]); |
||||
return sqlServerDatasourceParamDTO; |
||||
} |
||||
|
||||
@Override |
||||
public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { |
||||
SqlServerDatasourceParamDTO sqlServerParam = (SqlServerDatasourceParamDTO) datasourceParam; |
||||
String address = String.format("%s%s:%s", Constants.JDBC_SQLSERVER, sqlServerParam.getHost(), sqlServerParam.getPort()); |
||||
String jdbcUrl = address + ";databaseName=" + sqlServerParam.getDatabase(); |
||||
|
||||
SqlServerConnectionParam sqlServerConnectionParam = new SqlServerConnectionParam(); |
||||
sqlServerConnectionParam.setAddress(address); |
||||
sqlServerConnectionParam.setDatabase(sqlServerParam.getDatabase()); |
||||
sqlServerConnectionParam.setJdbcUrl(jdbcUrl); |
||||
sqlServerConnectionParam.setOther(transformOther(sqlServerParam.getOther())); |
||||
sqlServerConnectionParam.setUser(sqlServerParam.getUserName()); |
||||
sqlServerConnectionParam.setPassword(CommonUtils.encodePassword(sqlServerParam.getPassword())); |
||||
return sqlServerConnectionParam; |
||||
} |
||||
|
||||
@Override |
||||
public BaseConnectionParam createConnectionParams(String connectionJson) { |
||||
return JSONUtils.parseObject(connectionJson, SqlServerConnectionParam.class); |
||||
} |
||||
|
||||
@Override |
||||
public String getDatasourceDriver() { |
||||
return Constants.COM_SQLSERVER_JDBC_DRIVER; |
||||
} |
||||
|
||||
@Override |
||||
public String getJdbcUrl(ConnectionParam connectionParam) { |
||||
SqlServerConnectionParam sqlServerConnectionParam = (SqlServerConnectionParam) connectionParam; |
||||
|
||||
if (StringUtils.isNotEmpty(sqlServerConnectionParam.getOther())) { |
||||
return String.format("%s;%s", sqlServerConnectionParam.getJdbcUrl(), sqlServerConnectionParam.getOther()); |
||||
} |
||||
return sqlServerConnectionParam.getJdbcUrl(); |
||||
} |
||||
|
||||
@Override |
||||
public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException { |
||||
SqlServerConnectionParam sqlServerConnectionParam = (SqlServerConnectionParam) connectionParam; |
||||
Class.forName(getDatasourceDriver()); |
||||
return DriverManager.getConnection(getJdbcUrl(connectionParam), sqlServerConnectionParam.getUser(), |
||||
CommonUtils.decodePassword(sqlServerConnectionParam.getPassword())); |
||||
} |
||||
|
||||
@Override |
||||
public DbType getDbType() { |
||||
return DbType.SQLSERVER; |
||||
} |
||||
|
||||
private String transformOther(Map<String, String> otherMap) { |
||||
if (MapUtils.isEmpty(otherMap)) { |
||||
return null; |
||||
} |
||||
StringBuilder stringBuilder = new StringBuilder(); |
||||
otherMap.forEach((key, value) -> stringBuilder.append(String.format("%s=%s;", key, value))); |
||||
return stringBuilder.toString(); |
||||
} |
||||
|
||||
private Map<String, String> parseOther(String other) { |
||||
if (StringUtils.isEmpty(other)) { |
||||
return null; |
||||
} |
||||
Map<String, String> otherMap = new LinkedHashMap<>(); |
||||
for (String config : other.split(";")) { |
||||
otherMap.put(config.split("=")[0], config.split("=")[1]); |
||||
} |
||||
return otherMap; |
||||
} |
||||
} |
||||
@ -1,63 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.utils; |
||||
|
||||
/** |
||||
* http utils |
||||
*/ |
||||
public class IpUtils { |
||||
|
||||
private IpUtils() { |
||||
throw new UnsupportedOperationException("Construct IpUtils"); |
||||
} |
||||
|
||||
public static final String DOT = "."; |
||||
|
||||
/** |
||||
* ip str to long <p> |
||||
* |
||||
* @param ipStr ip string |
||||
* @return ip to long |
||||
*/ |
||||
public static Long ipToLong(String ipStr) { |
||||
String[] ipSet = ipStr.split("\\" + DOT); |
||||
|
||||
return Long.parseLong(ipSet[0]) << 24 | Long.parseLong(ipSet[1]) << 16 | Long.parseLong(ipSet[2]) << 8 | Long.parseLong(ipSet[3]); |
||||
} |
||||
|
||||
/** |
||||
* long to ip |
||||
* |
||||
* @param ipLong the long number converted from IP |
||||
* @return String |
||||
*/ |
||||
public static String longToIp(long ipLong) { |
||||
long[] ipNumbers = new long[4]; |
||||
long tmp = 0xFF; |
||||
ipNumbers[0] = ipLong >> 24 & tmp; |
||||
ipNumbers[1] = ipLong >> 16 & tmp; |
||||
ipNumbers[2] = ipLong >> 8 & tmp; |
||||
ipNumbers[3] = ipLong & tmp; |
||||
|
||||
return ipNumbers[0] + DOT |
||||
+ ipNumbers[1] + DOT |
||||
+ ipNumbers[2] + DOT |
||||
+ ipNumbers[3]; |
||||
} |
||||
|
||||
} |
||||
@ -0,0 +1,130 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.datasource; |
||||
|
||||
import org.apache.dolphinscheduler.common.datasource.mysql.MysqlConnectionParam; |
||||
import org.apache.dolphinscheduler.common.datasource.mysql.MysqlDatasourceParamDTO; |
||||
import org.apache.dolphinscheduler.common.datasource.mysql.MysqlDatasourceProcessor; |
||||
import org.apache.dolphinscheduler.common.enums.DbType; |
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils; |
||||
|
||||
import java.sql.Connection; |
||||
import java.sql.DriverManager; |
||||
import java.sql.SQLException; |
||||
|
||||
import org.junit.Assert; |
||||
import org.junit.Test; |
||||
import org.junit.runner.RunWith; |
||||
import org.mockito.Mockito; |
||||
import org.powermock.api.mockito.PowerMockito; |
||||
import org.powermock.core.classloader.annotations.PrepareForTest; |
||||
import org.powermock.modules.junit4.PowerMockRunner; |
||||
|
||||
@RunWith(PowerMockRunner.class) |
||||
@PrepareForTest({Class.class, DriverManager.class, MysqlDatasourceProcessor.class}) |
||||
public class DatasourceUtilTest { |
||||
|
||||
@Test |
||||
public void testCheckDatasourceParam() { |
||||
MysqlDatasourceParamDTO mysqlDatasourceParamDTO = new MysqlDatasourceParamDTO(); |
||||
mysqlDatasourceParamDTO.setHost("localhost"); |
||||
mysqlDatasourceParamDTO.setDatabase("default"); |
||||
mysqlDatasourceParamDTO.setOther(null); |
||||
DatasourceUtil.checkDatasourceParam(mysqlDatasourceParamDTO); |
||||
Assert.assertTrue(true); |
||||
} |
||||
|
||||
@Test |
||||
public void testBuildConnectionParams() { |
||||
MysqlDatasourceParamDTO mysqlDatasourceParamDTO = new MysqlDatasourceParamDTO(); |
||||
mysqlDatasourceParamDTO.setHost("localhost"); |
||||
mysqlDatasourceParamDTO.setDatabase("default"); |
||||
mysqlDatasourceParamDTO.setUserName("root"); |
||||
mysqlDatasourceParamDTO.setPort(3306); |
||||
mysqlDatasourceParamDTO.setPassword("123456"); |
||||
ConnectionParam connectionParam = DatasourceUtil.buildConnectionParams(mysqlDatasourceParamDTO); |
||||
Assert.assertNotNull(connectionParam); |
||||
} |
||||
|
||||
@Test |
||||
public void testBuildConnectionParams2() { |
||||
MysqlDatasourceParamDTO mysqlDatasourceParamDTO = new MysqlDatasourceParamDTO(); |
||||
mysqlDatasourceParamDTO.setHost("localhost"); |
||||
mysqlDatasourceParamDTO.setDatabase("default"); |
||||
mysqlDatasourceParamDTO.setUserName("root"); |
||||
mysqlDatasourceParamDTO.setPort(3306); |
||||
mysqlDatasourceParamDTO.setPassword("123456"); |
||||
ConnectionParam connectionParam = DatasourceUtil.buildConnectionParams(DbType.MYSQL, JSONUtils.toJsonString(mysqlDatasourceParamDTO)); |
||||
Assert.assertNotNull(connectionParam); |
||||
} |
||||
|
||||
@Test |
||||
public void testGetConnection() throws ClassNotFoundException, SQLException { |
||||
PowerMockito.mockStatic(Class.class); |
||||
PowerMockito.when(Class.forName(Mockito.any())).thenReturn(null); |
||||
PowerMockito.mockStatic(DriverManager.class); |
||||
PowerMockito.when(DriverManager.getConnection(Mockito.any(), Mockito.any(), Mockito.any())).thenReturn(null); |
||||
|
||||
MysqlConnectionParam connectionParam = new MysqlConnectionParam(); |
||||
connectionParam.setUser("root"); |
||||
connectionParam.setPassword("123456"); |
||||
Connection connection = DatasourceUtil.getConnection(DbType.MYSQL, connectionParam); |
||||
|
||||
Assert.assertNull(connection); |
||||
|
||||
} |
||||
|
||||
@Test |
||||
public void testGetJdbcUrl() { |
||||
MysqlConnectionParam mysqlConnectionParam = new MysqlConnectionParam(); |
||||
mysqlConnectionParam.setJdbcUrl("jdbc:mysql://localhost:3308"); |
||||
String jdbcUrl = DatasourceUtil.getJdbcUrl(DbType.MYSQL, mysqlConnectionParam); |
||||
Assert.assertEquals("jdbc:mysql://localhost:3308?allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false", |
||||
jdbcUrl); |
||||
} |
||||
|
||||
@Test |
||||
public void testBuildDatasourceParamDTO() { |
||||
MysqlConnectionParam connectionParam = new MysqlConnectionParam(); |
||||
connectionParam.setJdbcUrl("jdbc:mysql://localhost:3308?allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false"); |
||||
connectionParam.setAddress("jdbc:mysql://localhost:3308"); |
||||
connectionParam.setUser("root"); |
||||
connectionParam.setPassword("123456"); |
||||
|
||||
Assert.assertNotNull(DatasourceUtil.buildDatasourceParamDTO(DbType.MYSQL, JSONUtils.toJsonString(connectionParam))); |
||||
|
||||
} |
||||
|
||||
@Test |
||||
public void testGetDatasourceProcessor() { |
||||
Assert.assertNotNull(DatasourceUtil.getDatasourceProcessor(DbType.MYSQL)); |
||||
Assert.assertNotNull(DatasourceUtil.getDatasourceProcessor(DbType.POSTGRESQL)); |
||||
Assert.assertNotNull(DatasourceUtil.getDatasourceProcessor(DbType.HIVE)); |
||||
Assert.assertNotNull(DatasourceUtil.getDatasourceProcessor(DbType.SPARK)); |
||||
Assert.assertNotNull(DatasourceUtil.getDatasourceProcessor(DbType.CLICKHOUSE)); |
||||
Assert.assertNotNull(DatasourceUtil.getDatasourceProcessor(DbType.ORACLE)); |
||||
Assert.assertNotNull(DatasourceUtil.getDatasourceProcessor(DbType.SQLSERVER)); |
||||
Assert.assertNotNull(DatasourceUtil.getDatasourceProcessor(DbType.DB2)); |
||||
Assert.assertNotNull(DatasourceUtil.getDatasourceProcessor(DbType.PRESTO)); |
||||
} |
||||
|
||||
@Test(expected = Exception.class) |
||||
public void testGetDatasourceProcessorError() { |
||||
DatasourceUtil.getDatasourceProcessor(null); |
||||
} |
||||
} |
||||
@ -0,0 +1,83 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.datasource.clickhouse; |
||||
|
||||
import org.apache.dolphinscheduler.common.Constants; |
||||
import org.apache.dolphinscheduler.common.enums.DbType; |
||||
|
||||
import java.sql.DriverManager; |
||||
|
||||
import org.junit.Assert; |
||||
import org.junit.Test; |
||||
import org.junit.runner.RunWith; |
||||
import org.powermock.core.classloader.annotations.PrepareForTest; |
||||
import org.powermock.modules.junit4.PowerMockRunner; |
||||
|
||||
@RunWith(PowerMockRunner.class) |
||||
@PrepareForTest({Class.class, DriverManager.class}) |
||||
public class ClickHouseDatasourceProcessorTest { |
||||
|
||||
private ClickHouseDatasourceProcessor clickHouseDatasourceProcessor = new ClickHouseDatasourceProcessor(); |
||||
|
||||
@Test |
||||
public void testCreateConnectionParams() { |
||||
ClickHouseDatasourceParamDTO clickhouseConnectionParam = new ClickHouseDatasourceParamDTO(); |
||||
clickhouseConnectionParam.setUserName("user"); |
||||
clickhouseConnectionParam.setPassword("password"); |
||||
clickhouseConnectionParam.setHost("localhost"); |
||||
clickhouseConnectionParam.setPort(8123); |
||||
clickhouseConnectionParam.setDatabase("default"); |
||||
ClickhouseConnectionParam connectionParams = (ClickhouseConnectionParam) clickHouseDatasourceProcessor |
||||
.createConnectionParams(clickhouseConnectionParam); |
||||
Assert.assertNotNull(connectionParams); |
||||
Assert.assertEquals("jdbc:clickhouse://localhost:8123", connectionParams.getAddress()); |
||||
Assert.assertEquals("jdbc:clickhouse://localhost:8123/default", connectionParams.getJdbcUrl()); |
||||
} |
||||
|
||||
@Test |
||||
public void testCreateConnectionParams2() { |
||||
String connectionParamJson = "{\"address\":\"jdbc:clickhouse://localhost:8123\",\"database\":\"default\"," |
||||
+ "\"jdbcUrl\":\"jdbc:clickhouse://localhost:8123/default\",\"user\":\"default\",\"password\":\"123456\"}"; |
||||
ClickhouseConnectionParam clickhouseConnectionParam = (ClickhouseConnectionParam) clickHouseDatasourceProcessor |
||||
.createConnectionParams(connectionParamJson); |
||||
Assert.assertNotNull(clickhouseConnectionParam); |
||||
Assert.assertEquals("default", clickhouseConnectionParam.getUser()); |
||||
Assert.assertEquals("123456", clickhouseConnectionParam.getPassword()); |
||||
} |
||||
|
||||
@Test |
||||
public void testGetDatasourceDriver() { |
||||
Assert.assertNotNull(clickHouseDatasourceProcessor.getDatasourceDriver()); |
||||
Assert.assertEquals(Constants.COM_CLICKHOUSE_JDBC_DRIVER, clickHouseDatasourceProcessor.getDatasourceDriver()); |
||||
} |
||||
|
||||
@Test |
||||
public void testGetJdbcUrl() { |
||||
ClickhouseConnectionParam connectionParam = new ClickhouseConnectionParam(); |
||||
connectionParam.setUser("default"); |
||||
connectionParam.setJdbcUrl("jdbc:clickhouse://localhost:8123/default"); |
||||
connectionParam.setOther("other=other1"); |
||||
String jdbcUrl = clickHouseDatasourceProcessor.getJdbcUrl(connectionParam); |
||||
Assert.assertEquals("jdbc:clickhouse://localhost:8123/default?other=other1", jdbcUrl); |
||||
} |
||||
|
||||
@Test |
||||
public void testGetDbType() { |
||||
Assert.assertEquals(DbType.CLICKHOUSE, clickHouseDatasourceProcessor.getDbType()); |
||||
} |
||||
} |
||||
@ -0,0 +1,83 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.datasource.db2; |
||||
|
||||
import org.apache.dolphinscheduler.common.Constants; |
||||
import org.apache.dolphinscheduler.common.enums.DbType; |
||||
|
||||
import java.sql.DriverManager; |
||||
|
||||
import org.junit.Assert; |
||||
import org.junit.Test; |
||||
import org.junit.runner.RunWith; |
||||
import org.powermock.core.classloader.annotations.PrepareForTest; |
||||
import org.powermock.modules.junit4.PowerMockRunner; |
||||
|
||||
@RunWith(PowerMockRunner.class) |
||||
@PrepareForTest({Class.class, DriverManager.class}) |
||||
public class Db2DatasourceProcessorTest { |
||||
|
||||
private Db2DatasourceProcessor db2DatasourceProcessor = new Db2DatasourceProcessor(); |
||||
|
||||
@Test |
||||
public void testCreateConnectionParams() { |
||||
Db2DatasourceParamDTO db2DatasourceParamDTO = new Db2DatasourceParamDTO(); |
||||
db2DatasourceParamDTO.setUserName("root"); |
||||
db2DatasourceParamDTO.setPassword("123456"); |
||||
db2DatasourceParamDTO.setHost("localhost"); |
||||
db2DatasourceParamDTO.setPort(5142); |
||||
db2DatasourceParamDTO.setDatabase("default"); |
||||
|
||||
Db2ConnectionParam connectionParams = (Db2ConnectionParam) db2DatasourceProcessor |
||||
.createConnectionParams(db2DatasourceParamDTO); |
||||
Assert.assertNotNull(connectionParams); |
||||
Assert.assertEquals("jdbc:db2://localhost:5142", connectionParams.getAddress()); |
||||
Assert.assertEquals("jdbc:db2://localhost:5142/default", connectionParams.getJdbcUrl()); |
||||
} |
||||
|
||||
@Test |
||||
public void testCreateConnectionParams2() { |
||||
String connectionJson = "{\"user\":\"root\",\"password\":\"123456\",\"address\":\"jdbc:db2://localhost:5142\"" |
||||
+ ",\"database\":\"default\",\"jdbcUrl\":\"jdbc:db2://localhost:5142/default\"}"; |
||||
Db2ConnectionParam connectionParams = (Db2ConnectionParam) db2DatasourceProcessor |
||||
.createConnectionParams(connectionJson); |
||||
Assert.assertNotNull(connectionJson); |
||||
Assert.assertEquals("root", connectionParams.getUser()); |
||||
|
||||
} |
||||
|
||||
@Test |
||||
public void testGetDatasourceDriver() { |
||||
Assert.assertEquals(Constants.COM_DB2_JDBC_DRIVER, db2DatasourceProcessor.getDatasourceDriver()); |
||||
} |
||||
|
||||
@Test |
||||
public void testGetJdbcUrl() { |
||||
Db2ConnectionParam db2ConnectionParam = new Db2ConnectionParam(); |
||||
db2ConnectionParam.setJdbcUrl("jdbc:db2://localhost:5142/default"); |
||||
db2ConnectionParam.setOther("other=other"); |
||||
String jdbcUrl = db2DatasourceProcessor.getJdbcUrl(db2ConnectionParam); |
||||
Assert.assertEquals("jdbc:db2://localhost:5142/default;other=other", jdbcUrl); |
||||
} |
||||
|
||||
@Test |
||||
public void testGetDbType() { |
||||
Assert.assertEquals(DbType.DB2, db2DatasourceProcessor.getDbType()); |
||||
} |
||||
|
||||
} |
||||
@ -0,0 +1,79 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.datasource.hive; |
||||
|
||||
import org.apache.dolphinscheduler.common.Constants; |
||||
import org.apache.dolphinscheduler.common.enums.DbType; |
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils; |
||||
|
||||
import java.sql.DriverManager; |
||||
|
||||
import org.junit.Assert; |
||||
import org.junit.Test; |
||||
import org.junit.runner.RunWith; |
||||
import org.powermock.core.classloader.annotations.PrepareForTest; |
||||
import org.powermock.modules.junit4.PowerMockRunner; |
||||
|
||||
@RunWith(PowerMockRunner.class) |
||||
@PrepareForTest({Class.class, DriverManager.class}) |
||||
public class HiveDatasourceProcessorTest { |
||||
|
||||
private HiveDatasourceProcessor hiveDatasourceProcessor = new HiveDatasourceProcessor(); |
||||
|
||||
@Test |
||||
public void testCreateConnectionParams() { |
||||
HiveDataSourceParamDTO hiveDataSourceParamDTO = new HiveDataSourceParamDTO(); |
||||
hiveDataSourceParamDTO.setHost("localhost1,localhost2"); |
||||
hiveDataSourceParamDTO.setPort(5142); |
||||
hiveDataSourceParamDTO.setUserName("default"); |
||||
hiveDataSourceParamDTO.setDatabase("default"); |
||||
HiveConnectionParam connectionParams = (HiveConnectionParam) hiveDatasourceProcessor |
||||
.createConnectionParams(hiveDataSourceParamDTO); |
||||
System.out.println(JSONUtils.toJsonString(connectionParams)); |
||||
Assert.assertNotNull(connectionParams); |
||||
Assert.assertEquals("jdbc:hive2://localhost1:5142,localhost2:5142", connectionParams.getAddress()); |
||||
} |
||||
|
||||
@Test |
||||
public void testCreateConnectionParams2() { |
||||
String connectionParam = "{\"user\":\"default\",\"address\":\"jdbc:hive2://localhost1:5142,localhost2:5142\"" |
||||
+ ",\"jdbcUrl\":\"jdbc:hive2://localhost1:5142,localhost2:5142/default\"}"; |
||||
HiveConnectionParam connectionParams = (HiveConnectionParam) hiveDatasourceProcessor |
||||
.createConnectionParams(connectionParam); |
||||
Assert.assertNotNull(connectionParam); |
||||
Assert.assertEquals("default", connectionParams.getUser()); |
||||
} |
||||
|
||||
@Test |
||||
public void testGetDatasourceDriver() { |
||||
Assert.assertEquals(Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER, hiveDatasourceProcessor.getDatasourceDriver()); |
||||
} |
||||
|
||||
@Test |
||||
public void testGetJdbcUrl() { |
||||
HiveConnectionParam connectionParam = new HiveConnectionParam(); |
||||
connectionParam.setJdbcUrl("jdbc:hive2://localhost1:5142,localhost2:5142/default"); |
||||
Assert.assertEquals("jdbc:hive2://localhost1:5142,localhost2:5142/default", |
||||
hiveDatasourceProcessor.getJdbcUrl(connectionParam)); |
||||
} |
||||
|
||||
@Test |
||||
public void testGetDbType() { |
||||
Assert.assertEquals(DbType.HIVE, hiveDatasourceProcessor.getDbType()); |
||||
} |
||||
} |
||||
@ -0,0 +1,81 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.datasource.mysql; |
||||
|
||||
import org.apache.dolphinscheduler.common.Constants; |
||||
import org.apache.dolphinscheduler.common.enums.DbType; |
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils; |
||||
|
||||
import java.sql.DriverManager; |
||||
|
||||
import org.junit.Assert; |
||||
import org.junit.Test; |
||||
import org.junit.runner.RunWith; |
||||
import org.powermock.core.classloader.annotations.PrepareForTest; |
||||
import org.powermock.modules.junit4.PowerMockRunner; |
||||
|
||||
@RunWith(PowerMockRunner.class) |
||||
@PrepareForTest({Class.class, DriverManager.class}) |
||||
public class MysqlDatasourceProcessorTest { |
||||
|
||||
private MysqlDatasourceProcessor mysqlDatasourceProcessor = new MysqlDatasourceProcessor(); |
||||
|
||||
@Test |
||||
public void testCreateConnectionParams() { |
||||
MysqlDatasourceParamDTO mysqlDatasourceParamDTO = new MysqlDatasourceParamDTO(); |
||||
mysqlDatasourceParamDTO.setUserName("root"); |
||||
mysqlDatasourceParamDTO.setPassword("123456"); |
||||
mysqlDatasourceParamDTO.setHost("localhost"); |
||||
mysqlDatasourceParamDTO.setPort(3306); |
||||
mysqlDatasourceParamDTO.setDatabase("default"); |
||||
|
||||
MysqlConnectionParam connectionParams = (MysqlConnectionParam) mysqlDatasourceProcessor |
||||
.createConnectionParams(mysqlDatasourceParamDTO); |
||||
System.out.println(JSONUtils.toJsonString(connectionParams)); |
||||
Assert.assertEquals("jdbc:mysql://localhost:3306", connectionParams.getAddress()); |
||||
Assert.assertEquals("jdbc:mysql://localhost:3306/default", connectionParams.getJdbcUrl()); |
||||
} |
||||
|
||||
@Test |
||||
public void testCreateConnectionParams2() { |
||||
String connectionJson = "{\"user\":\"root\",\"password\":\"123456\",\"address\":\"jdbc:mysql://localhost:3306\"" |
||||
+ ",\"database\":\"default\",\"jdbcUrl\":\"jdbc:mysql://localhost:3306/default\"}"; |
||||
MysqlConnectionParam connectionParams = (MysqlConnectionParam) mysqlDatasourceProcessor |
||||
.createConnectionParams(connectionJson); |
||||
Assert.assertNotNull(connectionJson); |
||||
Assert.assertEquals("root", connectionParams.getUser()); |
||||
} |
||||
|
||||
@Test |
||||
public void testGetDatasourceDriver() { |
||||
Assert.assertEquals(Constants.COM_MYSQL_JDBC_DRIVER, mysqlDatasourceProcessor.getDatasourceDriver()); |
||||
} |
||||
|
||||
@Test |
||||
public void testGetJdbcUrl() { |
||||
MysqlConnectionParam mysqlConnectionParam = new MysqlConnectionParam(); |
||||
mysqlConnectionParam.setJdbcUrl("jdbc:mysql://localhost:3306/default"); |
||||
Assert.assertEquals("jdbc:mysql://localhost:3306/default?allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false", |
||||
mysqlDatasourceProcessor.getJdbcUrl(mysqlConnectionParam)); |
||||
} |
||||
|
||||
@Test |
||||
public void testGetDbType() { |
||||
Assert.assertEquals(DbType.MYSQL, mysqlDatasourceProcessor.getDbType()); |
||||
} |
||||
} |
||||
@ -0,0 +1,83 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.datasource.oracle; |
||||
|
||||
import org.apache.dolphinscheduler.common.Constants; |
||||
import org.apache.dolphinscheduler.common.enums.DbConnectType; |
||||
import org.apache.dolphinscheduler.common.enums.DbType; |
||||
|
||||
import java.sql.DriverManager; |
||||
|
||||
import org.junit.Assert; |
||||
import org.junit.Test; |
||||
import org.junit.runner.RunWith; |
||||
import org.powermock.core.classloader.annotations.PrepareForTest; |
||||
import org.powermock.modules.junit4.PowerMockRunner; |
||||
|
||||
@RunWith(PowerMockRunner.class) |
||||
@PrepareForTest({Class.class, DriverManager.class}) |
||||
public class OracleDatasourceProcessorTest { |
||||
|
||||
private OracleDatasourceProcessor oracleDatasourceProcessor = new OracleDatasourceProcessor(); |
||||
|
||||
@Test |
||||
public void testCreateConnectionParams() { |
||||
OracleDatasourceParamDTO oracleDatasourceParamDTO = new OracleDatasourceParamDTO(); |
||||
oracleDatasourceParamDTO.setConnectType(DbConnectType.ORACLE_SID); |
||||
oracleDatasourceParamDTO.setHost("localhost"); |
||||
oracleDatasourceParamDTO.setPort(3308); |
||||
oracleDatasourceParamDTO.setUserName("root"); |
||||
oracleDatasourceParamDTO.setPassword("123456"); |
||||
oracleDatasourceParamDTO.setDatabase("default"); |
||||
|
||||
OracleConnectionParam connectionParams = (OracleConnectionParam) oracleDatasourceProcessor |
||||
.createConnectionParams(oracleDatasourceParamDTO); |
||||
Assert.assertNotNull(connectionParams); |
||||
Assert.assertEquals("jdbc:oracle:thin:@localhost:3308", connectionParams.getAddress()); |
||||
Assert.assertEquals("jdbc:oracle:thin:@localhost:3308/default", connectionParams.getJdbcUrl()); |
||||
} |
||||
|
||||
@Test |
||||
public void testCreateConnectionParams2() { |
||||
String connectionJson = "{\"user\":\"root\",\"password\":\"123456\",\"address\":\"jdbc:oracle:thin:@localhost:3308\"" |
||||
+ ",\"database\":\"default\",\"jdbcUrl\":\"jdbc:oracle:thin:@localhost:3308/default\",\"connectType\":\"ORACLE_SID\"}"; |
||||
OracleConnectionParam connectionParams = (OracleConnectionParam) oracleDatasourceProcessor |
||||
.createConnectionParams(connectionJson); |
||||
Assert.assertNotNull(connectionParams); |
||||
Assert.assertEquals("root", connectionParams.getUser()); |
||||
} |
||||
|
||||
@Test |
||||
public void testGetDatasourceDriver() { |
||||
Assert.assertEquals(Constants.COM_ORACLE_JDBC_DRIVER, oracleDatasourceProcessor.getDatasourceDriver()); |
||||
} |
||||
|
||||
@Test |
||||
public void testGetJdbcUrl() { |
||||
OracleConnectionParam oracleConnectionParam = new OracleConnectionParam(); |
||||
oracleConnectionParam.setJdbcUrl("jdbc:oracle:thin:@localhost:3308/default"); |
||||
oracleConnectionParam.setOther("other=other"); |
||||
Assert.assertEquals("jdbc:oracle:thin:@localhost:3308/default?other=other", |
||||
oracleDatasourceProcessor.getJdbcUrl(oracleConnectionParam)); |
||||
} |
||||
|
||||
@Test |
||||
public void getDbType() { |
||||
Assert.assertEquals(DbType.ORACLE, oracleDatasourceProcessor.getDbType()); |
||||
} |
||||
} |
||||
@ -0,0 +1,83 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.datasource.postgresql; |
||||
|
||||
import org.apache.dolphinscheduler.common.Constants; |
||||
import org.apache.dolphinscheduler.common.enums.DbType; |
||||
|
||||
import java.sql.DriverManager; |
||||
|
||||
import org.junit.Assert; |
||||
import org.junit.Test; |
||||
import org.junit.runner.RunWith; |
||||
import org.powermock.core.classloader.annotations.PrepareForTest; |
||||
import org.powermock.modules.junit4.PowerMockRunner; |
||||
|
||||
@RunWith(PowerMockRunner.class) |
||||
@PrepareForTest({Class.class, DriverManager.class}) |
||||
public class PostgreSqlDatasourceProcessorTest { |
||||
|
||||
private PostgreSqlDatasourceProcessor postgreSqlDatasourceProcessor = new PostgreSqlDatasourceProcessor(); |
||||
|
||||
@Test |
||||
public void testCreateConnectionParams() { |
||||
PostgreSqlDatasourceParamDTO postgreSqlDatasourceParamDTO = new PostgreSqlDatasourceParamDTO(); |
||||
postgreSqlDatasourceParamDTO.setUserName("root"); |
||||
postgreSqlDatasourceParamDTO.setPassword("123456"); |
||||
postgreSqlDatasourceParamDTO.setHost("localhost"); |
||||
postgreSqlDatasourceParamDTO.setPort(3308); |
||||
postgreSqlDatasourceParamDTO.setDatabase("default"); |
||||
|
||||
PostgreSqlConnectionParam connectionParams = (PostgreSqlConnectionParam) postgreSqlDatasourceProcessor |
||||
.createConnectionParams(postgreSqlDatasourceParamDTO); |
||||
Assert.assertEquals("jdbc:postgresql://localhost:3308", connectionParams.getAddress()); |
||||
Assert.assertEquals("jdbc:postgresql://localhost:3308/default", connectionParams.getJdbcUrl()); |
||||
Assert.assertEquals("root", connectionParams.getUser()); |
||||
} |
||||
|
||||
@Test |
||||
public void testCreateConnectionParams2() { |
||||
String connectionJson = "{\"user\":\"root\",\"password\":\"123456\",\"address\":\"jdbc:postgresql://localhost:3308\"" |
||||
+ ",\"database\":\"default\",\"jdbcUrl\":\"jdbc:postgresql://localhost:3308/default\"}"; |
||||
PostgreSqlConnectionParam connectionParams = (PostgreSqlConnectionParam) postgreSqlDatasourceProcessor |
||||
.createConnectionParams(connectionJson); |
||||
Assert.assertNotNull(connectionParams); |
||||
Assert.assertEquals("root", connectionParams.getUser()); |
||||
} |
||||
|
||||
@Test |
||||
public void testGetDatasourceDriver() { |
||||
Assert.assertEquals(Constants.ORG_POSTGRESQL_DRIVER, postgreSqlDatasourceProcessor.getDatasourceDriver()); |
||||
} |
||||
|
||||
@Test |
||||
public void testGetJdbcUrl() { |
||||
PostgreSqlConnectionParam postgreSqlConnectionParam = new PostgreSqlConnectionParam(); |
||||
postgreSqlConnectionParam.setJdbcUrl("jdbc:postgresql://localhost:3308/default"); |
||||
postgreSqlConnectionParam.setOther("other"); |
||||
|
||||
String jdbcUrl = postgreSqlDatasourceProcessor.getJdbcUrl(postgreSqlConnectionParam); |
||||
Assert.assertEquals("jdbc:postgresql://localhost:3308/default?other", jdbcUrl); |
||||
|
||||
} |
||||
|
||||
@Test |
||||
public void testGetDbType() { |
||||
Assert.assertEquals(DbType.POSTGRESQL, postgreSqlDatasourceProcessor.getDbType()); |
||||
} |
||||
} |
||||
@ -0,0 +1,81 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.datasource.presto; |
||||
|
||||
import org.apache.dolphinscheduler.common.Constants; |
||||
import org.apache.dolphinscheduler.common.enums.DbType; |
||||
|
||||
import java.sql.DriverManager; |
||||
|
||||
import org.junit.Assert; |
||||
import org.junit.Test; |
||||
import org.junit.runner.RunWith; |
||||
import org.powermock.core.classloader.annotations.PrepareForTest; |
||||
import org.powermock.modules.junit4.PowerMockRunner; |
||||
|
||||
@RunWith(PowerMockRunner.class) |
||||
@PrepareForTest({Class.class, DriverManager.class}) |
||||
public class PrestoDatasourceProcessorTest { |
||||
|
||||
private PrestoDatasourceProcessor prestoDatasourceProcessor = new PrestoDatasourceProcessor(); |
||||
|
||||
@Test |
||||
public void testCreateConnectionParams() { |
||||
PrestoDatasourceParamDTO prestoDatasourceParamDTO = new PrestoDatasourceParamDTO(); |
||||
prestoDatasourceParamDTO.setHost("localhost"); |
||||
prestoDatasourceParamDTO.setPort(1234); |
||||
prestoDatasourceParamDTO.setDatabase("default"); |
||||
prestoDatasourceParamDTO.setUserName("root"); |
||||
prestoDatasourceParamDTO.setPassword("123456"); |
||||
|
||||
PrestoConnectionParam connectionParams = (PrestoConnectionParam) prestoDatasourceProcessor |
||||
.createConnectionParams(prestoDatasourceParamDTO); |
||||
Assert.assertEquals("jdbc:presto://localhost:1234", connectionParams.getAddress()); |
||||
Assert.assertEquals("jdbc:presto://localhost:1234/default", connectionParams.getJdbcUrl()); |
||||
} |
||||
|
||||
@Test |
||||
public void testCreateConnectionParams2() { |
||||
String connectionJson = "{\"user\":\"root\",\"password\":\"123456\",\"address\":\"jdbc:presto://localhost:1234\"" |
||||
+ ",\"database\":\"default\",\"jdbcUrl\":\"jdbc:presto://localhost:1234/default\"}"; |
||||
PrestoConnectionParam connectionParams = (PrestoConnectionParam) prestoDatasourceProcessor |
||||
.createConnectionParams(connectionJson); |
||||
Assert.assertNotNull(connectionParams); |
||||
Assert.assertEquals("root", connectionParams.getUser()); |
||||
} |
||||
|
||||
@Test |
||||
public void testGetDatasourceDriver() { |
||||
Assert.assertEquals(Constants.COM_PRESTO_JDBC_DRIVER, prestoDatasourceProcessor.getDatasourceDriver()); |
||||
} |
||||
|
||||
@Test |
||||
public void testGetJdbcUrl() { |
||||
PrestoConnectionParam prestoConnectionParam = new PrestoConnectionParam(); |
||||
prestoConnectionParam.setJdbcUrl("jdbc:postgresql://localhost:1234/default"); |
||||
prestoConnectionParam.setOther("other"); |
||||
Assert.assertEquals("jdbc:postgresql://localhost:1234/default?other", |
||||
prestoDatasourceProcessor.getJdbcUrl(prestoConnectionParam)); |
||||
|
||||
} |
||||
|
||||
@Test |
||||
public void testGetDbType() { |
||||
Assert.assertEquals(DbType.PRESTO, prestoDatasourceProcessor.getDbType()); |
||||
} |
||||
} |
||||
@ -0,0 +1,80 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.datasource.spark; |
||||
|
||||
import org.apache.dolphinscheduler.common.Constants; |
||||
import org.apache.dolphinscheduler.common.enums.DbType; |
||||
|
||||
import java.sql.DriverManager; |
||||
|
||||
import org.junit.Assert; |
||||
import org.junit.Test; |
||||
import org.junit.runner.RunWith; |
||||
import org.powermock.core.classloader.annotations.PrepareForTest; |
||||
import org.powermock.modules.junit4.PowerMockRunner; |
||||
|
||||
@RunWith(PowerMockRunner.class) |
||||
@PrepareForTest({Class.class, DriverManager.class}) |
||||
public class SparkDatasourceProcessorTest { |
||||
|
||||
private SparkDatasourceProcessor sparkDatasourceProcessor = new SparkDatasourceProcessor(); |
||||
|
||||
@Test |
||||
public void testCreateConnectionParams() { |
||||
SparkDatasourceParamDTO sparkDatasourceParamDTO = new SparkDatasourceParamDTO(); |
||||
sparkDatasourceParamDTO.setUserName("root"); |
||||
sparkDatasourceParamDTO.setPassword("12345"); |
||||
sparkDatasourceParamDTO.setHost("localhost1,localhost2"); |
||||
sparkDatasourceParamDTO.setPort(1234); |
||||
sparkDatasourceParamDTO.setDatabase("default"); |
||||
|
||||
SparkConnectionParam connectionParams = (SparkConnectionParam) sparkDatasourceProcessor |
||||
.createConnectionParams(sparkDatasourceParamDTO); |
||||
Assert.assertEquals("jdbc:hive2://localhost1:1234,localhost2:1234", connectionParams.getAddress()); |
||||
Assert.assertEquals("jdbc:hive2://localhost1:1234,localhost2:1234/default", connectionParams.getJdbcUrl()); |
||||
} |
||||
|
||||
@Test |
||||
public void testCreateConnectionParams2() { |
||||
String connectionJson = "{\"user\":\"root\",\"password\":\"12345\",\"address\":\"jdbc:hive2://localhost1:1234,localhost2:1234\"" |
||||
+ ",\"database\":\"default\",\"jdbcUrl\":\"jdbc:hive2://localhost1:1234,localhost2:1234/default\"}"; |
||||
SparkConnectionParam connectionParams = (SparkConnectionParam) sparkDatasourceProcessor |
||||
.createConnectionParams(connectionJson); |
||||
Assert.assertNotNull(connectionParams); |
||||
Assert.assertEquals("root", connectionParams.getUser()); |
||||
} |
||||
|
||||
@Test |
||||
public void testGetDatasourceDriver() { |
||||
Assert.assertEquals(Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER, sparkDatasourceProcessor.getDatasourceDriver()); |
||||
} |
||||
|
||||
@Test |
||||
public void testGetJdbcUrl() { |
||||
SparkConnectionParam sparkConnectionParam = new SparkConnectionParam(); |
||||
sparkConnectionParam.setJdbcUrl("jdbc:hive2://localhost1:1234,localhost2:1234/default"); |
||||
sparkConnectionParam.setOther("other"); |
||||
Assert.assertEquals("jdbc:hive2://localhost1:1234,localhost2:1234/default;other", |
||||
sparkDatasourceProcessor.getJdbcUrl(sparkConnectionParam)); |
||||
} |
||||
|
||||
@Test |
||||
public void testGetDbType() { |
||||
Assert.assertEquals(DbType.SPARK, sparkDatasourceProcessor.getDbType()); |
||||
} |
||||
} |
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue