Browse Source
* Task plug-in optimization, task-related classes are migrated to the task plugin module * remove tasktype enum class. * fix code style. * fix code style * fix code style * fix ProcessServiceTest.testUpdateResourceInfo test class * fix code style * fix test task plugin manager check parameters null * fix test ut * update aws license * change TaskTimeoutStrategy class3.0.0/version-upgrade
Kerwin
2 years ago
committed by
GitHub
473 changed files with 3714 additions and 7172 deletions
@ -1,28 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.enums; |
||||
|
||||
/** |
||||
* parameter of stored procedure |
||||
*/ |
||||
public enum Direct { |
||||
/** |
||||
* 0 in; 1 out; |
||||
*/ |
||||
IN,OUT |
||||
} |
||||
@ -1,84 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.enums; |
||||
|
||||
import com.baomidou.mybatisplus.annotation.EnumValue; |
||||
|
||||
/** |
||||
* task node type |
||||
*/ |
||||
public enum TaskType { |
||||
/** |
||||
* 0 SHELL |
||||
* 1 SQL |
||||
* 2 SUB_PROCESS |
||||
* 3 PROCEDURE |
||||
* 4 MR |
||||
* 5 SPARK |
||||
* 6 PYTHON |
||||
* 7 DEPENDENT |
||||
* 8 FLINK |
||||
* 9 HTTP |
||||
* 10 DATAX |
||||
* 11 CONDITIONS |
||||
* 12 SQOOP |
||||
* 13 SEATUNNEL |
||||
* 14 SWITCH |
||||
* 15 PIGEON |
||||
* 16 DATA_QUALITY |
||||
* 17 EMR |
||||
* 18 BLOCKING |
||||
*/ |
||||
SHELL(0, "SHELL"), |
||||
SQL(1, "SQL"), |
||||
SUB_PROCESS(2, "SUB_PROCESS"), |
||||
PROCEDURE(3, "PROCEDURE"), |
||||
MR(4, "MR"), |
||||
SPARK(5, "SPARK"), |
||||
PYTHON(6, "PYTHON"), |
||||
DEPENDENT(7, "DEPENDENT"), |
||||
FLINK(8, "FLINK"), |
||||
HTTP(9, "HTTP"), |
||||
DATAX(10, "DATAX"), |
||||
CONDITIONS(11, "CONDITIONS"), |
||||
SQOOP(12, "SQOOP"), |
||||
SEATUNNEL(13, "SEATUNNEL"), |
||||
SWITCH(14, "SWITCH"), |
||||
PIGEON(15, "PIGEON"), |
||||
DATA_QUALITY(16, "DATA_QUALITY"), |
||||
EMR(17, "EMR"), |
||||
BLOCKING(18, "BLOCKING"); |
||||
; |
||||
|
||||
TaskType(int code, String desc) { |
||||
this.code = code; |
||||
this.desc = desc; |
||||
} |
||||
|
||||
@EnumValue |
||||
private final int code; |
||||
private final String desc; |
||||
|
||||
public int getCode() { |
||||
return code; |
||||
} |
||||
|
||||
public String getDesc() { |
||||
return desc; |
||||
} |
||||
} |
||||
@ -1,140 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.process; |
||||
|
||||
import org.apache.dolphinscheduler.common.enums.DataType; |
||||
import org.apache.dolphinscheduler.common.enums.Direct; |
||||
|
||||
import java.io.Serializable; |
||||
import java.util.Objects; |
||||
|
||||
public class Property implements Serializable { |
||||
/** |
||||
* key |
||||
*/ |
||||
private String prop; |
||||
|
||||
/** |
||||
* input/output |
||||
*/ |
||||
private Direct direct; |
||||
|
||||
/** |
||||
* data type |
||||
*/ |
||||
private DataType type; |
||||
|
||||
/** |
||||
* value |
||||
*/ |
||||
private String value; |
||||
|
||||
public Property() { |
||||
} |
||||
|
||||
public Property(String prop,Direct direct,DataType type,String value) { |
||||
this.prop = prop; |
||||
this.direct = direct; |
||||
this.type = type; |
||||
this.value = value; |
||||
} |
||||
|
||||
/** |
||||
* getter method |
||||
* |
||||
* @return the prop |
||||
* @see Property#prop |
||||
*/ |
||||
public String getProp() { |
||||
return prop; |
||||
} |
||||
|
||||
/** |
||||
* setter method |
||||
* |
||||
* @param prop the prop to set |
||||
* @see Property#prop |
||||
*/ |
||||
public void setProp(String prop) { |
||||
this.prop = prop; |
||||
} |
||||
|
||||
/** |
||||
* getter method |
||||
* |
||||
* @return the value |
||||
* @see Property#value |
||||
*/ |
||||
public String getValue() { |
||||
return value; |
||||
} |
||||
|
||||
/** |
||||
* setter method |
||||
* |
||||
* @param value the value to set |
||||
* @see Property#value |
||||
*/ |
||||
public void setValue(String value) { |
||||
this.value = value; |
||||
} |
||||
|
||||
public Direct getDirect() { |
||||
return direct; |
||||
} |
||||
|
||||
public void setDirect(Direct direct) { |
||||
this.direct = direct; |
||||
} |
||||
|
||||
public DataType getType() { |
||||
return type; |
||||
} |
||||
|
||||
public void setType(DataType type) { |
||||
this.type = type; |
||||
} |
||||
|
||||
@Override |
||||
public boolean equals(Object o) { |
||||
if (this == o) { |
||||
return true; |
||||
} |
||||
if (o == null || getClass() != o.getClass()) { |
||||
return false; |
||||
} |
||||
Property property = (Property) o; |
||||
return Objects.equals(prop, property.prop) |
||||
&& Objects.equals(value, property.value); |
||||
} |
||||
|
||||
@Override |
||||
public int hashCode() { |
||||
return Objects.hash(prop, value); |
||||
} |
||||
|
||||
@Override |
||||
public String toString() { |
||||
return "Property{" |
||||
+ "prop='" + prop + '\'' |
||||
+ ", direct=" + direct |
||||
+ ", type=" + type |
||||
+ ", value='" + value + '\'' |
||||
+ '}'; |
||||
} |
||||
} |
||||
@ -1,63 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.process; |
||||
|
||||
/** |
||||
* resource info |
||||
*/ |
||||
public class ResourceInfo { |
||||
/** |
||||
* res id of the resource that was uploaded |
||||
*/ |
||||
private int id; |
||||
|
||||
private String res; |
||||
|
||||
/** |
||||
* full name of the resource that was uploaded |
||||
*/ |
||||
private String resourceName; |
||||
|
||||
public ResourceInfo() { |
||||
// do nothing, void constructor
|
||||
} |
||||
|
||||
public int getId() { |
||||
return id; |
||||
} |
||||
|
||||
public void setId(int id) { |
||||
this.id = id; |
||||
} |
||||
|
||||
public String getRes() { |
||||
return res; |
||||
} |
||||
|
||||
public void setRes(String res) { |
||||
this.res = res; |
||||
} |
||||
|
||||
public String getResourceName() { |
||||
return resourceName; |
||||
} |
||||
|
||||
public void setResourceName(String resourceName) { |
||||
this.resourceName = resourceName; |
||||
} |
||||
} |
||||
@ -1,181 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.task; |
||||
|
||||
import org.apache.dolphinscheduler.common.enums.Direct; |
||||
import org.apache.dolphinscheduler.common.process.Property; |
||||
import org.apache.dolphinscheduler.common.process.ResourceInfo; |
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils; |
||||
|
||||
import org.apache.commons.collections.CollectionUtils; |
||||
import org.apache.commons.lang.StringUtils; |
||||
|
||||
import java.util.ArrayList; |
||||
import java.util.HashMap; |
||||
import java.util.Iterator; |
||||
import java.util.LinkedHashMap; |
||||
import java.util.List; |
||||
import java.util.Map; |
||||
|
||||
import com.fasterxml.jackson.core.type.TypeReference; |
||||
import com.fasterxml.jackson.databind.JsonNode; |
||||
import com.fasterxml.jackson.databind.node.ArrayNode; |
||||
|
||||
/** |
||||
* job params related class
|
||||
*/ |
||||
public abstract class AbstractParameters implements IParameters { |
||||
|
||||
@Override |
||||
public abstract boolean checkParameters(); |
||||
|
||||
@Override |
||||
public abstract List<ResourceInfo> getResourceFilesList(); |
||||
|
||||
/** |
||||
* local parameters |
||||
*/ |
||||
public List<Property> localParams; |
||||
|
||||
/** |
||||
* var pool |
||||
*/ |
||||
public List<Property> varPool; |
||||
|
||||
/** |
||||
* get local parameters list |
||||
* |
||||
* @return Property list |
||||
*/ |
||||
public List<Property> getLocalParams() { |
||||
return localParams; |
||||
} |
||||
|
||||
public void setLocalParams(List<Property> localParams) { |
||||
this.localParams = localParams; |
||||
} |
||||
|
||||
/** |
||||
* get local parameters map |
||||
* |
||||
* @return parameters map |
||||
*/ |
||||
public Map<String, Property> getLocalParametersMap() { |
||||
if (localParams != null) { |
||||
Map<String, Property> localParametersMaps = new LinkedHashMap<>(); |
||||
|
||||
for (Property property : localParams) { |
||||
localParametersMaps.put(property.getProp(), property); |
||||
} |
||||
return localParametersMaps; |
||||
} |
||||
return null; |
||||
} |
||||
|
||||
/** |
||||
* get varPool map |
||||
* |
||||
* @return parameters map |
||||
*/ |
||||
public Map<String, Property> getVarPoolMap() { |
||||
if (varPool != null) { |
||||
Map<String, Property> varPoolMap = new LinkedHashMap<>(); |
||||
for (Property property : varPool) { |
||||
varPoolMap.put(property.getProp(), property); |
||||
} |
||||
return varPoolMap; |
||||
} |
||||
return null; |
||||
} |
||||
|
||||
public List<Property> getVarPool() { |
||||
return varPool; |
||||
} |
||||
|
||||
public void setVarPool(String varPool) { |
||||
if (StringUtils.isEmpty(varPool)) { |
||||
this.varPool = new ArrayList<>(); |
||||
} else { |
||||
this.varPool = JSONUtils.toList(varPool, Property.class); |
||||
} |
||||
} |
||||
|
||||
public void dealOutParam(String result) { |
||||
if (CollectionUtils.isEmpty(localParams)) { |
||||
return; |
||||
} |
||||
List<Property> outProperty = getOutProperty(localParams); |
||||
if (CollectionUtils.isEmpty(outProperty)) { |
||||
return; |
||||
} |
||||
if (StringUtils.isEmpty(result)) { |
||||
varPool.addAll(outProperty); |
||||
return; |
||||
} |
||||
Map<String, String> taskResult = getMapByString(result); |
||||
if (taskResult == null || taskResult.size() == 0) { |
||||
return; |
||||
} |
||||
for (Property info : outProperty) { |
||||
info.setValue(taskResult.get(info.getProp())); |
||||
varPool.add(info); |
||||
} |
||||
} |
||||
|
||||
public List<Property> getOutProperty(List<Property> params) { |
||||
if (CollectionUtils.isEmpty(params)) { |
||||
return new ArrayList<>(); |
||||
} |
||||
List<Property> result = new ArrayList<>(); |
||||
for (Property info : params) { |
||||
if (info.getDirect() == Direct.OUT) { |
||||
result.add(info); |
||||
} |
||||
} |
||||
return result; |
||||
} |
||||
|
||||
public List<Map<String, String>> getListMapByString(String json) { |
||||
List<Map<String, String>> allParams = new ArrayList<>(); |
||||
ArrayNode paramsByJson = JSONUtils.parseArray(json); |
||||
Iterator<JsonNode> listIterator = paramsByJson.iterator(); |
||||
while (listIterator.hasNext()) { |
||||
Map<String, String> param = JSONUtils.parseObject(listIterator.next().toString(), new TypeReference<Map<String, String>>() {}); |
||||
allParams.add(param); |
||||
} |
||||
return allParams; |
||||
} |
||||
|
||||
/** |
||||
* shell's result format is key=value$VarPool$key=value$VarPool$ |
||||
* @param result |
||||
* @return |
||||
*/ |
||||
public static Map<String, String> getMapByString(String result) { |
||||
String[] formatResult = result.split("\\$VarPool\\$"); |
||||
Map<String, String> format = new HashMap<>(); |
||||
for (String info : formatResult) { |
||||
if (!StringUtils.isEmpty(info) && info.contains("=")) { |
||||
String[] keyValue = info.split("="); |
||||
format.put(keyValue[0], keyValue[1]); |
||||
} |
||||
} |
||||
return format; |
||||
} |
||||
|
||||
} |
||||
@ -1,252 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.task.datax; |
||||
|
||||
import org.apache.dolphinscheduler.common.enums.Flag; |
||||
import org.apache.dolphinscheduler.common.process.ResourceInfo; |
||||
import org.apache.dolphinscheduler.common.task.AbstractParameters; |
||||
|
||||
import org.apache.commons.lang.StringUtils; |
||||
|
||||
import java.util.ArrayList; |
||||
import java.util.List; |
||||
|
||||
/** |
||||
* DataX parameter |
||||
*/ |
||||
public class DataxParameters extends AbstractParameters { |
||||
|
||||
/** |
||||
* if custom json config，eg 0, 1 |
||||
*/ |
||||
private int customConfig; |
||||
|
||||
/** |
||||
* if customConfig eq 1 ,then json is usable |
||||
*/ |
||||
private String json; |
||||
|
||||
/** |
||||
* data source type，eg MYSQL, POSTGRES ... |
||||
*/ |
||||
private String dsType; |
||||
|
||||
/** |
||||
* datasource id |
||||
*/ |
||||
private int dataSource; |
||||
|
||||
/** |
||||
* data target type，eg MYSQL, POSTGRES ... |
||||
*/ |
||||
private String dtType; |
||||
|
||||
/** |
||||
* datatarget id |
||||
*/ |
||||
private int dataTarget; |
||||
|
||||
/** |
||||
* sql |
||||
*/ |
||||
private String sql; |
||||
|
||||
/** |
||||
* target table |
||||
*/ |
||||
private String targetTable; |
||||
|
||||
/** |
||||
* Pre Statements |
||||
*/ |
||||
private List<String> preStatements; |
||||
|
||||
/** |
||||
* Post Statements |
||||
*/ |
||||
private List<String> postStatements; |
||||
|
||||
/** |
||||
* speed byte num |
||||
*/ |
||||
private int jobSpeedByte; |
||||
|
||||
/** |
||||
* speed record count |
||||
*/ |
||||
private int jobSpeedRecord; |
||||
|
||||
/** |
||||
* Xms memory |
||||
*/ |
||||
private int xms; |
||||
|
||||
/** |
||||
* Xmx memory |
||||
*/ |
||||
private int xmx; |
||||
|
||||
public int getCustomConfig() { |
||||
return customConfig; |
||||
} |
||||
|
||||
public void setCustomConfig(int customConfig) { |
||||
this.customConfig = customConfig; |
||||
} |
||||
|
||||
public String getJson() { |
||||
return json; |
||||
} |
||||
|
||||
public void setJson(String json) { |
||||
this.json = json; |
||||
} |
||||
|
||||
public String getDsType() { |
||||
return dsType; |
||||
} |
||||
|
||||
public void setDsType(String dsType) { |
||||
this.dsType = dsType; |
||||
} |
||||
|
||||
public int getDataSource() { |
||||
return dataSource; |
||||
} |
||||
|
||||
public void setDataSource(int dataSource) { |
||||
this.dataSource = dataSource; |
||||
} |
||||
|
||||
public String getDtType() { |
||||
return dtType; |
||||
} |
||||
|
||||
public void setDtType(String dtType) { |
||||
this.dtType = dtType; |
||||
} |
||||
|
||||
public int getDataTarget() { |
||||
return dataTarget; |
||||
} |
||||
|
||||
public void setDataTarget(int dataTarget) { |
||||
this.dataTarget = dataTarget; |
||||
} |
||||
|
||||
public String getSql() { |
||||
return sql; |
||||
} |
||||
|
||||
public void setSql(String sql) { |
||||
this.sql = sql; |
||||
} |
||||
|
||||
public String getTargetTable() { |
||||
return targetTable; |
||||
} |
||||
|
||||
public void setTargetTable(String targetTable) { |
||||
this.targetTable = targetTable; |
||||
} |
||||
|
||||
public List<String> getPreStatements() { |
||||
return preStatements; |
||||
} |
||||
|
||||
public void setPreStatements(List<String> preStatements) { |
||||
this.preStatements = preStatements; |
||||
} |
||||
|
||||
public List<String> getPostStatements() { |
||||
return postStatements; |
||||
} |
||||
|
||||
public void setPostStatements(List<String> postStatements) { |
||||
this.postStatements = postStatements; |
||||
} |
||||
|
||||
public int getJobSpeedByte() { |
||||
return jobSpeedByte; |
||||
} |
||||
|
||||
public void setJobSpeedByte(int jobSpeedByte) { |
||||
this.jobSpeedByte = jobSpeedByte; |
||||
} |
||||
|
||||
public int getJobSpeedRecord() { |
||||
return jobSpeedRecord; |
||||
} |
||||
|
||||
public void setJobSpeedRecord(int jobSpeedRecord) { |
||||
this.jobSpeedRecord = jobSpeedRecord; |
||||
} |
||||
|
||||
public int getXms() { |
||||
return xms; |
||||
} |
||||
|
||||
public void setXms(int xms) { |
||||
this.xms = xms; |
||||
} |
||||
|
||||
public int getXmx() { |
||||
return xmx; |
||||
} |
||||
|
||||
public void setXmx(int xmx) { |
||||
this.xmx = xmx; |
||||
} |
||||
|
||||
@Override |
||||
public boolean checkParameters() { |
||||
if (customConfig == Flag.NO.ordinal()) { |
||||
return dataSource != 0 |
||||
&& dataTarget != 0 |
||||
&& !StringUtils.isEmpty(sql) |
||||
&& !StringUtils.isEmpty(targetTable); |
||||
} else { |
||||
return !StringUtils.isEmpty(json); |
||||
} |
||||
} |
||||
|
||||
@Override |
||||
public List<ResourceInfo> getResourceFilesList() { |
||||
return new ArrayList<>(); |
||||
} |
||||
|
||||
@Override |
||||
public String toString() { |
||||
return "DataxParameters{" |
||||
+ "customConfig=" + customConfig |
||||
+ ", json='" + json + '\'' |
||||
+ ", dsType='" + dsType + '\'' |
||||
+ ", dataSource=" + dataSource |
||||
+ ", dtType='" + dtType + '\'' |
||||
+ ", dataTarget=" + dataTarget |
||||
+ ", sql='" + sql + '\'' |
||||
+ ", targetTable='" + targetTable + '\'' |
||||
+ ", preStatements=" + preStatements |
||||
+ ", postStatements=" + postStatements |
||||
+ ", jobSpeedByte=" + jobSpeedByte |
||||
+ ", jobSpeedRecord=" + jobSpeedRecord |
||||
+ ", xms=" + xms |
||||
+ ", xmx=" + xmx |
||||
+ '}'; |
||||
} |
||||
} |
||||
@ -1,103 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.task.dq; |
||||
|
||||
import org.apache.dolphinscheduler.common.process.ResourceInfo; |
||||
import org.apache.dolphinscheduler.common.task.AbstractParameters; |
||||
import org.apache.dolphinscheduler.common.task.spark.SparkParameters; |
||||
|
||||
import org.apache.commons.collections.MapUtils; |
||||
|
||||
import java.util.ArrayList; |
||||
import java.util.List; |
||||
import java.util.Map; |
||||
|
||||
import org.slf4j.Logger; |
||||
import org.slf4j.LoggerFactory; |
||||
|
||||
/** |
||||
* DataQualityParameters |
||||
*/ |
||||
public class DataQualityParameters extends AbstractParameters { |
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(DataQualityParameters.class); |
||||
|
||||
/** |
||||
* rule id |
||||
*/ |
||||
private int ruleId; |
||||
/** |
||||
* rule input entry value map |
||||
*/ |
||||
private Map<String,String> ruleInputParameter; |
||||
/** |
||||
* spark parameters |
||||
*/ |
||||
private SparkParameters sparkParameters; |
||||
|
||||
public int getRuleId() { |
||||
return ruleId; |
||||
} |
||||
|
||||
public void setRuleId(int ruleId) { |
||||
this.ruleId = ruleId; |
||||
} |
||||
|
||||
public Map<String, String> getRuleInputParameter() { |
||||
return ruleInputParameter; |
||||
} |
||||
|
||||
public void setRuleInputParameter(Map<String, String> ruleInputParameter) { |
||||
this.ruleInputParameter = ruleInputParameter; |
||||
} |
||||
|
||||
/** |
||||
* In this function ,we need more detailed check every parameter, |
||||
* if the parameter is non-conformant will return false |
||||
* @return boolean result |
||||
*/ |
||||
@Override |
||||
public boolean checkParameters() { |
||||
|
||||
if (ruleId == 0) { |
||||
logger.error("rule id is null"); |
||||
return false; |
||||
} |
||||
|
||||
if (MapUtils.isEmpty(ruleInputParameter)) { |
||||
logger.error("rule input parameter is empty"); |
||||
return false; |
||||
} |
||||
|
||||
return sparkParameters != null; |
||||
} |
||||
|
||||
@Override |
||||
public List<ResourceInfo> getResourceFilesList() { |
||||
return new ArrayList<>(); |
||||
} |
||||
|
||||
public SparkParameters getSparkParameters() { |
||||
return sparkParameters; |
||||
} |
||||
|
||||
public void setSparkParameters(SparkParameters sparkParameters) { |
||||
this.sparkParameters = sparkParameters; |
||||
} |
||||
|
||||
} |
||||
@ -1,59 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.task.emr; |
||||
|
||||
import org.apache.dolphinscheduler.common.process.ResourceInfo; |
||||
import org.apache.dolphinscheduler.common.task.AbstractParameters; |
||||
import org.apache.dolphinscheduler.spi.utils.StringUtils; |
||||
|
||||
import java.util.Collections; |
||||
import java.util.List; |
||||
|
||||
public class EmrParameters extends AbstractParameters { |
||||
/** |
||||
* job flow define in json format |
||||
*/ |
||||
private String jobFlowDefineJson; |
||||
|
||||
@Override |
||||
public boolean checkParameters() { |
||||
return StringUtils.isNotEmpty(jobFlowDefineJson); |
||||
} |
||||
|
||||
|
||||
@Override |
||||
public List<ResourceInfo> getResourceFilesList() { |
||||
return Collections.emptyList(); |
||||
} |
||||
|
||||
|
||||
public String getJobFlowDefineJson() { |
||||
return jobFlowDefineJson; |
||||
} |
||||
|
||||
public void setJobFlowDefineJson(String jobFlowDefineJson) { |
||||
this.jobFlowDefineJson = jobFlowDefineJson; |
||||
} |
||||
|
||||
@Override |
||||
public String toString() { |
||||
return "EmrParameters{" + |
||||
"jobFlowDefineJson='" + jobFlowDefineJson + '\'' + |
||||
'}'; |
||||
} |
||||
} |
||||
@ -1,241 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.task.flink; |
||||
|
||||
import org.apache.dolphinscheduler.common.enums.ProgramType; |
||||
import org.apache.dolphinscheduler.common.process.ResourceInfo; |
||||
import org.apache.dolphinscheduler.common.task.AbstractParameters; |
||||
|
||||
import java.util.ArrayList; |
||||
import java.util.List; |
||||
|
||||
/** |
||||
* flink parameters |
||||
*/ |
||||
public class FlinkParameters extends AbstractParameters { |
||||
|
||||
/** |
||||
* major jar |
||||
*/ |
||||
private ResourceInfo mainJar; |
||||
|
||||
/** |
||||
* major class
|
||||
*/ |
||||
private String mainClass; |
||||
|
||||
/** |
||||
* deploy mode yarn-cluster yarn-local |
||||
*/ |
||||
private String deployMode; |
||||
|
||||
/** |
||||
* arguments |
||||
*/ |
||||
private String mainArgs; |
||||
|
||||
/** |
||||
* slot count |
||||
*/ |
||||
private int slot; |
||||
|
||||
/** |
||||
* parallelism |
||||
*/ |
||||
private int parallelism; |
||||
|
||||
/** |
||||
* yarn application name |
||||
*/ |
||||
private String appName; |
||||
|
||||
/** |
||||
* taskManager count |
||||
*/ |
||||
private int taskManager; |
||||
|
||||
/** |
||||
* job manager memory |
||||
*/ |
||||
private String jobManagerMemory; |
||||
|
||||
/** |
||||
* task manager memory |
||||
*/ |
||||
private String taskManagerMemory; |
||||
|
||||
/** |
||||
* resource list |
||||
*/ |
||||
private List<ResourceInfo> resourceList = new ArrayList<>(); |
||||
|
||||
/** |
||||
* The YARN queue to submit to |
||||
*/ |
||||
private String queue; |
||||
|
||||
/** |
||||
* other arguments |
||||
*/ |
||||
private String others; |
||||
|
||||
/** |
||||
* flink version |
||||
*/ |
||||
private String flinkVersion; |
||||
|
||||
/** |
||||
* program type |
||||
* 0 JAVA,1 SCALA,2 PYTHON |
||||
*/ |
||||
private ProgramType programType; |
||||
|
||||
public ResourceInfo getMainJar() { |
||||
return mainJar; |
||||
} |
||||
|
||||
public void setMainJar(ResourceInfo mainJar) { |
||||
this.mainJar = mainJar; |
||||
} |
||||
|
||||
public String getMainClass() { |
||||
return mainClass; |
||||
} |
||||
|
||||
public void setMainClass(String mainClass) { |
||||
this.mainClass = mainClass; |
||||
} |
||||
|
||||
public String getDeployMode() { |
||||
return deployMode; |
||||
} |
||||
|
||||
public void setDeployMode(String deployMode) { |
||||
this.deployMode = deployMode; |
||||
} |
||||
|
||||
public String getMainArgs() { |
||||
return mainArgs; |
||||
} |
||||
|
||||
public void setMainArgs(String mainArgs) { |
||||
this.mainArgs = mainArgs; |
||||
} |
||||
|
||||
public int getSlot() { |
||||
return slot; |
||||
} |
||||
|
||||
public void setSlot(int slot) { |
||||
this.slot = slot; |
||||
} |
||||
|
||||
public int getParallelism() { |
||||
return parallelism; |
||||
} |
||||
|
||||
public void setParallelism(int parallelism) { |
||||
this.parallelism = parallelism; |
||||
} |
||||
|
||||
public String getAppName() { |
||||
return appName; |
||||
} |
||||
|
||||
public void setAppName(String appName) { |
||||
this.appName = appName; |
||||
} |
||||
|
||||
public int getTaskManager() { |
||||
return taskManager; |
||||
} |
||||
|
||||
public void setTaskManager(int taskManager) { |
||||
this.taskManager = taskManager; |
||||
} |
||||
|
||||
public String getJobManagerMemory() { |
||||
return jobManagerMemory; |
||||
} |
||||
|
||||
public void setJobManagerMemory(String jobManagerMemory) { |
||||
this.jobManagerMemory = jobManagerMemory; |
||||
} |
||||
|
||||
public String getTaskManagerMemory() { |
||||
return taskManagerMemory; |
||||
} |
||||
|
||||
public void setTaskManagerMemory(String taskManagerMemory) { |
||||
this.taskManagerMemory = taskManagerMemory; |
||||
} |
||||
|
||||
public String getQueue() { |
||||
return queue; |
||||
} |
||||
|
||||
public void setQueue(String queue) { |
||||
this.queue = queue; |
||||
} |
||||
|
||||
public List<ResourceInfo> getResourceList() { |
||||
return resourceList; |
||||
} |
||||
|
||||
public void setResourceList(List<ResourceInfo> resourceList) { |
||||
this.resourceList = resourceList; |
||||
} |
||||
|
||||
public String getOthers() { |
||||
return others; |
||||
} |
||||
|
||||
public void setOthers(String others) { |
||||
this.others = others; |
||||
} |
||||
|
||||
public ProgramType getProgramType() { |
||||
return programType; |
||||
} |
||||
|
||||
public void setProgramType(ProgramType programType) { |
||||
this.programType = programType; |
||||
} |
||||
|
||||
public String getFlinkVersion() { |
||||
return flinkVersion; |
||||
} |
||||
|
||||
public void setFlinkVersion(String flinkVersion) { |
||||
this.flinkVersion = flinkVersion; |
||||
} |
||||
|
||||
@Override |
||||
public boolean checkParameters() { |
||||
return mainJar != null && programType != null; |
||||
} |
||||
|
||||
@Override |
||||
public List<ResourceInfo> getResourceFilesList() { |
||||
if (mainJar != null && !resourceList.contains(mainJar)) { |
||||
resourceList.add(mainJar); |
||||
} |
||||
return resourceList; |
||||
} |
||||
|
||||
} |
||||
@ -1,138 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.task.http; |
||||
|
||||
import org.apache.dolphinscheduler.common.enums.HttpCheckCondition; |
||||
import org.apache.dolphinscheduler.common.enums.HttpMethod; |
||||
import org.apache.dolphinscheduler.common.process.HttpProperty; |
||||
import org.apache.dolphinscheduler.common.process.ResourceInfo; |
||||
import org.apache.dolphinscheduler.common.task.AbstractParameters; |
||||
|
||||
import org.apache.commons.lang.StringUtils; |
||||
|
||||
import java.util.ArrayList; |
||||
import java.util.List; |
||||
|
||||
/** |
||||
* http parameter |
||||
*/ |
||||
public class HttpParameters extends AbstractParameters { |
||||
/** |
||||
* url |
||||
*/ |
||||
private String url; |
||||
|
||||
/** |
||||
* httpMethod |
||||
*/ |
||||
private HttpMethod httpMethod; |
||||
|
||||
/** |
||||
* http params |
||||
*/ |
||||
private List<HttpProperty> httpParams; |
||||
|
||||
/** |
||||
* httpCheckCondition |
||||
*/ |
||||
private HttpCheckCondition httpCheckCondition = HttpCheckCondition.STATUS_CODE_DEFAULT; |
||||
|
||||
/** |
||||
* condition |
||||
*/ |
||||
private String condition; |
||||
|
||||
|
||||
/** |
||||
* Connect Timeout |
||||
* Unit: ms |
||||
*/ |
||||
private int connectTimeout; |
||||
|
||||
/** |
||||
* Socket Timeout |
||||
* Unit: ms |
||||
*/ |
||||
private int socketTimeout; |
||||
|
||||
@Override |
||||
public boolean checkParameters() { |
||||
return !StringUtils.isEmpty(url); |
||||
} |
||||
|
||||
@Override |
||||
public List<ResourceInfo> getResourceFilesList() { |
||||
return new ArrayList<>(); |
||||
} |
||||
|
||||
public String getUrl() { |
||||
return url; |
||||
} |
||||
|
||||
public void setUrl(String url) { |
||||
this.url = url; |
||||
} |
||||
|
||||
public HttpMethod getHttpMethod() { |
||||
return httpMethod; |
||||
} |
||||
|
||||
public void setHttpMethod(HttpMethod httpMethod) { |
||||
this.httpMethod = httpMethod; |
||||
} |
||||
|
||||
public List<HttpProperty> getHttpParams() { |
||||
return httpParams; |
||||
} |
||||
|
||||
public void setHttpParams(List<HttpProperty> httpParams) { |
||||
this.httpParams = httpParams; |
||||
} |
||||
|
||||
public HttpCheckCondition getHttpCheckCondition() { |
||||
return httpCheckCondition; |
||||
} |
||||
|
||||
public void setHttpCheckCondition(HttpCheckCondition httpCheckCondition) { |
||||
this.httpCheckCondition = httpCheckCondition; |
||||
} |
||||
|
||||
public String getCondition() { |
||||
return condition; |
||||
} |
||||
|
||||
public void setCondition(String condition) { |
||||
this.condition = condition; |
||||
} |
||||
|
||||
public int getConnectTimeout() { |
||||
return connectTimeout; |
||||
} |
||||
|
||||
public void setConnectTimeout(int connectTimeout) { |
||||
this.connectTimeout = connectTimeout; |
||||
} |
||||
|
||||
public int getSocketTimeout() { |
||||
return socketTimeout; |
||||
} |
||||
|
||||
public void setSocketTimeout(int socketTimeout) { |
||||
this.socketTimeout = socketTimeout; |
||||
} |
||||
} |
||||
@ -1,160 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.task.mr; |
||||
|
||||
import org.apache.dolphinscheduler.common.enums.ProgramType; |
||||
import org.apache.dolphinscheduler.common.process.ResourceInfo; |
||||
import org.apache.dolphinscheduler.common.task.AbstractParameters; |
||||
|
||||
import java.util.ArrayList; |
||||
import java.util.List; |
||||
|
||||
/** |
||||
* mapreduce parameters |
||||
*/ |
||||
public class MapReduceParameters extends AbstractParameters { |
||||
|
||||
/** |
||||
* major jar |
||||
*/ |
||||
private ResourceInfo mainJar; |
||||
|
||||
/** |
||||
* major class
|
||||
*/ |
||||
private String mainClass; |
||||
|
||||
/** |
||||
* arguments |
||||
*/ |
||||
private String mainArgs; |
||||
|
||||
/** |
||||
* other arguments |
||||
*/ |
||||
private String others; |
||||
|
||||
/** |
||||
* app name |
||||
*/ |
||||
private String appName; |
||||
|
||||
/** |
||||
* queue |
||||
*/ |
||||
private String queue; |
||||
|
||||
/** |
||||
* resource list |
||||
*/ |
||||
private List<ResourceInfo> resourceList = new ArrayList<>(); |
||||
|
||||
/** |
||||
* program type |
||||
* 0 JAVA,1 SCALA,2 PYTHON |
||||
*/ |
||||
private ProgramType programType; |
||||
|
||||
public String getMainClass() { |
||||
return mainClass; |
||||
} |
||||
|
||||
public void setMainClass(String mainClass) { |
||||
this.mainClass = mainClass; |
||||
} |
||||
|
||||
public String getMainArgs() { |
||||
return mainArgs; |
||||
} |
||||
|
||||
public void setMainArgs(String mainArgs) { |
||||
this.mainArgs = mainArgs; |
||||
} |
||||
|
||||
public String getOthers() { |
||||
return others; |
||||
} |
||||
|
||||
public void setOthers(String others) { |
||||
this.others = others; |
||||
} |
||||
|
||||
public String getAppName() { |
||||
return appName; |
||||
} |
||||
|
||||
public void setAppName(String appName) { |
||||
this.appName = appName; |
||||
} |
||||
|
||||
public String getQueue() { |
||||
return queue; |
||||
} |
||||
|
||||
public void setQueue(String queue) { |
||||
this.queue = queue; |
||||
} |
||||
|
||||
public List<ResourceInfo> getResourceList() { |
||||
return this.resourceList; |
||||
} |
||||
|
||||
public void setResourceList(List<ResourceInfo> resourceList) { |
||||
this.resourceList = resourceList; |
||||
} |
||||
|
||||
public void setMainJar(ResourceInfo mainJar) { |
||||
this.mainJar = mainJar; |
||||
} |
||||
|
||||
public ResourceInfo getMainJar() { |
||||
return mainJar; |
||||
} |
||||
|
||||
public ProgramType getProgramType() { |
||||
return programType; |
||||
} |
||||
|
||||
public void setProgramType(ProgramType programType) { |
||||
this.programType = programType; |
||||
} |
||||
|
||||
@Override |
||||
public boolean checkParameters() { |
||||
return this.mainJar != null && this.programType != null; |
||||
} |
||||
|
||||
@Override |
||||
public List<ResourceInfo> getResourceFilesList() { |
||||
if (mainJar != null && !resourceList.contains(mainJar)) { |
||||
resourceList.add(mainJar); |
||||
} |
||||
|
||||
return resourceList; |
||||
} |
||||
|
||||
@Override |
||||
public String toString() { |
||||
return "mainJar= " + mainJar |
||||
+ "mainClass=" + mainClass |
||||
+ "mainArgs=" + mainArgs |
||||
+ "queue=" + queue |
||||
+ "other mainArgs=" + others |
||||
; |
||||
} |
||||
} |
||||
@ -1,90 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.task.procedure; |
||||
|
||||
import org.apache.dolphinscheduler.common.process.ResourceInfo; |
||||
import org.apache.dolphinscheduler.common.task.AbstractParameters; |
||||
|
||||
import org.apache.commons.lang.StringUtils; |
||||
|
||||
import java.util.ArrayList; |
||||
import java.util.List; |
||||
|
||||
/** |
||||
* procedure parameter |
||||
*/ |
||||
public class ProcedureParameters extends AbstractParameters { |
||||
|
||||
/** |
||||
* data source type，eg MYSQL, POSTGRES, HIVE ... |
||||
*/ |
||||
private String type; |
||||
|
||||
/** |
||||
* data source id |
||||
*/ |
||||
private int datasource; |
||||
|
||||
/** |
||||
* procedure name |
||||
*/ |
||||
private String method; |
||||
|
||||
public String getType() { |
||||
return type; |
||||
} |
||||
|
||||
public void setType(String type) { |
||||
this.type = type; |
||||
} |
||||
|
||||
public int getDatasource() { |
||||
return datasource; |
||||
} |
||||
|
||||
public void setDatasource(int datasource) { |
||||
this.datasource = datasource; |
||||
} |
||||
|
||||
public String getMethod() { |
||||
return method; |
||||
} |
||||
|
||||
public void setMethod(String method) { |
||||
this.method = method; |
||||
} |
||||
|
||||
@Override |
||||
public boolean checkParameters() { |
||||
return datasource != 0 && !StringUtils.isEmpty(type) && !StringUtils.isEmpty(method); |
||||
} |
||||
|
||||
@Override |
||||
public List<ResourceInfo> getResourceFilesList() { |
||||
return new ArrayList<>(); |
||||
} |
||||
|
||||
@Override |
||||
public String toString() { |
||||
return "ProcessdureParam{" |
||||
+ "type='" + type + '\'' |
||||
+ ", datasource=" + datasource |
||||
+ ", method='" + method + '\'' |
||||
+ '}'; |
||||
} |
||||
} |
||||
@ -1,61 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.task.python; |
||||
|
||||
import org.apache.dolphinscheduler.common.process.ResourceInfo; |
||||
import org.apache.dolphinscheduler.common.task.AbstractParameters; |
||||
|
||||
import java.util.List; |
||||
|
||||
public class PythonParameters extends AbstractParameters { |
||||
/** |
||||
* origin python script |
||||
*/ |
||||
private String rawScript; |
||||
|
||||
/** |
||||
* resource list |
||||
*/ |
||||
private List<ResourceInfo> resourceList; |
||||
|
||||
public String getRawScript() { |
||||
return rawScript; |
||||
} |
||||
|
||||
public void setRawScript(String rawScript) { |
||||
this.rawScript = rawScript; |
||||
} |
||||
|
||||
public List<ResourceInfo> getResourceList() { |
||||
return resourceList; |
||||
} |
||||
|
||||
public void setResourceList(List<ResourceInfo> resourceList) { |
||||
this.resourceList = resourceList; |
||||
} |
||||
|
||||
@Override |
||||
public boolean checkParameters() { |
||||
return rawScript != null && !rawScript.isEmpty(); |
||||
} |
||||
|
||||
@Override |
||||
public List<ResourceInfo> getResourceFilesList() { |
||||
return this.resourceList; |
||||
} |
||||
} |
||||
@ -1,65 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.task.shell; |
||||
|
||||
import org.apache.dolphinscheduler.common.process.ResourceInfo; |
||||
import org.apache.dolphinscheduler.common.task.AbstractParameters; |
||||
|
||||
import java.util.List; |
||||
|
||||
/** |
||||
* shell parameters |
||||
*/ |
||||
public class ShellParameters extends AbstractParameters { |
||||
/** |
||||
* shell script |
||||
*/ |
||||
private String rawScript; |
||||
|
||||
/** |
||||
* resource list |
||||
*/ |
||||
private List<ResourceInfo> resourceList; |
||||
|
||||
public String getRawScript() { |
||||
return rawScript; |
||||
} |
||||
|
||||
public void setRawScript(String rawScript) { |
||||
this.rawScript = rawScript; |
||||
} |
||||
|
||||
public List<ResourceInfo> getResourceList() { |
||||
return resourceList; |
||||
} |
||||
|
||||
public void setResourceList(List<ResourceInfo> resourceList) { |
||||
this.resourceList = resourceList; |
||||
} |
||||
|
||||
@Override |
||||
public boolean checkParameters() { |
||||
return rawScript != null && !rawScript.isEmpty(); |
||||
} |
||||
|
||||
@Override |
||||
public List<ResourceInfo> getResourceFilesList() { |
||||
return resourceList; |
||||
} |
||||
|
||||
} |
||||
@ -1,242 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.task.spark; |
||||
|
||||
import org.apache.dolphinscheduler.common.enums.ProgramType; |
||||
import org.apache.dolphinscheduler.common.process.ResourceInfo; |
||||
import org.apache.dolphinscheduler.common.task.AbstractParameters; |
||||
|
||||
import java.util.ArrayList; |
||||
import java.util.List; |
||||
|
||||
/** |
||||
* spark parameters |
||||
*/ |
||||
public class SparkParameters extends AbstractParameters { |
||||
|
||||
/** |
||||
* main jar |
||||
*/ |
||||
private ResourceInfo mainJar; |
||||
|
||||
/** |
||||
* main class
|
||||
*/ |
||||
private String mainClass; |
||||
|
||||
/** |
||||
* deploy mode |
||||
*/ |
||||
private String deployMode; |
||||
|
||||
/** |
||||
* arguments |
||||
*/ |
||||
private String mainArgs; |
||||
|
||||
/** |
||||
* driver-cores Number of cores used by the driver, only in cluster mode |
||||
*/ |
||||
private int driverCores; |
||||
|
||||
/** |
||||
* driver-memory Memory for driver |
||||
*/ |
||||
|
||||
private String driverMemory; |
||||
|
||||
/** |
||||
* num-executors Number of executors to launch |
||||
*/ |
||||
private int numExecutors; |
||||
|
||||
/** |
||||
* executor-cores Number of cores per executor |
||||
*/ |
||||
private int executorCores; |
||||
|
||||
/** |
||||
* Memory per executor |
||||
*/ |
||||
private String executorMemory; |
||||
|
||||
/** |
||||
* app name |
||||
*/ |
||||
private String appName; |
||||
|
||||
/** |
||||
* The YARN queue to submit to |
||||
*/ |
||||
private String queue; |
||||
|
||||
/** |
||||
* other arguments |
||||
*/ |
||||
private String others; |
||||
|
||||
/** |
||||
* program type |
||||
* 0 JAVA,1 SCALA,2 PYTHON |
||||
*/ |
||||
private ProgramType programType; |
||||
|
||||
/** |
||||
* spark version |
||||
*/ |
||||
private String sparkVersion; |
||||
|
||||
/** |
||||
* resource list |
||||
*/ |
||||
private List<ResourceInfo> resourceList = new ArrayList<>(); |
||||
|
||||
public ResourceInfo getMainJar() { |
||||
return mainJar; |
||||
} |
||||
|
||||
public void setMainJar(ResourceInfo mainJar) { |
||||
this.mainJar = mainJar; |
||||
} |
||||
|
||||
public String getMainClass() { |
||||
return mainClass; |
||||
} |
||||
|
||||
public void setMainClass(String mainClass) { |
||||
this.mainClass = mainClass; |
||||
} |
||||
|
||||
public String getDeployMode() { |
||||
return deployMode; |
||||
} |
||||
|
||||
public void setDeployMode(String deployMode) { |
||||
this.deployMode = deployMode; |
||||
} |
||||
|
||||
public String getMainArgs() { |
||||
return mainArgs; |
||||
} |
||||
|
||||
public void setMainArgs(String mainArgs) { |
||||
this.mainArgs = mainArgs; |
||||
} |
||||
|
||||
public int getDriverCores() { |
||||
return driverCores; |
||||
} |
||||
|
||||
public void setDriverCores(int driverCores) { |
||||
this.driverCores = driverCores; |
||||
} |
||||
|
||||
public String getDriverMemory() { |
||||
return driverMemory; |
||||
} |
||||
|
||||
public void setDriverMemory(String driverMemory) { |
||||
this.driverMemory = driverMemory; |
||||
} |
||||
|
||||
public int getNumExecutors() { |
||||
return numExecutors; |
||||
} |
||||
|
||||
public void setNumExecutors(int numExecutors) { |
||||
this.numExecutors = numExecutors; |
||||
} |
||||
|
||||
public int getExecutorCores() { |
||||
return executorCores; |
||||
} |
||||
|
||||
public void setExecutorCores(int executorCores) { |
||||
this.executorCores = executorCores; |
||||
} |
||||
|
||||
public String getExecutorMemory() { |
||||
return executorMemory; |
||||
} |
||||
|
||||
public void setExecutorMemory(String executorMemory) { |
||||
this.executorMemory = executorMemory; |
||||
} |
||||
|
||||
public String getAppName() { |
||||
return appName; |
||||
} |
||||
|
||||
public void setAppName(String appName) { |
||||
this.appName = appName; |
||||
} |
||||
|
||||
public String getQueue() { |
||||
return queue; |
||||
} |
||||
|
||||
public void setQueue(String queue) { |
||||
this.queue = queue; |
||||
} |
||||
|
||||
public String getOthers() { |
||||
return others; |
||||
} |
||||
|
||||
public void setOthers(String others) { |
||||
this.others = others; |
||||
} |
||||
|
||||
public List<ResourceInfo> getResourceList() { |
||||
return resourceList; |
||||
} |
||||
|
||||
public void setResourceList(List<ResourceInfo> resourceList) { |
||||
this.resourceList = resourceList; |
||||
} |
||||
|
||||
public ProgramType getProgramType() { |
||||
return programType; |
||||
} |
||||
|
||||
public void setProgramType(ProgramType programType) { |
||||
this.programType = programType; |
||||
} |
||||
|
||||
public String getSparkVersion() { |
||||
return sparkVersion; |
||||
} |
||||
|
||||
public void setSparkVersion(String sparkVersion) { |
||||
this.sparkVersion = sparkVersion; |
||||
} |
||||
|
||||
@Override |
||||
public boolean checkParameters() { |
||||
return mainJar != null && programType != null; |
||||
} |
||||
|
||||
@Override |
||||
public List<ResourceInfo> getResourceFilesList() { |
||||
if (mainJar != null && !resourceList.contains(mainJar)) { |
||||
resourceList.add(mainJar); |
||||
} |
||||
return resourceList; |
||||
} |
||||
|
||||
} |
||||
@ -1,43 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.task.sql; |
||||
|
||||
import org.apache.dolphinscheduler.common.process.Property; |
||||
|
||||
import java.util.Map; |
||||
|
||||
/** |
||||
* Used to contains both prepared sql string and its to-be-bind parameters |
||||
*/ |
||||
public class SqlBinds { |
||||
private final String sql; |
||||
private final Map<Integer, Property> paramsMap; |
||||
|
||||
public SqlBinds(String sql, Map<Integer, Property> paramsMap) { |
||||
this.sql = sql; |
||||
this.paramsMap = paramsMap; |
||||
} |
||||
|
||||
public String getSql() { |
||||
return sql; |
||||
} |
||||
|
||||
public Map<Integer, Property> getParamsMap() { |
||||
return paramsMap; |
||||
} |
||||
} |
||||
@ -1,295 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.task.sql; |
||||
|
||||
import org.apache.dolphinscheduler.common.enums.DataType; |
||||
import org.apache.dolphinscheduler.common.process.Property; |
||||
import org.apache.dolphinscheduler.common.process.ResourceInfo; |
||||
import org.apache.dolphinscheduler.common.task.AbstractParameters; |
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils; |
||||
|
||||
import org.apache.commons.collections.CollectionUtils; |
||||
import org.apache.commons.lang.StringUtils; |
||||
|
||||
import java.util.ArrayList; |
||||
import java.util.HashMap; |
||||
import java.util.List; |
||||
import java.util.Map; |
||||
import java.util.Set; |
||||
|
||||
/** |
||||
* Sql/Hql parameter |
||||
*/ |
||||
public class SqlParameters extends AbstractParameters { |
||||
/** |
||||
* data source type，eg MYSQL, POSTGRES, HIVE ... |
||||
*/ |
||||
private String type; |
||||
|
||||
/** |
||||
* datasource id |
||||
*/ |
||||
private int datasource; |
||||
|
||||
/** |
||||
* sql |
||||
*/ |
||||
private String sql; |
||||
|
||||
/** |
||||
* sql type |
||||
* 0 query |
||||
* 1 NON_QUERY |
||||
*/ |
||||
private int sqlType; |
||||
|
||||
/** |
||||
* send email |
||||
*/ |
||||
private Boolean sendEmail; |
||||
|
||||
/** |
||||
* display rows |
||||
*/ |
||||
private int displayRows; |
||||
|
||||
/** |
||||
* udf list |
||||
*/ |
||||
private String udfs; |
||||
/** |
||||
* show type |
||||
* 0 TABLE |
||||
* 1 TEXT |
||||
* 2 attachment |
||||
* 3 TABLE+attachment |
||||
*/ |
||||
private String showType; |
||||
/** |
||||
* SQL connection parameters |
||||
*/ |
||||
private String connParams; |
||||
/** |
||||
* Pre Statements |
||||
*/ |
||||
private List<String> preStatements; |
||||
/** |
||||
* Post Statements |
||||
*/ |
||||
private List<String> postStatements; |
||||
|
||||
/** |
||||
* groupId |
||||
*/ |
||||
private int groupId; |
||||
/** |
||||
* title |
||||
*/ |
||||
private String title; |
||||
|
||||
private int limit; |
||||
|
||||
public int getLimit() { |
||||
return limit; |
||||
} |
||||
|
||||
public void setLimit(int limit) { |
||||
this.limit = limit; |
||||
} |
||||
|
||||
public String getType() { |
||||
return type; |
||||
} |
||||
|
||||
public void setType(String type) { |
||||
this.type = type; |
||||
} |
||||
|
||||
public int getDatasource() { |
||||
return datasource; |
||||
} |
||||
|
||||
public void setDatasource(int datasource) { |
||||
this.datasource = datasource; |
||||
} |
||||
|
||||
public String getSql() { |
||||
return sql; |
||||
} |
||||
|
||||
public void setSql(String sql) { |
||||
this.sql = sql; |
||||
} |
||||
|
||||
public String getUdfs() { |
||||
return udfs; |
||||
} |
||||
|
||||
public void setUdfs(String udfs) { |
||||
this.udfs = udfs; |
||||
} |
||||
|
||||
public int getSqlType() { |
||||
return sqlType; |
||||
} |
||||
|
||||
public void setSqlType(int sqlType) { |
||||
this.sqlType = sqlType; |
||||
} |
||||
|
||||
public Boolean getSendEmail() { |
||||
return sendEmail; |
||||
} |
||||
|
||||
public void setSendEmail(Boolean sendEmail) { |
||||
this.sendEmail = sendEmail; |
||||
} |
||||
|
||||
public int getDisplayRows() { |
||||
return displayRows; |
||||
} |
||||
|
||||
public void setDisplayRows(int displayRows) { |
||||
this.displayRows = displayRows; |
||||
} |
||||
|
||||
public String getShowType() { |
||||
return showType; |
||||
} |
||||
|
||||
public void setShowType(String showType) { |
||||
this.showType = showType; |
||||
} |
||||
|
||||
public String getConnParams() { |
||||
return connParams; |
||||
} |
||||
|
||||
public void setConnParams(String connParams) { |
||||
this.connParams = connParams; |
||||
} |
||||
|
||||
public String getTitle() { |
||||
return title; |
||||
} |
||||
|
||||
public void setTitle(String title) { |
||||
this.title = title; |
||||
} |
||||
|
||||
public List<String> getPreStatements() { |
||||
return preStatements; |
||||
} |
||||
|
||||
public void setPreStatements(List<String> preStatements) { |
||||
this.preStatements = preStatements; |
||||
} |
||||
|
||||
public List<String> getPostStatements() { |
||||
return postStatements; |
||||
} |
||||
|
||||
public void setPostStatements(List<String> postStatements) { |
||||
this.postStatements = postStatements; |
||||
} |
||||
|
||||
public int getGroupId() { |
||||
return groupId; |
||||
} |
||||
|
||||
public void setGroupId(int groupId) { |
||||
this.groupId = groupId; |
||||
} |
||||
|
||||
@Override |
||||
public boolean checkParameters() { |
||||
return datasource != 0 && !StringUtils.isEmpty(type) && !StringUtils.isEmpty(sql); |
||||
} |
||||
|
||||
@Override |
||||
public List<ResourceInfo> getResourceFilesList() { |
||||
return new ArrayList<>(); |
||||
} |
||||
|
||||
@Override |
||||
public void dealOutParam(String result) { |
||||
if (CollectionUtils.isEmpty(localParams)) { |
||||
return; |
||||
} |
||||
List<Property> outProperty = getOutProperty(localParams); |
||||
if (CollectionUtils.isEmpty(outProperty)) { |
||||
return; |
||||
} |
||||
if (StringUtils.isEmpty(result)) { |
||||
varPool.addAll(outProperty); |
||||
return; |
||||
} |
||||
List<Map<String, String>> sqlResult = getListMapByString(result); |
||||
if (CollectionUtils.isEmpty(sqlResult)) { |
||||
return; |
||||
} |
||||
//if sql return more than one line
|
||||
if (sqlResult.size() > 1) { |
||||
Map<String, List<String>> sqlResultFormat = new HashMap<>(); |
||||
//init sqlResultFormat
|
||||
Set<String> keySet = sqlResult.get(0).keySet(); |
||||
for (String key : keySet) { |
||||
sqlResultFormat.put(key, new ArrayList<>()); |
||||
} |
||||
for (Map<String, String> info : sqlResult) { |
||||
info.forEach((key, value) -> { |
||||
sqlResultFormat.get(key).add(value); |
||||
}); |
||||
} |
||||
for (Property info : outProperty) { |
||||
if (info.getType() == DataType.LIST) { |
||||
info.setValue(JSONUtils.toJsonString(sqlResultFormat.get(info.getProp()))); |
||||
varPool.add(info); |
||||
} |
||||
} |
||||
} else { |
||||
//result only one line
|
||||
Map<String, String> firstRow = sqlResult.get(0); |
||||
for (Property info : outProperty) { |
||||
info.setValue(String.valueOf(firstRow.get(info.getProp()))); |
||||
varPool.add(info); |
||||
} |
||||
} |
||||
|
||||
} |
||||
|
||||
@Override |
||||
public String toString() { |
||||
return "SqlParameters{" |
||||
+ "type='" + type + '\'' |
||||
+ ", datasource=" + datasource |
||||
+ ", sql='" + sql + '\'' |
||||
+ ", sqlType=" + sqlType |
||||
+ ", sendEmail=" + sendEmail |
||||
+ ", displayRows=" + displayRows |
||||
+ ", limit=" + limit |
||||
+ ", udfs='" + udfs + '\'' |
||||
+ ", showType='" + showType + '\'' |
||||
+ ", connParams='" + connParams + '\'' |
||||
+ ", groupId='" + groupId + '\'' |
||||
+ ", title='" + title + '\'' |
||||
+ ", preStatements=" + preStatements |
||||
+ ", postStatements=" + postStatements |
||||
+ '}'; |
||||
} |
||||
} |
||||
@ -1,205 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.task.sqoop; |
||||
|
||||
import org.apache.dolphinscheduler.common.enums.SqoopJobType; |
||||
import org.apache.dolphinscheduler.common.process.Property; |
||||
import org.apache.dolphinscheduler.common.process.ResourceInfo; |
||||
import org.apache.dolphinscheduler.common.task.AbstractParameters; |
||||
|
||||
import org.apache.commons.lang.StringUtils; |
||||
|
||||
import java.util.ArrayList; |
||||
import java.util.List; |
||||
|
||||
/** |
||||
* sqoop parameters |
||||
*/ |
||||
public class SqoopParameters extends AbstractParameters { |
||||
|
||||
/** |
||||
* sqoop job type: |
||||
* CUSTOM - custom sqoop job |
||||
* TEMPLATE - sqoop template job |
||||
*/ |
||||
private String jobType; |
||||
|
||||
/** |
||||
* customJob eq 1, use customShell |
||||
*/ |
||||
private String customShell; |
||||
|
||||
/** |
||||
* sqoop job name - map-reduce job name |
||||
*/ |
||||
private String jobName; |
||||
|
||||
/** |
||||
* model type |
||||
*/ |
||||
private String modelType; |
||||
/** |
||||
* concurrency |
||||
*/ |
||||
private int concurrency; |
||||
/** |
||||
* source type |
||||
*/ |
||||
private String sourceType; |
||||
/** |
||||
* target type |
||||
*/ |
||||
private String targetType; |
||||
/** |
||||
* source params |
||||
*/ |
||||
private String sourceParams; |
||||
/** |
||||
* target params |
||||
*/ |
||||
private String targetParams; |
||||
|
||||
/** |
||||
* hadoop custom param for sqoop job |
||||
*/ |
||||
private List<Property> hadoopCustomParams; |
||||
|
||||
/** |
||||
* sqoop advanced param |
||||
*/ |
||||
private List<Property> sqoopAdvancedParams; |
||||
|
||||
public String getModelType() { |
||||
return modelType; |
||||
} |
||||
|
||||
public void setModelType(String modelType) { |
||||
this.modelType = modelType; |
||||
} |
||||
|
||||
public int getConcurrency() { |
||||
return concurrency; |
||||
} |
||||
|
||||
public void setConcurrency(int concurrency) { |
||||
this.concurrency = concurrency; |
||||
} |
||||
|
||||
public String getSourceType() { |
||||
return sourceType; |
||||
} |
||||
|
||||
public void setSourceType(String sourceType) { |
||||
this.sourceType = sourceType; |
||||
} |
||||
|
||||
public String getTargetType() { |
||||
return targetType; |
||||
} |
||||
|
||||
public void setTargetType(String targetType) { |
||||
this.targetType = targetType; |
||||
} |
||||
|
||||
public String getSourceParams() { |
||||
return sourceParams; |
||||
} |
||||
|
||||
public void setSourceParams(String sourceParams) { |
||||
this.sourceParams = sourceParams; |
||||
} |
||||
|
||||
public String getTargetParams() { |
||||
return targetParams; |
||||
} |
||||
|
||||
public void setTargetParams(String targetParams) { |
||||
this.targetParams = targetParams; |
||||
} |
||||
|
||||
public String getJobType() { |
||||
return jobType; |
||||
} |
||||
|
||||
public void setJobType(String jobType) { |
||||
this.jobType = jobType; |
||||
} |
||||
|
||||
public String getJobName() { |
||||
return jobName; |
||||
} |
||||
|
||||
public void setJobName(String jobName) { |
||||
this.jobName = jobName; |
||||
} |
||||
|
||||
public String getCustomShell() { |
||||
return customShell; |
||||
} |
||||
|
||||
public void setCustomShell(String customShell) { |
||||
this.customShell = customShell; |
||||
} |
||||
|
||||
public List<Property> getHadoopCustomParams() { |
||||
return hadoopCustomParams; |
||||
} |
||||
|
||||
public void setHadoopCustomParams(List<Property> hadoopCustomParams) { |
||||
this.hadoopCustomParams = hadoopCustomParams; |
||||
} |
||||
|
||||
public List<Property> getSqoopAdvancedParams() { |
||||
return sqoopAdvancedParams; |
||||
} |
||||
|
||||
public void setSqoopAdvancedParams(List<Property> sqoopAdvancedParams) { |
||||
this.sqoopAdvancedParams = sqoopAdvancedParams; |
||||
} |
||||
|
||||
@Override |
||||
public boolean checkParameters() { |
||||
|
||||
boolean sqoopParamsCheck = false; |
||||
|
||||
if (StringUtils.isEmpty(jobType)) { |
||||
return sqoopParamsCheck; |
||||
} |
||||
|
||||
if (SqoopJobType.TEMPLATE.getDescp().equals(jobType)) { |
||||
sqoopParamsCheck = StringUtils.isEmpty(customShell) |
||||
&& !StringUtils.isEmpty(modelType) |
||||
&& !StringUtils.isEmpty(jobName) |
||||
&& concurrency != 0 |
||||
&& !StringUtils.isEmpty(sourceType) |
||||
&& !StringUtils.isEmpty(targetType) |
||||
&& !StringUtils.isEmpty(sourceParams) |
||||
&& !StringUtils.isEmpty(targetParams); |
||||
} else if (SqoopJobType.CUSTOM.getDescp().equals(jobType)) { |
||||
sqoopParamsCheck = !StringUtils.isEmpty(customShell) |
||||
&& StringUtils.isEmpty(jobName); |
||||
} |
||||
|
||||
return sqoopParamsCheck; |
||||
} |
||||
|
||||
@Override |
||||
public List<ResourceInfo> getResourceFilesList() { |
||||
return new ArrayList<>(); |
||||
} |
||||
} |
||||
@ -1,37 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.task.sqoop.sources; |
||||
|
||||
/** |
||||
* source hdfs parameter |
||||
*/ |
||||
public class SourceHdfsParameter { |
||||
|
||||
/** |
||||
* export dir |
||||
*/ |
||||
private String exportDir; |
||||
|
||||
public String getExportDir() { |
||||
return exportDir; |
||||
} |
||||
|
||||
public void setExportDir(String exportDir) { |
||||
this.exportDir = exportDir; |
||||
} |
||||
} |
||||
@ -1,73 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.task.sqoop.sources; |
||||
|
||||
/** |
||||
* source hive parameter |
||||
*/ |
||||
public class SourceHiveParameter { |
||||
|
||||
/** |
||||
* hive database |
||||
*/ |
||||
private String hiveDatabase; |
||||
/** |
||||
* hive table |
||||
*/ |
||||
private String hiveTable; |
||||
/** |
||||
* hive partition key |
||||
*/ |
||||
private String hivePartitionKey; |
||||
/** |
||||
* hive partition value |
||||
*/ |
||||
private String hivePartitionValue; |
||||
|
||||
public String getHiveDatabase() { |
||||
return hiveDatabase; |
||||
} |
||||
|
||||
public void setHiveDatabase(String hiveDatabase) { |
||||
this.hiveDatabase = hiveDatabase; |
||||
} |
||||
|
||||
public String getHiveTable() { |
||||
return hiveTable; |
||||
} |
||||
|
||||
public void setHiveTable(String hiveTable) { |
||||
this.hiveTable = hiveTable; |
||||
} |
||||
|
||||
public String getHivePartitionKey() { |
||||
return hivePartitionKey; |
||||
} |
||||
|
||||
public void setHivePartitionKey(String hivePartitionKey) { |
||||
this.hivePartitionKey = hivePartitionKey; |
||||
} |
||||
|
||||
public String getHivePartitionValue() { |
||||
return hivePartitionValue; |
||||
} |
||||
|
||||
public void setHivePartitionValue(String hivePartitionValue) { |
||||
this.hivePartitionValue = hivePartitionValue; |
||||
} |
||||
} |
||||
@ -1,137 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.task.sqoop.sources; |
||||
|
||||
import org.apache.dolphinscheduler.common.process.Property; |
||||
|
||||
import java.util.List; |
||||
|
||||
/** |
||||
* source mysql parameter |
||||
*/ |
||||
public class SourceMysqlParameter { |
||||
|
||||
/** |
||||
* src datasource |
||||
*/ |
||||
private int srcDatasource; |
||||
/** |
||||
* src table |
||||
*/ |
||||
private String srcTable; |
||||
/** |
||||
* src query type |
||||
*/ |
||||
private int srcQueryType; |
||||
/** |
||||
* src query sql |
||||
*/ |
||||
private String srcQuerySql; |
||||
/** |
||||
* src column type |
||||
*/ |
||||
private int srcColumnType; |
||||
/** |
||||
* src columns |
||||
*/ |
||||
private String srcColumns; |
||||
/** |
||||
* src condition list |
||||
*/ |
||||
private List<Property> srcConditionList; |
||||
/** |
||||
* map column hive |
||||
*/ |
||||
private List<Property> mapColumnHive; |
||||
/** |
||||
* map column java |
||||
*/ |
||||
private List<Property> mapColumnJava; |
||||
|
||||
public int getSrcDatasource() { |
||||
return srcDatasource; |
||||
} |
||||
|
||||
public void setSrcDatasource(int srcDatasource) { |
||||
this.srcDatasource = srcDatasource; |
||||
} |
||||
|
||||
public String getSrcTable() { |
||||
return srcTable; |
||||
} |
||||
|
||||
public void setSrcTable(String srcTable) { |
||||
this.srcTable = srcTable; |
||||
} |
||||
|
||||
public int getSrcQueryType() { |
||||
return srcQueryType; |
||||
} |
||||
|
||||
public void setSrcQueryType(int srcQueryType) { |
||||
this.srcQueryType = srcQueryType; |
||||
} |
||||
|
||||
public String getSrcQuerySql() { |
||||
return srcQuerySql; |
||||
} |
||||
|
||||
public void setSrcQuerySql(String srcQuerySql) { |
||||
this.srcQuerySql = srcQuerySql; |
||||
} |
||||
|
||||
public int getSrcColumnType() { |
||||
return srcColumnType; |
||||
} |
||||
|
||||
public void setSrcColumnType(int srcColumnType) { |
||||
this.srcColumnType = srcColumnType; |
||||
} |
||||
|
||||
public String getSrcColumns() { |
||||
return srcColumns; |
||||
} |
||||
|
||||
public void setSrcColumns(String srcColumns) { |
||||
this.srcColumns = srcColumns; |
||||
} |
||||
|
||||
public List<Property> getSrcConditionList() { |
||||
return srcConditionList; |
||||
} |
||||
|
||||
public void setSrcConditionList(List<Property> srcConditionList) { |
||||
this.srcConditionList = srcConditionList; |
||||
} |
||||
|
||||
public List<Property> getMapColumnHive() { |
||||
return mapColumnHive; |
||||
} |
||||
|
||||
public void setMapColumnHive(List<Property> mapColumnHive) { |
||||
this.mapColumnHive = mapColumnHive; |
||||
} |
||||
|
||||
public List<Property> getMapColumnJava() { |
||||
return mapColumnJava; |
||||
} |
||||
|
||||
public void setMapColumnJava(List<Property> mapColumnJava) { |
||||
this.mapColumnJava = mapColumnJava; |
||||
} |
||||
} |
||||
@ -1,98 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.task.sqoop.targets; |
||||
|
||||
/** |
||||
* target hdfs parameter |
||||
*/ |
||||
public class TargetHdfsParameter { |
||||
|
||||
/** |
||||
* target dir |
||||
*/ |
||||
private String targetPath; |
||||
/** |
||||
* delete target dir |
||||
*/ |
||||
private boolean deleteTargetDir; |
||||
/** |
||||
* file type |
||||
*/ |
||||
private String fileType; |
||||
/** |
||||
* compression codec |
||||
*/ |
||||
private String compressionCodec; |
||||
/** |
||||
* fields terminated |
||||
*/ |
||||
private String fieldsTerminated; |
||||
/** |
||||
* lines terminated |
||||
*/ |
||||
private String linesTerminated; |
||||
|
||||
public String getTargetPath() { |
||||
return targetPath; |
||||
} |
||||
|
||||
public void setTargetPath(String targetPath) { |
||||
this.targetPath = targetPath; |
||||
} |
||||
|
||||
public boolean isDeleteTargetDir() { |
||||
return deleteTargetDir; |
||||
} |
||||
|
||||
public void setDeleteTargetDir(boolean deleteTargetDir) { |
||||
this.deleteTargetDir = deleteTargetDir; |
||||
} |
||||
|
||||
public String getFileType() { |
||||
return fileType; |
||||
} |
||||
|
||||
public void setFileType(String fileType) { |
||||
this.fileType = fileType; |
||||
} |
||||
|
||||
public String getCompressionCodec() { |
||||
return compressionCodec; |
||||
} |
||||
|
||||
public void setCompressionCodec(String compressionCodec) { |
||||
this.compressionCodec = compressionCodec; |
||||
} |
||||
|
||||
public String getFieldsTerminated() { |
||||
return fieldsTerminated; |
||||
} |
||||
|
||||
public void setFieldsTerminated(String fieldsTerminated) { |
||||
this.fieldsTerminated = fieldsTerminated; |
||||
} |
||||
|
||||
public String getLinesTerminated() { |
||||
return linesTerminated; |
||||
} |
||||
|
||||
public void setLinesTerminated(String linesTerminated) { |
||||
this.linesTerminated = linesTerminated; |
||||
} |
||||
|
||||
} |
||||
@ -1,121 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.task.sqoop.targets; |
||||
|
||||
/** |
||||
* target hive parameter |
||||
*/ |
||||
public class TargetHiveParameter { |
||||
|
||||
/** |
||||
* hive database |
||||
*/ |
||||
private String hiveDatabase; |
||||
/** |
||||
* hive table |
||||
*/ |
||||
private String hiveTable; |
||||
/** |
||||
* create hive table |
||||
*/ |
||||
private boolean createHiveTable; |
||||
/** |
||||
* drop delimiter |
||||
*/ |
||||
private boolean dropDelimiter; |
||||
/** |
||||
* hive overwrite |
||||
*/ |
||||
private boolean hiveOverWrite; |
||||
/** |
||||
* replace delimiter |
||||
*/ |
||||
private String replaceDelimiter; |
||||
/** |
||||
* hive partition key |
||||
*/ |
||||
private String hivePartitionKey; |
||||
/** |
||||
* hive partition value |
||||
*/ |
||||
private String hivePartitionValue; |
||||
|
||||
public String getHiveDatabase() { |
||||
return hiveDatabase; |
||||
} |
||||
|
||||
public void setHiveDatabase(String hiveDatabase) { |
||||
this.hiveDatabase = hiveDatabase; |
||||
} |
||||
|
||||
public String getHiveTable() { |
||||
return hiveTable; |
||||
} |
||||
|
||||
public void setHiveTable(String hiveTable) { |
||||
this.hiveTable = hiveTable; |
||||
} |
||||
|
||||
public boolean isCreateHiveTable() { |
||||
return createHiveTable; |
||||
} |
||||
|
||||
public void setCreateHiveTable(boolean createHiveTable) { |
||||
this.createHiveTable = createHiveTable; |
||||
} |
||||
|
||||
public boolean isDropDelimiter() { |
||||
return dropDelimiter; |
||||
} |
||||
|
||||
public void setDropDelimiter(boolean dropDelimiter) { |
||||
this.dropDelimiter = dropDelimiter; |
||||
} |
||||
|
||||
public boolean isHiveOverWrite() { |
||||
return hiveOverWrite; |
||||
} |
||||
|
||||
public void setHiveOverWrite(boolean hiveOverWrite) { |
||||
this.hiveOverWrite = hiveOverWrite; |
||||
} |
||||
|
||||
public String getReplaceDelimiter() { |
||||
return replaceDelimiter; |
||||
} |
||||
|
||||
public void setReplaceDelimiter(String replaceDelimiter) { |
||||
this.replaceDelimiter = replaceDelimiter; |
||||
} |
||||
|
||||
public String getHivePartitionKey() { |
||||
return hivePartitionKey; |
||||
} |
||||
|
||||
public void setHivePartitionKey(String hivePartitionKey) { |
||||
this.hivePartitionKey = hivePartitionKey; |
||||
} |
||||
|
||||
public String getHivePartitionValue() { |
||||
return hivePartitionValue; |
||||
} |
||||
|
||||
public void setHivePartitionValue(String hivePartitionValue) { |
||||
this.hivePartitionValue = hivePartitionValue; |
||||
} |
||||
} |
||||
@ -1,133 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.task.sqoop.targets; |
||||
|
||||
/** |
||||
* target mysql parameter |
||||
*/ |
||||
public class TargetMysqlParameter { |
||||
|
||||
/** |
||||
* target datasource |
||||
*/ |
||||
private int targetDatasource; |
||||
/** |
||||
* target table |
||||
*/ |
||||
private String targetTable; |
||||
/** |
||||
* target columns |
||||
*/ |
||||
private String targetColumns; |
||||
/** |
||||
* fields terminated |
||||
*/ |
||||
private String fieldsTerminated; |
||||
/** |
||||
* lines terminated |
||||
*/ |
||||
private String linesTerminated; |
||||
/** |
||||
* pre query |
||||
*/ |
||||
private String preQuery; |
||||
/** |
||||
* is update |
||||
*/ |
||||
private boolean isUpdate; |
||||
/** |
||||
* target update key |
||||
*/ |
||||
private String targetUpdateKey; |
||||
/** |
||||
* target update mode |
||||
*/ |
||||
private String targetUpdateMode; |
||||
|
||||
public int getTargetDatasource() { |
||||
return targetDatasource; |
||||
} |
||||
|
||||
public void setTargetDatasource(int targetDatasource) { |
||||
this.targetDatasource = targetDatasource; |
||||
} |
||||
|
||||
public String getTargetTable() { |
||||
return targetTable; |
||||
} |
||||
|
||||
public void setTargetTable(String targetTable) { |
||||
this.targetTable = targetTable; |
||||
} |
||||
|
||||
public String getTargetColumns() { |
||||
return targetColumns; |
||||
} |
||||
|
||||
public void setTargetColumns(String targetColumns) { |
||||
this.targetColumns = targetColumns; |
||||
} |
||||
|
||||
public String getFieldsTerminated() { |
||||
return fieldsTerminated; |
||||
} |
||||
|
||||
public void setFieldsTerminated(String fieldsTerminated) { |
||||
this.fieldsTerminated = fieldsTerminated; |
||||
} |
||||
|
||||
public String getLinesTerminated() { |
||||
return linesTerminated; |
||||
} |
||||
|
||||
public void setLinesTerminated(String linesTerminated) { |
||||
this.linesTerminated = linesTerminated; |
||||
} |
||||
|
||||
public String getPreQuery() { |
||||
return preQuery; |
||||
} |
||||
|
||||
public void setPreQuery(String preQuery) { |
||||
this.preQuery = preQuery; |
||||
} |
||||
|
||||
public boolean getIsUpdate() { |
||||
return isUpdate; |
||||
} |
||||
|
||||
public void setUpdate(boolean update) { |
||||
isUpdate = update; |
||||
} |
||||
|
||||
public String getTargetUpdateKey() { |
||||
return targetUpdateKey; |
||||
} |
||||
|
||||
public void setTargetUpdateKey(String targetUpdateKey) { |
||||
this.targetUpdateKey = targetUpdateKey; |
||||
} |
||||
|
||||
public String getTargetUpdateMode() { |
||||
return targetUpdateMode; |
||||
} |
||||
|
||||
public void setTargetUpdateMode(String targetUpdateMode) { |
||||
this.targetUpdateMode = targetUpdateMode; |
||||
} |
||||
} |
||||
@ -1,58 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.task.tis; |
||||
|
||||
import org.apache.dolphinscheduler.common.process.ResourceInfo; |
||||
import org.apache.dolphinscheduler.common.task.AbstractParameters; |
||||
import org.apache.dolphinscheduler.spi.utils.StringUtils; |
||||
|
||||
import java.util.Collections; |
||||
import java.util.List; |
||||
|
||||
import org.slf4j.Logger; |
||||
import org.slf4j.LoggerFactory; |
||||
|
||||
/** |
||||
* TIS parameter |
||||
*/ |
||||
public class PigeonCommonParameters extends AbstractParameters { |
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(PigeonCommonParameters.class); |
||||
/** |
||||
* TIS target job name |
||||
*/ |
||||
private String jobName; |
||||
|
||||
public String getTargetJobName() { |
||||
return jobName; |
||||
} |
||||
|
||||
public void setTargetJobName(String jobName) { |
||||
this.jobName = jobName; |
||||
} |
||||
|
||||
@Override |
||||
public boolean checkParameters() { |
||||
return StringUtils.isNotBlank(this.jobName); |
||||
} |
||||
|
||||
@Override |
||||
public List<ResourceInfo> getResourceFilesList() { |
||||
return Collections.emptyList(); |
||||
} |
||||
} |
||||
@ -1,106 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.utils; |
||||
|
||||
import org.apache.dolphinscheduler.common.task.AbstractParameters; |
||||
import org.apache.dolphinscheduler.common.task.blocking.BlockingParameters; |
||||
import org.apache.dolphinscheduler.common.task.conditions.ConditionsParameters; |
||||
import org.apache.dolphinscheduler.common.task.datax.DataxParameters; |
||||
import org.apache.dolphinscheduler.common.task.dependent.DependentParameters; |
||||
import org.apache.dolphinscheduler.common.task.dq.DataQualityParameters; |
||||
import org.apache.dolphinscheduler.common.task.emr.EmrParameters; |
||||
import org.apache.dolphinscheduler.common.task.flink.FlinkParameters; |
||||
import org.apache.dolphinscheduler.common.task.http.HttpParameters; |
||||
import org.apache.dolphinscheduler.common.task.mr.MapReduceParameters; |
||||
import org.apache.dolphinscheduler.common.task.procedure.ProcedureParameters; |
||||
import org.apache.dolphinscheduler.common.task.python.PythonParameters; |
||||
import org.apache.dolphinscheduler.common.task.shell.ShellParameters; |
||||
import org.apache.dolphinscheduler.common.task.spark.SparkParameters; |
||||
import org.apache.dolphinscheduler.common.task.sql.SqlParameters; |
||||
import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters; |
||||
import org.apache.dolphinscheduler.common.task.subprocess.SubProcessParameters; |
||||
import org.apache.dolphinscheduler.common.task.switchtask.SwitchParameters; |
||||
import org.apache.dolphinscheduler.common.task.tis.PigeonCommonParameters; |
||||
|
||||
import org.slf4j.Logger; |
||||
import org.slf4j.LoggerFactory; |
||||
|
||||
/** |
||||
* task parameters utils |
||||
*/ |
||||
public class TaskParametersUtils { |
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(TaskParametersUtils.class); |
||||
|
||||
private TaskParametersUtils() { |
||||
throw new UnsupportedOperationException("Construct TaskParametersUtils"); |
||||
} |
||||
|
||||
/** |
||||
* get task parameters |
||||
* |
||||
* @param taskType task type |
||||
* @param parameter parameter |
||||
* @return task parameters |
||||
*/ |
||||
public static AbstractParameters getParameters(String taskType, String parameter) { |
||||
switch (taskType) { |
||||
case "SUB_PROCESS": |
||||
return JSONUtils.parseObject(parameter, SubProcessParameters.class); |
||||
case "SHELL": |
||||
case "SEATUNNEL": |
||||
return JSONUtils.parseObject(parameter, ShellParameters.class); |
||||
case "PROCEDURE": |
||||
return JSONUtils.parseObject(parameter, ProcedureParameters.class); |
||||
case "SQL": |
||||
return JSONUtils.parseObject(parameter, SqlParameters.class); |
||||
case "MR": |
||||
return JSONUtils.parseObject(parameter, MapReduceParameters.class); |
||||
case "SPARK": |
||||
return JSONUtils.parseObject(parameter, SparkParameters.class); |
||||
case "PYTHON": |
||||
return JSONUtils.parseObject(parameter, PythonParameters.class); |
||||
case "DEPENDENT": |
||||
return JSONUtils.parseObject(parameter, DependentParameters.class); |
||||
case "FLINK": |
||||
return JSONUtils.parseObject(parameter, FlinkParameters.class); |
||||
case "HTTP": |
||||
return JSONUtils.parseObject(parameter, HttpParameters.class); |
||||
case "DATAX": |
||||
return JSONUtils.parseObject(parameter, DataxParameters.class); |
||||
case "CONDITIONS": |
||||
return JSONUtils.parseObject(parameter, ConditionsParameters.class); |
||||
case "SQOOP": |
||||
return JSONUtils.parseObject(parameter, SqoopParameters.class); |
||||
case "DATA_QUALITY": |
||||
return JSONUtils.parseObject(parameter, DataQualityParameters.class); |
||||
case "SWITCH": |
||||
return JSONUtils.parseObject(parameter, SwitchParameters.class); |
||||
case "BLOCKING": |
||||
return JSONUtils.parseObject(parameter, BlockingParameters.class); |
||||
case "PIGEON": |
||||
return JSONUtils.parseObject(parameter, PigeonCommonParameters.class); |
||||
case "EMR": |
||||
return JSONUtils.parseObject(parameter, EmrParameters.class); |
||||
default: |
||||
logger.error("not support task type: {}", taskType); |
||||
return null; |
||||
} |
||||
|
||||
} |
||||
} |
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue