百岁
3 years ago
committed by
GitHub
19 changed files with 1112 additions and 3 deletions
@ -0,0 +1,97 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?> |
||||
<!-- |
||||
~ Licensed to the Apache Software Foundation (ASF) under one or more |
||||
~ contributor license agreements. See the NOTICE file distributed with |
||||
~ this work for additional information regarding copyright ownership. |
||||
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
~ (the "License"); you may not use this file except in compliance with |
||||
~ the License. You may obtain a copy of the License at |
||||
~ |
||||
~ http://www.apache.org/licenses/LICENSE-2.0 |
||||
~ |
||||
~ Unless required by applicable law or agreed to in writing, software |
||||
~ distributed under the License is distributed on an "AS IS" BASIS, |
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
~ See the License for the specific language governing permissions and |
||||
~ limitations under the License. |
||||
--> |
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" |
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" |
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> |
||||
<parent> |
||||
<artifactId>dolphinscheduler-task-plugin</artifactId> |
||||
<groupId>org.apache.dolphinscheduler</groupId> |
||||
<version>1.3.6-SNAPSHOT</version> |
||||
</parent> |
||||
|
||||
<modelVersion>4.0.0</modelVersion> |
||||
|
||||
<artifactId>dolphinscheduler-task-tis</artifactId> |
||||
|
||||
<dependencies> |
||||
|
||||
<dependency> |
||||
<groupId>org.apache.dolphinscheduler</groupId> |
||||
<artifactId>dolphinscheduler-task-api</artifactId> |
||||
<version>${project.version}</version> |
||||
</dependency> |
||||
<dependency> |
||||
<groupId>org.slf4j</groupId> |
||||
<artifactId>slf4j-api</artifactId> |
||||
</dependency> |
||||
|
||||
<!--https://github.com/dreamhead/moco/blob/master/moco-doc/usage.md#socket--> |
||||
<dependency> |
||||
<groupId>com.github.dreamhead</groupId> |
||||
<artifactId>moco-core</artifactId> |
||||
<version>1.2.0</version> |
||||
<scope>test</scope> |
||||
</dependency> |
||||
<dependency> |
||||
<groupId>com.github.dreamhead</groupId> |
||||
<artifactId>moco-runner</artifactId> |
||||
<version>1.2.0</version> |
||||
<scope>test</scope> |
||||
<exclusions> |
||||
<exclusion> |
||||
<groupId>commons-cli</groupId> |
||||
<artifactId>commons-cli</artifactId> |
||||
</exclusion> |
||||
</exclusions> |
||||
</dependency> |
||||
|
||||
<dependency> |
||||
<groupId>org.asynchttpclient</groupId> |
||||
<artifactId>async-http-client</artifactId> |
||||
<version>2.12.3</version> |
||||
</dependency> |
||||
|
||||
<dependency> |
||||
<groupId>org.apache.httpcomponents</groupId> |
||||
<artifactId>httpclient</artifactId> |
||||
</dependency> |
||||
<dependency> |
||||
<groupId>org.apache.httpcomponents</groupId> |
||||
<artifactId>httpcore</artifactId> |
||||
</dependency> |
||||
<dependency> |
||||
<groupId>org.apache.dolphinscheduler</groupId> |
||||
<artifactId>dolphinscheduler-common</artifactId> |
||||
</dependency> |
||||
<dependency> |
||||
<groupId>org.mockito</groupId> |
||||
<artifactId>mockito-core</artifactId> |
||||
</dependency> |
||||
<dependency> |
||||
<groupId>org.apache.dolphinscheduler</groupId> |
||||
<artifactId>dolphinscheduler-server</artifactId> |
||||
<scope>provided</scope> |
||||
</dependency> |
||||
<dependency> |
||||
<groupId>org.powermock</groupId> |
||||
<artifactId>powermock-api-mockito2</artifactId> |
||||
</dependency> |
||||
|
||||
</dependencies> |
||||
|
||||
</project> |
||||
@ -0,0 +1,62 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.plugin.task.tis; |
||||
|
||||
import org.apache.dolphinscheduler.spi.task.AbstractParameters; |
||||
import org.apache.dolphinscheduler.spi.task.ResourceInfo; |
||||
import org.apache.dolphinscheduler.spi.utils.StringUtils; |
||||
|
||||
import java.util.Collections; |
||||
import java.util.List; |
||||
|
||||
import org.slf4j.Logger; |
||||
import org.slf4j.LoggerFactory; |
||||
|
||||
/** |
||||
* TIS parameter |
||||
*/ |
||||
public class TISParameters extends AbstractParameters { |
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(TISParameters.class); |
||||
/** |
||||
* TIS target job name |
||||
*/ |
||||
private String targetJobName; |
||||
|
||||
public String getTargetJobName() { |
||||
return targetJobName; |
||||
} |
||||
|
||||
public void setTargetJobName(String targetJobName) { |
||||
this.targetJobName = targetJobName; |
||||
} |
||||
|
||||
@Override |
||||
public boolean checkParameters() { |
||||
if (StringUtils.isBlank(this.targetJobName)) { |
||||
logger.error("checkParameters faild targetJobName can not be null"); |
||||
return false; |
||||
} |
||||
return true; |
||||
} |
||||
|
||||
@Override |
||||
public List<ResourceInfo> getResourceFilesList() { |
||||
return Collections.emptyList(); |
||||
} |
||||
} |
||||
@ -0,0 +1,27 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.plugin.task.tis; |
||||
|
||||
public class TISParamsConstants { |
||||
|
||||
public static String NAME_TARGET_JOB_NAME = "targetJobName"; |
||||
public static String TARGET_JOB_NAME = NAME_TARGET_JOB_NAME; |
||||
|
||||
private TISParamsConstants() { |
||||
} |
||||
} |
||||
@ -0,0 +1,345 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.plugin.task.tis; |
||||
|
||||
import org.apache.dolphinscheduler.common.utils.CollectionUtils; |
||||
import org.apache.dolphinscheduler.spi.task.AbstractParameters; |
||||
import org.apache.dolphinscheduler.spi.task.AbstractTask; |
||||
import org.apache.dolphinscheduler.spi.task.TaskConstants; |
||||
import org.apache.dolphinscheduler.spi.task.TaskRequest; |
||||
import org.apache.dolphinscheduler.spi.utils.JSONUtils; |
||||
import org.apache.dolphinscheduler.spi.utils.StringUtils; |
||||
|
||||
import org.apache.http.HttpEntity; |
||||
import org.apache.http.StatusLine; |
||||
import org.apache.http.client.methods.CloseableHttpResponse; |
||||
import org.apache.http.client.methods.HttpPost; |
||||
import org.apache.http.entity.StringEntity; |
||||
import org.apache.http.impl.client.CloseableHttpClient; |
||||
import org.apache.http.impl.client.HttpClients; |
||||
import org.apache.http.util.EntityUtils; |
||||
|
||||
import java.net.HttpURLConnection; |
||||
import java.nio.charset.StandardCharsets; |
||||
import java.util.List; |
||||
import java.util.Map; |
||||
import java.util.Objects; |
||||
import java.util.stream.Collectors; |
||||
|
||||
import org.asynchttpclient.Dsl; |
||||
import org.asynchttpclient.ws.WebSocket; |
||||
import org.asynchttpclient.ws.WebSocketListener; |
||||
import org.asynchttpclient.ws.WebSocketUpgradeHandler; |
||||
import org.slf4j.Logger; |
||||
|
||||
/** |
||||
* TIS DataX Task |
||||
**/ |
||||
public class TISTask extends AbstractTask { |
||||
|
||||
public static final String WS_REQUEST_PATH = "/tjs/download/logfeedback"; |
||||
public static final String KEY_POOL_VAR_TIS_HOST = "tisHost"; |
||||
private final TaskRequest taskExecutionContext; |
||||
|
||||
private TISParameters tisParameters; |
||||
|
||||
public TISTask(TaskRequest taskExecutionContext, Logger logger) { |
||||
super(taskExecutionContext, logger); |
||||
this.taskExecutionContext = taskExecutionContext; |
||||
} |
||||
|
||||
@Override |
||||
public void init() { |
||||
super.init(); |
||||
logger.info("tis task params {}", taskExecutionContext.getTaskParams()); |
||||
tisParameters = JSONUtils.parseObject(taskExecutionContext.getTaskParams(), TISParameters.class); |
||||
if (!tisParameters.checkParameters()) { |
||||
throw new RuntimeException("datax task params is not valid"); |
||||
} |
||||
} |
||||
|
||||
@Override |
||||
public void handle() throws Exception { |
||||
// Trigger TIS DataX pipeline
|
||||
logger.info("start execute TIS task"); |
||||
long startTime = System.currentTimeMillis(); |
||||
String targetJobName = this.tisParameters.getTargetJobName(); |
||||
final String tisHost = taskExecutionContext.getDefinedParams().get(KEY_POOL_VAR_TIS_HOST); |
||||
if (StringUtils.isEmpty(tisHost)) { |
||||
throw new IllegalStateException("global var '" + KEY_POOL_VAR_TIS_HOST + "' can not be empty"); |
||||
} |
||||
try { |
||||
final String triggerUrl = String.format("http://%s/tjs/coredefine/coredefine.ajax", tisHost); |
||||
final String getStatusUrl = String.format("http://%s/tjs/config/config.ajax?action=collection_action&emethod=get_task_status", tisHost); |
||||
HttpPost post = new HttpPost(triggerUrl); |
||||
post.addHeader("appname", targetJobName); |
||||
addFormUrlencoded(post); |
||||
StringEntity entity = new StringEntity("action=datax_action&emethod=trigger_fullbuild_task", StandardCharsets.UTF_8); |
||||
post.setEntity(entity); |
||||
BizResult ajaxResult = null; |
||||
ExecResult execState = null; |
||||
int taskId; |
||||
WebSocket webSocket = null; |
||||
try (CloseableHttpClient client = HttpClients.createDefault(); |
||||
// trigger to start TIS dataX task
|
||||
CloseableHttpResponse response = client.execute(post)) { |
||||
ajaxResult = processResponse(triggerUrl, response, BizResult.class); |
||||
if (!ajaxResult.isSuccess()) { |
||||
List<String> errormsg = ajaxResult.getErrormsg(); |
||||
StringBuffer errs = new StringBuffer(); |
||||
if (CollectionUtils.isNotEmpty(errormsg)) { |
||||
errs.append(",errs:").append(errormsg.stream().collect(Collectors.joining(","))); |
||||
} |
||||
throw new Exception("trigger TIS job faild taskName:" + targetJobName + errs.toString()); |
||||
} |
||||
taskId = ajaxResult.getBizresult().getTaskid(); |
||||
|
||||
webSocket = receiveRealtimeLog(tisHost, targetJobName, taskId); |
||||
|
||||
setAppIds(String.valueOf(taskId)); |
||||
|
||||
CloseableHttpResponse status = null; |
||||
|
||||
while (true) { |
||||
try { |
||||
post = new HttpPost(getStatusUrl); |
||||
entity = new StringEntity("{\n taskid: " + taskId + "\n, log: false }", StandardCharsets.UTF_8); |
||||
post.setEntity(entity); |
||||
status = client.execute(post); |
||||
StatusResult execStatus = processResponse(getStatusUrl, status, StatusResult.class); |
||||
Map bizresult = execStatus.getBizresult(); |
||||
Map s = (Map) bizresult.get("status"); |
||||
execState = ExecResult.parse((Integer) s.get("state")); |
||||
if (execState == ExecResult.SUCCESS || execState == ExecResult.FAILD) { |
||||
break; |
||||
} |
||||
Thread.sleep(3000); |
||||
} finally { |
||||
status.close(); |
||||
} |
||||
} |
||||
} finally { |
||||
try { |
||||
webSocket.sendCloseFrame(); |
||||
} catch (Throwable e) { |
||||
logger.warn(e.getMessage(), e); |
||||
} |
||||
} |
||||
|
||||
long costTime = System.currentTimeMillis() - startTime; |
||||
logger.info("TIS task: {},taskId:{} costTime : {} milliseconds, statusCode : {}", |
||||
targetJobName, taskId, costTime, (execState == ExecResult.SUCCESS) ? "'success'" : "'failure'"); |
||||
setExitStatusCode((execState == ExecResult.SUCCESS) ? TaskConstants.EXIT_CODE_SUCCESS : TaskConstants.EXIT_CODE_FAILURE); |
||||
} catch (Exception e) { |
||||
logger.error("execute TIS dataX faild,TIS task name:" + targetJobName, e); |
||||
setExitStatusCode(TaskConstants.EXIT_CODE_FAILURE); |
||||
} |
||||
} |
||||
|
||||
private void addFormUrlencoded(HttpPost post) { |
||||
post.addHeader("content-type", "application/x-www-form-urlencoded"); |
||||
} |
||||
|
||||
@Override |
||||
public void cancelApplication(boolean status) throws Exception { |
||||
super.cancelApplication(status); |
||||
} |
||||
|
||||
private WebSocket receiveRealtimeLog(final String tisHost, String dataXName, int taskId) throws InterruptedException, java.util.concurrent.ExecutionException { |
||||
|
||||
WebSocketUpgradeHandler.Builder upgradeHandlerBuilder |
||||
= new WebSocketUpgradeHandler.Builder(); |
||||
WebSocketUpgradeHandler wsHandler = upgradeHandlerBuilder |
||||
.addWebSocketListener(new WebSocketListener() { |
||||
@Override |
||||
public void onOpen(WebSocket websocket) { |
||||
// WebSocket connection opened
|
||||
} |
||||
|
||||
@Override |
||||
public void onClose(WebSocket websocket, int code, String reason) { |
||||
// WebSocket connection closed
|
||||
} |
||||
|
||||
public void onTextFrame(String payload, boolean finalFragment, int rsv) { |
||||
ExecLog execLog = JSONUtils.parseObject(payload, ExecLog.class); |
||||
logger.info(execLog.getMsg()); |
||||
} |
||||
|
||||
@Override |
||||
public void onError(Throwable t) { |
||||
// WebSocket connection error
|
||||
logger.error(t.getMessage(), t); |
||||
} |
||||
}).build(); |
||||
WebSocket webSocketClient = Dsl.asyncHttpClient() |
||||
.prepareGet(String.format("ws://%s" + WS_REQUEST_PATH, tisHost)) |
||||
// .addHeader("header_name", "header_value")
|
||||
.addQueryParam("logtype", "full") |
||||
.addQueryParam("collection", dataXName) |
||||
.addQueryParam("taskid", String.valueOf(taskId)) |
||||
.setRequestTimeout(5000) |
||||
.execute(wsHandler) |
||||
.get(); |
||||
|
||||
return webSocketClient; |
||||
} |
||||
|
||||
private <T extends AjaxResult> T processResponse(String applyUrl, CloseableHttpResponse response, Class<T> clazz) throws Exception { |
||||
StatusLine resStatus = response.getStatusLine(); |
||||
if (HttpURLConnection.HTTP_OK != resStatus.getStatusCode()) { |
||||
throw new IllegalStateException("request server " + applyUrl + " faild:" + resStatus.getReasonPhrase()); |
||||
} |
||||
HttpEntity entity = response.getEntity(); |
||||
String resp = EntityUtils.toString(entity, StandardCharsets.UTF_8); |
||||
T result = JSONUtils.parseObject(resp, clazz); |
||||
return result; |
||||
} |
||||
|
||||
@Override |
||||
public AbstractParameters getParameters() { |
||||
Objects.requireNonNull(this.tisParameters, "tisParameters can not be null"); |
||||
return this.tisParameters; |
||||
} |
||||
|
||||
private static class BizResult extends AjaxResult<TriggerBuildResult> { |
||||
private TriggerBuildResult bizresult; |
||||
|
||||
public TriggerBuildResult getBizresult() { |
||||
return this.bizresult; |
||||
} |
||||
|
||||
public void setBizresult(TriggerBuildResult bizresult) { |
||||
this.bizresult = bizresult; |
||||
} |
||||
} |
||||
|
||||
private static class StatusResult extends AjaxResult<Map> { |
||||
private Map bizresult; |
||||
|
||||
public Map getBizresult() { |
||||
return this.bizresult; |
||||
} |
||||
|
||||
public void setBizresult(Map bizresult) { |
||||
this.bizresult = bizresult; |
||||
} |
||||
} |
||||
|
||||
private abstract static class AjaxResult<T> { |
||||
|
||||
private boolean success; |
||||
|
||||
private List<String> errormsg; |
||||
|
||||
private List<String> msg; |
||||
|
||||
public abstract T getBizresult(); |
||||
|
||||
public boolean isSuccess() { |
||||
return success; |
||||
} |
||||
|
||||
public void setSuccess(boolean success) { |
||||
this.success = success; |
||||
} |
||||
|
||||
public List<String> getErrormsg() { |
||||
return this.errormsg; |
||||
} |
||||
|
||||
public void setErrormsg(List<String> errormsg) { |
||||
this.errormsg = errormsg; |
||||
} |
||||
|
||||
public List<String> getMsg() { |
||||
return this.msg; |
||||
} |
||||
|
||||
public void setMsg(List<String> msg) { |
||||
this.msg = msg; |
||||
} |
||||
|
||||
} |
||||
|
||||
private static class TriggerBuildResult { |
||||
private int taskid; |
||||
|
||||
public int getTaskid() { |
||||
return taskid; |
||||
} |
||||
|
||||
public void setTaskid(int taskid) { |
||||
this.taskid = taskid; |
||||
} |
||||
} |
||||
|
||||
private enum ExecResult { |
||||
|
||||
SUCCESS(1), FAILD(-1), DOING(2), ASYN_DOING(22), CANCEL(3); |
||||
|
||||
private final int value; |
||||
|
||||
public static ExecResult parse(int value) { |
||||
for (ExecResult r : values()) { |
||||
if (r.value == value) { |
||||
return r; |
||||
} |
||||
} |
||||
throw new IllegalStateException("vale:" + value + " is illegal"); |
||||
} |
||||
|
||||
private ExecResult(int value) { |
||||
this.value = value; |
||||
} |
||||
|
||||
public int getValue() { |
||||
return this.value; |
||||
} |
||||
} |
||||
|
||||
private static class ExecLog { |
||||
private String logType; |
||||
private String msg; |
||||
private int taskId; |
||||
|
||||
public String getLogType() { |
||||
return logType; |
||||
} |
||||
|
||||
public void setLogType(String logType) { |
||||
this.logType = logType; |
||||
} |
||||
|
||||
public String getMsg() { |
||||
return msg; |
||||
} |
||||
|
||||
public void setMsg(String msg) { |
||||
this.msg = msg; |
||||
} |
||||
|
||||
public int getTaskId() { |
||||
return taskId; |
||||
} |
||||
|
||||
public void setTaskId(int taskId) { |
||||
this.taskId = taskId; |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,34 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.plugin.task.tis; |
||||
|
||||
import org.apache.dolphinscheduler.spi.task.AbstractTask; |
||||
import org.apache.dolphinscheduler.spi.task.TaskChannel; |
||||
import org.apache.dolphinscheduler.spi.task.TaskRequest; |
||||
|
||||
public class TISTaskChannel implements TaskChannel { |
||||
@Override |
||||
public void cancelApplication(boolean status) { |
||||
|
||||
} |
||||
|
||||
@Override |
||||
public AbstractTask createTask(TaskRequest taskRequest, org.slf4j.Logger logger) { |
||||
return new TISTask(taskRequest, logger); |
||||
} |
||||
} |
||||
@ -0,0 +1,51 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.plugin.task.tis; |
||||
|
||||
import org.apache.dolphinscheduler.spi.params.InputParam; |
||||
import org.apache.dolphinscheduler.spi.params.base.PluginParams; |
||||
import org.apache.dolphinscheduler.spi.params.base.Validate; |
||||
import org.apache.dolphinscheduler.spi.task.TaskChannel; |
||||
import org.apache.dolphinscheduler.spi.task.TaskChannelFactory; |
||||
|
||||
import java.util.Arrays; |
||||
import java.util.List; |
||||
|
||||
/** |
||||
* TIS endpoint |
||||
**/ |
||||
public class TISTaskChannelFactory implements TaskChannelFactory { |
||||
|
||||
@Override |
||||
public TaskChannel create() { |
||||
return new TISTaskChannel(); |
||||
} |
||||
|
||||
@Override |
||||
public String getName() { |
||||
return "TIS"; |
||||
} |
||||
|
||||
@Override |
||||
public List<PluginParams> getParams() { |
||||
InputParam webHookParam = InputParam.newBuilder(TISParamsConstants.NAME_TARGET_JOB_NAME, TISParamsConstants.TARGET_JOB_NAME) |
||||
.addValidate(Validate.newBuilder().setRequired(true).build()) |
||||
.build(); |
||||
return Arrays.asList(webHookParam); |
||||
} |
||||
} |
||||
@ -0,0 +1,142 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.plugin.task.tis; |
||||
|
||||
import static com.github.dreamhead.moco.Moco.pathResource; |
||||
import static com.github.dreamhead.moco.MocoJsonRunner.jsonHttpServer; |
||||
import static com.github.dreamhead.moco.Runner.running; |
||||
|
||||
import org.apache.dolphinscheduler.server.worker.task.TaskProps; |
||||
import org.apache.dolphinscheduler.spi.task.ExecutionStatus; |
||||
import org.apache.dolphinscheduler.spi.task.TaskRequest; |
||||
|
||||
import org.apache.commons.io.IOUtils; |
||||
|
||||
import java.io.InputStream; |
||||
import java.nio.charset.StandardCharsets; |
||||
import java.util.Collections; |
||||
import java.util.Date; |
||||
import java.util.Map; |
||||
import java.util.Objects; |
||||
import java.util.UUID; |
||||
|
||||
import org.junit.Assert; |
||||
import org.junit.Before; |
||||
import org.junit.Test; |
||||
import org.mockito.Mockito; |
||||
import org.powermock.api.mockito.PowerMockito; |
||||
import org.slf4j.Logger; |
||||
import org.slf4j.LoggerFactory; |
||||
|
||||
import com.github.dreamhead.moco.HttpServer; |
||||
|
||||
public class TISTaskTest { |
||||
private static final Logger logger = LoggerFactory.getLogger(TISTaskTest.class); |
||||
private TISTask tisTask; |
||||
|
||||
private TaskRequest taskExecutionContext; |
||||
|
||||
@Before |
||||
public void before() throws Exception { |
||||
|
||||
TaskProps props = new TaskProps(); |
||||
props.setExecutePath("/tmp"); |
||||
props.setTaskAppId(String.valueOf(System.currentTimeMillis())); |
||||
props.setTaskInstanceId(1); |
||||
props.setTenantCode("1"); |
||||
props.setEnvFile(".dolphinscheduler_env.sh"); |
||||
props.setTaskStartTime(new Date()); |
||||
props.setTaskTimeout(0); |
||||
props.setTaskParams("{\"targetJobName\":\"mysql_elastic\"}"); |
||||
|
||||
taskExecutionContext = Mockito.mock(TaskRequest.class); |
||||
Mockito.when(taskExecutionContext.getTaskParams()).thenReturn(props.getTaskParams()); |
||||
Mockito.when(taskExecutionContext.getExecutePath()).thenReturn("/tmp"); |
||||
Mockito.when(taskExecutionContext.getTaskAppId()).thenReturn(UUID.randomUUID().toString()); |
||||
Mockito.when(taskExecutionContext.getTenantCode()).thenReturn("root"); |
||||
Mockito.when(taskExecutionContext.getStartTime()).thenReturn(new Date()); |
||||
Mockito.when(taskExecutionContext.getTaskTimeout()).thenReturn(10000); |
||||
Mockito.when(taskExecutionContext.getLogPath()).thenReturn("/tmp/dx"); |
||||
// Mockito.when(taskExecutionContext.getVarPool())
|
||||
// .thenReturn("[{\"direct\":\"IN\",\"prop\":\"" + TISTask.KEY_POOL_VAR_TIS_HOST + "\",\"type\":\"VARCHAR\",\"value\":\"127.0.0.1:8080\"}]");
|
||||
Map<String, String> gloabParams = Collections.singletonMap(TISTask.KEY_POOL_VAR_TIS_HOST, "127.0.0.1:8080"); |
||||
Mockito.when(taskExecutionContext.getDefinedParams()).thenReturn(gloabParams); |
||||
|
||||
tisTask = PowerMockito.spy(new TISTask(taskExecutionContext, logger)); |
||||
tisTask.init(); |
||||
|
||||
} |
||||
|
||||
/** |
||||
* Method: DataxTask() |
||||
*/ |
||||
@Test |
||||
public void testDataxTask() |
||||
throws Exception { |
||||
TaskProps props = new TaskProps(); |
||||
props.setExecutePath("/tmp"); |
||||
props.setTaskAppId(String.valueOf(System.currentTimeMillis())); |
||||
props.setTaskInstanceId(1); |
||||
props.setTenantCode("1"); |
||||
Assert.assertNotNull(new TISTask(null, logger)); |
||||
} |
||||
|
||||
@Test |
||||
public void testInit() |
||||
throws Exception { |
||||
try { |
||||
tisTask.init(); |
||||
} catch (Exception e) { |
||||
Assert.fail(e.getMessage()); |
||||
} |
||||
} |
||||
|
||||
@Test |
||||
public void testHandle() |
||||
throws Exception { |
||||
HttpServer server = jsonHttpServer(8080, pathResource("org/apache/dolphinscheduler/plugin/task/tis/TISTaskTest.json")); |
||||
|
||||
running(server, () -> { |
||||
tisTask.handle(); |
||||
|
||||
Assert.assertEquals("TIS execute be success", ExecutionStatus.SUCCESS, tisTask.getExitStatus()); |
||||
}); |
||||
} |
||||
|
||||
private String loadResContent(String resName) { |
||||
try (InputStream i = this.getClass().getResourceAsStream(resName)) { |
||||
Objects.requireNonNull(i, "resource " + resName + " relevant stream content can not be null"); |
||||
String content = IOUtils.toString(i, StandardCharsets.UTF_8); |
||||
|
||||
return content; |
||||
} catch (Exception e) { |
||||
throw new RuntimeException(e); |
||||
} |
||||
} |
||||
|
||||
@Test |
||||
public void testCancelApplication() |
||||
throws Exception { |
||||
try { |
||||
tisTask.cancelApplication(true); |
||||
} catch (Exception e) { |
||||
Assert.fail(e.getMessage()); |
||||
} |
||||
} |
||||
|
||||
} |
||||
@ -0,0 +1,60 @@
|
||||
[ |
||||
{ |
||||
"description": "trigger TIS task execute", |
||||
"request": { |
||||
"uri": "/tjs/coredefine/coredefine.ajax", |
||||
"method": "post", |
||||
"headers": { |
||||
"Content-Type": "application/x-www-form-urlencoded", |
||||
"appname": "mysql_elastic" |
||||
}, |
||||
"text": "action=datax_action&emethod=trigger_fullbuild_task" |
||||
}, |
||||
"response": { |
||||
"text": "{\n \"success\":true,\n \"errormsg\":[],\n \"msg\":[],\n \"bizresult\":{\"taskid\": \"1087\"}\n}" |
||||
} |
||||
}, |
||||
{ |
||||
"description": "Get TIS task execute status", |
||||
"request": { |
||||
"uri": "/tjs/config/config.ajax", |
||||
"method": "post", |
||||
"headers": { |
||||
"Content-Type": "text/plain; charset=UTF-8" |
||||
}, |
||||
"queries": { |
||||
"action": "collection_action", |
||||
"emethod": "get_task_status" |
||||
}, |
||||
"text": "{\n taskid: 1087\n, log: false }" |
||||
}, |
||||
"response": { |
||||
"seq": [ |
||||
{ |
||||
"text": "{\n \"success\": true,\n \"errormsg\": [\n \"err1\"\n ],\n \"bizresult\": {\n \"status\": {\n \"state\": 2\n }\n }\n}" |
||||
}, |
||||
{ |
||||
"text": "{\n \"success\": true,\n \"errormsg\": [\n \"err1\"\n ],\n \"bizresult\": {\n \"status\": {\n \"state\": 1\n }\n }\n}" |
||||
} |
||||
] |
||||
} |
||||
}, |
||||
{ |
||||
"websocket": { |
||||
"uri": "/tjs/download/logfeedback", |
||||
"connected": "connected", |
||||
"sessions": [ |
||||
{ |
||||
"request": { |
||||
"text": "logtype=full&collection=mysql_elastic&taskid=1087" |
||||
}, |
||||
"response": { |
||||
"broadcast": { |
||||
"content": "{\n \"logType\": \"FULL\",\n \"msg\": \"message 1\",\n \"taskId\": \"1087\"\n}" |
||||
} |
||||
} |
||||
} |
||||
] |
||||
} |
||||
} |
||||
] |
||||
@ -0,0 +1,225 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0 |
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
<template> |
||||
<div class="datax-model"> |
||||
<m-list-box> |
||||
<div slot="text">{{$t('TargetJobName')}}</div> |
||||
<div slot="content"> |
||||
<el-input |
||||
type="input" |
||||
size="small" |
||||
v-model="targetJobName" |
||||
:placeholder="$t('Please enter TIS DataX job name')"> |
||||
</el-input> |
||||
</div> |
||||
</m-list-box> |
||||
</div> |
||||
</template> |
||||
<script> |
||||
import _ from 'lodash' |
||||
import mListBox from './_source/listBox' |
||||
import disabledState from '@/module/mixin/disabledState' |
||||
|
||||
export default { |
||||
name: 'tis', |
||||
|
||||
data () { |
||||
return { |
||||
// target table |
||||
targetJobName: '' |
||||
} |
||||
}, |
||||
mixins: [disabledState], |
||||
props: { |
||||
backfillItem: Object, |
||||
createNodeId: Number |
||||
}, |
||||
methods: { |
||||
setEditorVal () { |
||||
// this.item = editor.getValue() |
||||
// this.scriptBoxDialog = true |
||||
}, |
||||
getSriptBoxValue (val) { |
||||
}, |
||||
/** |
||||
* return pre statements |
||||
*/ |
||||
_onPreStatements (a) { |
||||
this.preStatements = a |
||||
}, |
||||
/** |
||||
* return post statements |
||||
*/ |
||||
_onPostStatements (a) { |
||||
this.postStatements = a |
||||
}, |
||||
/** |
||||
* return localParams |
||||
*/ |
||||
_onLocalParams (a) { |
||||
this.localParams = a |
||||
}, |
||||
/** |
||||
* verification |
||||
*/ |
||||
_verification () { |
||||
// storage |
||||
this.$emit('on-params', { |
||||
targetJobName: this.targetJobName |
||||
}) |
||||
|
||||
return true |
||||
}, |
||||
/** |
||||
* Processing code highlighting |
||||
*/ |
||||
_handlerEditor () { |
||||
// this._destroyEditor() |
||||
|
||||
// editor |
||||
// editor = codemirror('code-sql-mirror', { |
||||
// mode: 'sql', |
||||
// readOnly: this.isDetails |
||||
// }) |
||||
// |
||||
// this.keypress = () => { |
||||
// if (!editor.getOption('readOnly')) { |
||||
// editor.showHint({ |
||||
// completeSingle: false |
||||
// }) |
||||
// } |
||||
// } |
||||
// |
||||
// // Monitor keyboard |
||||
// editor.on('keypress', this.keypress) |
||||
// |
||||
// editor.on('changes', () => { |
||||
// this._cacheParams() |
||||
// }) |
||||
// |
||||
// editor.setValue(this.sql) |
||||
// |
||||
// return editor |
||||
}, |
||||
// _handlerJsonEditor () { |
||||
// this._destroyJsonEditor() |
||||
// |
||||
// // jsonEditor |
||||
// jsonEditor = codemirror('code-json-mirror', { |
||||
// mode: 'json', |
||||
// readOnly: this.isDetails |
||||
// }) |
||||
// |
||||
// this.keypress = () => { |
||||
// if (!jsonEditor.getOption('readOnly')) { |
||||
// jsonEditor.showHint({ |
||||
// completeSingle: false |
||||
// }) |
||||
// } |
||||
// } |
||||
// |
||||
// // Monitor keyboard |
||||
// jsonEditor.on('keypress', this.keypress) |
||||
// |
||||
// jsonEditor.on('changes', () => { |
||||
// // this._cacheParams() |
||||
// }) |
||||
// |
||||
// jsonEditor.setValue(this.json) |
||||
// |
||||
// return jsonEditor |
||||
// }, |
||||
_cacheParams () { |
||||
this.$emit('on-cache-params', { |
||||
// dsType: this.dsType, |
||||
// dataSource: this.rtDatasource, |
||||
// dtType: this.dtType, |
||||
// dataTarget: this.rtDatatarget, |
||||
// sql: editor ? editor.getValue() : '', |
||||
targetJobName: this.targetJobName |
||||
// jobSpeedByte: this.jobSpeedByte * 1024, |
||||
// jobSpeedRecord: this.jobSpeedRecord, |
||||
// preStatements: this.preStatements, |
||||
// postStatements: this.postStatements, |
||||
// xms: +this.xms, |
||||
// xmx: +this.xmx |
||||
}) |
||||
} |
||||
// _destroyEditor () { |
||||
// if (editor) { |
||||
// editor.toTextArea() // Uninstall |
||||
// editor.off($('.code-sql-mirror'), 'keypress', this.keypress) |
||||
// editor.off($('.code-sql-mirror'), 'changes', this.changes) |
||||
// } |
||||
// }, |
||||
// _destroyJsonEditor () { |
||||
// if (jsonEditor) { |
||||
// jsonEditor.toTextArea() // Uninstall |
||||
// jsonEditor.off($('.code-json-mirror'), 'keypress', this.keypress) |
||||
// jsonEditor.off($('.code-json-mirror'), 'changes', this.changes) |
||||
// } |
||||
// } |
||||
}, |
||||
created () { |
||||
let o = this.backfillItem |
||||
|
||||
// Non-null objects represent backfill |
||||
if (!_.isEmpty(o)) { |
||||
// backfill |
||||
this.targetJobName = o.params.targetJobName || '' |
||||
} |
||||
}, |
||||
mounted () { |
||||
// if (this.customConfig) { |
||||
// setTimeout(() => { |
||||
// this._handlerJsonEditor() |
||||
// }, 200) |
||||
// } else { |
||||
// setTimeout(() => { |
||||
// this._handlerEditor() |
||||
// }, 200) |
||||
// } |
||||
}, |
||||
destroyed () { |
||||
// /** |
||||
// * Destroy the editor instance |
||||
// */ |
||||
// if (editor) { |
||||
// editor.toTextArea() // Uninstall |
||||
// editor.off($('.code-sql-mirror'), 'keypress', this.keypress) |
||||
// } |
||||
// if (jsonEditor) { |
||||
// jsonEditor.toTextArea() // Uninstall |
||||
// jsonEditor.off($('.code-json-mirror'), 'keypress', this.keypress) |
||||
// } |
||||
}, |
||||
watch: { |
||||
// Watch the cacheParams |
||||
cacheParams (val) { |
||||
this._cacheParams() |
||||
} |
||||
}, |
||||
computed: { |
||||
cacheParams () { |
||||
return { |
||||
targetJobName: this.targetJobName |
||||
} |
||||
} |
||||
}, |
||||
components: { mListBox } |
||||
} |
||||
</script> |
||||
|
After Width: | Height: | Size: 2.2 KiB |
Loading…
Reference in new issue