You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
216 lines
8.0 KiB
216 lines
8.0 KiB
/* |
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
|
* contributor license agreements. See the NOTICE file distributed with |
|
* this work for additional information regarding copyright ownership. |
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
|
* (the "License"); you may not use this file except in compliance with |
|
* the License. You may obtain a copy of the License at |
|
* |
|
* http://www.apache.org/licenses/LICENSE-2.0 |
|
* |
|
* Unless required by applicable law or agreed to in writing, software |
|
* distributed under the License is distributed on an "AS IS" BASIS, |
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
* See the License for the specific language governing permissions and |
|
* limitations under the License. |
|
*/ |
|
|
|
package org.apache.dolphinscheduler.plugin.task.seatunnel; |
|
|
|
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_FAILURE; |
|
import static org.apache.dolphinscheduler.plugin.task.seatunnel.Constants.CONFIG_OPTIONS; |
|
|
|
import org.apache.dolphinscheduler.common.utils.JSONUtils; |
|
import org.apache.dolphinscheduler.plugin.task.api.AbstractRemoteTask; |
|
import org.apache.dolphinscheduler.plugin.task.api.ShellCommandExecutor; |
|
import org.apache.dolphinscheduler.plugin.task.api.TaskCallBack; |
|
import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; |
|
import org.apache.dolphinscheduler.plugin.task.api.TaskException; |
|
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; |
|
import org.apache.dolphinscheduler.plugin.task.api.model.Property; |
|
import org.apache.dolphinscheduler.plugin.task.api.model.TaskResponse; |
|
import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; |
|
import org.apache.dolphinscheduler.plugin.task.api.shell.IShellInterceptorBuilder; |
|
import org.apache.dolphinscheduler.plugin.task.api.shell.ShellInterceptorBuilderFactory; |
|
import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; |
|
|
|
import org.apache.commons.io.FileUtils; |
|
import org.apache.commons.lang3.BooleanUtils; |
|
|
|
import java.io.File; |
|
import java.io.IOException; |
|
import java.nio.charset.StandardCharsets; |
|
import java.nio.file.Files; |
|
import java.nio.file.Paths; |
|
import java.util.ArrayList; |
|
import java.util.Collections; |
|
import java.util.List; |
|
import java.util.Map; |
|
|
|
import lombok.extern.slf4j.Slf4j; |
|
|
|
@Slf4j |
|
public class SeatunnelTask extends AbstractRemoteTask { |
|
|
|
private static final String SEATUNNEL_BIN_DIR = "${SEATUNNEL_HOME}/bin/"; |
|
|
|
/** |
|
* seatunnel parameters |
|
*/ |
|
private SeatunnelParameters seatunnelParameters; |
|
|
|
/** |
|
* shell command executor |
|
*/ |
|
private ShellCommandExecutor shellCommandExecutor; |
|
|
|
/** |
|
* taskExecutionContext |
|
*/ |
|
protected final TaskExecutionContext taskExecutionContext; |
|
|
|
/** |
|
* constructor |
|
* |
|
* @param taskExecutionContext taskExecutionContext |
|
*/ |
|
public SeatunnelTask(TaskExecutionContext taskExecutionContext) { |
|
super(taskExecutionContext); |
|
|
|
this.taskExecutionContext = taskExecutionContext; |
|
this.shellCommandExecutor = new ShellCommandExecutor(this::logHandle, taskExecutionContext); |
|
} |
|
|
|
@Override |
|
public List<String> getApplicationIds() throws TaskException { |
|
return Collections.emptyList(); |
|
} |
|
|
|
@Override |
|
public void init() { |
|
log.info("Intialize SeaTunnel task params {}", JSONUtils.toPrettyJsonString(seatunnelParameters)); |
|
if (seatunnelParameters == null || !seatunnelParameters.checkParameters()) { |
|
throw new TaskException("SeaTunnel task params is not valid"); |
|
} |
|
} |
|
|
|
// todo split handle to submit and track |
|
@Override |
|
public void handle(TaskCallBack taskCallBack) throws TaskException { |
|
try { |
|
// construct process |
|
String command = buildCommand(); |
|
IShellInterceptorBuilder<?, ?> shellActuatorBuilder = ShellInterceptorBuilderFactory.newBuilder() |
|
.appendScript(command); |
|
|
|
TaskResponse commandExecuteResult = shellCommandExecutor.run(shellActuatorBuilder, taskCallBack); |
|
setExitStatusCode(commandExecuteResult.getExitStatusCode()); |
|
setAppIds(String.join(TaskConstants.COMMA, getApplicationIds())); |
|
setProcessId(commandExecuteResult.getProcessId()); |
|
seatunnelParameters.dealOutParam(shellCommandExecutor.getTaskOutputParams()); |
|
} catch (InterruptedException e) { |
|
Thread.currentThread().interrupt(); |
|
log.error("The current SeaTunnel task has been interrupted", e); |
|
setExitStatusCode(EXIT_CODE_FAILURE); |
|
throw new TaskException("The current SeaTunnel task has been interrupted", e); |
|
} catch (Exception e) { |
|
log.error("SeaTunnel task error", e); |
|
setExitStatusCode(EXIT_CODE_FAILURE); |
|
throw new TaskException("Execute Seatunnel task failed", e); |
|
} |
|
} |
|
|
|
@Override |
|
public void submitApplication() throws TaskException { |
|
|
|
} |
|
|
|
@Override |
|
public void trackApplicationStatus() throws TaskException { |
|
|
|
} |
|
|
|
@Override |
|
public void cancelApplication() throws TaskException { |
|
// cancel process |
|
try { |
|
shellCommandExecutor.cancelApplication(); |
|
} catch (Exception e) { |
|
throw new TaskException("cancel application error", e); |
|
} |
|
} |
|
|
|
private String buildCommand() throws Exception { |
|
|
|
List<String> args = new ArrayList<>(); |
|
args.add(SEATUNNEL_BIN_DIR + seatunnelParameters.getStartupScript()); |
|
args.addAll(buildOptions()); |
|
|
|
String command = String.join(" ", args); |
|
log.info("SeaTunnel task command: {}", command); |
|
|
|
return command; |
|
} |
|
|
|
protected List<String> buildOptions() throws Exception { |
|
List<String> args = new ArrayList<>(); |
|
if (BooleanUtils.isTrue(seatunnelParameters.getUseCustom())) { |
|
args.add(CONFIG_OPTIONS); |
|
args.add(buildCustomConfigCommand()); |
|
} else { |
|
seatunnelParameters.getResourceList().forEach(resourceInfo -> { |
|
args.add(CONFIG_OPTIONS); |
|
// TODO: Need further check for refactored resource center |
|
// TODO Currently resourceName is `/xxx.sh`, it has more `/` and needs to be optimized |
|
args.add(resourceInfo.getResourceName().replaceFirst(".*:", "")); |
|
}); |
|
} |
|
return args; |
|
} |
|
|
|
protected String buildCustomConfigCommand() throws Exception { |
|
String config = buildCustomConfigContent(); |
|
String filePath = buildConfigFilePath(); |
|
createConfigFileIfNotExists(config, filePath); |
|
|
|
return filePath; |
|
} |
|
|
|
private String buildCustomConfigContent() { |
|
log.info("raw custom config content : {}", seatunnelParameters.getRawScript()); |
|
String script = seatunnelParameters.getRawScript().replaceAll("\\r\\n", System.lineSeparator()); |
|
script = parseScript(script); |
|
return script; |
|
} |
|
|
|
private String buildConfigFilePath() { |
|
return String.format("%s/seatunnel_%s.conf", taskExecutionContext.getExecutePath(), |
|
taskExecutionContext.getTaskAppId()); |
|
} |
|
|
|
private void createConfigFileIfNotExists(String script, String scriptFile) throws IOException { |
|
log.info("tenantCode :{}, task dir:{}", taskExecutionContext.getTenantCode(), |
|
taskExecutionContext.getExecutePath()); |
|
|
|
if (!Files.exists(Paths.get(scriptFile))) { |
|
log.info("generate script file:{}", scriptFile); |
|
|
|
// write data to file |
|
FileUtils.writeStringToFile(new File(scriptFile), script, StandardCharsets.UTF_8); |
|
} |
|
} |
|
|
|
@Override |
|
public AbstractParameters getParameters() { |
|
return seatunnelParameters; |
|
} |
|
|
|
private String parseScript(String script) { |
|
Map<String, Property> paramsMap = taskExecutionContext.getPrepareParamsMap(); |
|
return ParameterUtils.convertParameterPlaceholders(script, ParameterUtils.convert(paramsMap)); |
|
} |
|
|
|
public void setSeatunnelParameters(SeatunnelParameters seatunnelParameters) { |
|
this.seatunnelParameters = seatunnelParameters; |
|
} |
|
}
|
|
|