Browse Source

Don't source env file in task, this will override tenant's environment (#13028)

3.2.0-release
Wenjun Ruan 2 years ago committed by GitHub
parent
commit
32a8241ef2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 8
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ShellCommandExecutor.java
  2. 5
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskExecutionContext.java
  3. 4
      dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecuteRunnable.java
  4. 16
      script/env/dolphinscheduler_env.sh

8
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ShellCommandExecutor.java

@ -95,10 +95,6 @@ public class ShellCommandExecutor extends AbstractCommandExecutor {
sb.append("cd /d %~dp0").append(System.lineSeparator()); sb.append("cd /d %~dp0").append(System.lineSeparator());
if (StringUtils.isNotBlank(taskRequest.getEnvironmentConfig())) { if (StringUtils.isNotBlank(taskRequest.getEnvironmentConfig())) {
sb.append(taskRequest.getEnvironmentConfig()).append(System.lineSeparator()); sb.append(taskRequest.getEnvironmentConfig()).append(System.lineSeparator());
} else {
if (taskRequest.getEnvFile() != null) {
sb.append("call ").append(taskRequest.getEnvFile()).append(System.lineSeparator());
}
} }
} else { } else {
sb.append("#!/bin/bash").append(System.lineSeparator()); sb.append("#!/bin/bash").append(System.lineSeparator());
@ -106,10 +102,6 @@ public class ShellCommandExecutor extends AbstractCommandExecutor {
sb.append("cd $BASEDIR").append(System.lineSeparator()); sb.append("cd $BASEDIR").append(System.lineSeparator());
if (StringUtils.isNotBlank(taskRequest.getEnvironmentConfig())) { if (StringUtils.isNotBlank(taskRequest.getEnvironmentConfig())) {
sb.append(taskRequest.getEnvironmentConfig()).append(System.lineSeparator()); sb.append(taskRequest.getEnvironmentConfig()).append(System.lineSeparator());
} else {
if (taskRequest.getEnvFile() != null) {
sb.append("source ").append(taskRequest.getEnvFile()).append(System.lineSeparator());
}
} }
} }
sb.append(execCommand); sb.append(execCommand);

5
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskExecutionContext.java

@ -166,11 +166,6 @@ public class TaskExecutionContext implements Serializable {
*/ */
private String taskParams; private String taskParams;
/**
* envFile
*/
private String envFile;
/** /**
* environmentConfig * environmentConfig
*/ */

4
dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecuteRunnable.java

@ -196,10 +196,6 @@ public abstract class WorkerTaskExecuteRunnable implements Runnable {
taskExecutionContext.setStartTime(taskStartTime); taskExecutionContext.setStartTime(taskStartTime);
logger.info("Set task startTime: {}", taskStartTime); logger.info("Set task startTime: {}", taskStartTime);
String systemEnvPath = CommonUtils.getSystemEnvPath();
taskExecutionContext.setEnvFile(systemEnvPath);
logger.info("Set task envFile: {}", systemEnvPath);
String taskAppId = String.format("%s_%s", taskExecutionContext.getProcessInstanceId(), String taskAppId = String.format("%s_%s", taskExecutionContext.getProcessInstanceId(),
taskExecutionContext.getTaskInstanceId()); taskExecutionContext.getTaskInstanceId());
taskExecutionContext.setTaskAppId(taskAppId); taskExecutionContext.setTaskAppId(taskAppId);

16
script/env/dolphinscheduler_env.sh vendored

@ -15,26 +15,10 @@
# limitations under the License. # limitations under the License.
# #
# JAVA_HOME, will use it to start DolphinScheduler server
export JAVA_HOME=${JAVA_HOME:-/opt/java/openjdk}
# Never put sensitive config such as database password here in your production environment, # Never put sensitive config such as database password here in your production environment,
# this file will be sourced everytime a new task is executed. # this file will be sourced everytime a new task is executed.
# Tasks related configurations, need to change the configuration if you use the related tasks.
export HADOOP_HOME=${HADOOP_HOME:-/opt/soft/hadoop}
export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/opt/soft/hadoop/etc/hadoop}
export SPARK_HOME=${SPARK_HOME:-/opt/soft/spark}
export PYTHON_HOME=${PYTHON_HOME:-/opt/soft/python}
export HIVE_HOME=${HIVE_HOME:-/opt/soft/hive}
export FLINK_HOME=${FLINK_HOME:-/opt/soft/flink}
export DATAX_HOME=${DATAX_HOME:-/opt/soft/datax}
export SEATUNNEL_HOME=${SEATUNNEL_HOME:-/opt/soft/seatunnel}
export CHUNJUN_HOME=${CHUNJUN_HOME:-/opt/soft/chunjun}
export LINKIS_HOME=${LINKIS_HOME:-/opt/soft/linkis}
export PATH=$HADOOP_HOME/bin:$SPARK_HOME/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$SEATUNNEL_HOME/bin:$CHUNJUN_HOME/bin:$PATH
# applicationId auto collection related configuration, the following configurations are unnecessary if setting appId.collect=log # applicationId auto collection related configuration, the following configurations are unnecessary if setting appId.collect=log
#export HADOOP_CLASSPATH=`hadoop classpath`:${DOLPHINSCHEDULER_HOME}/tools/libs/* #export HADOOP_CLASSPATH=`hadoop classpath`:${DOLPHINSCHEDULER_HOME}/tools/libs/*
#export SPARK_DIST_CLASSPATH=$HADOOP_CLASSPATH:$SPARK_DIST_CLASS_PATH #export SPARK_DIST_CLASSPATH=$HADOOP_CLASSPATH:$SPARK_DIST_CLASS_PATH

Loading…
Cancel
Save