diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-env.xml b/ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-env.xml index 8e14716d05..18501c6d0a 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-env.xml +++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-env.xml @@ -112,7 +112,8 @@ export SPARK_HOME2=/opt/soft/spark2 export PYTHON_HOME=/opt/soft/python export JAVA_HOME=/opt/soft/java export HIVE_HOME=/opt/soft/hive -export FLINK_HOME=/opt/soft/flink +export FLINK_HOME=/opt/soft/flink +export DATAX_HOME=/opt/soft/datax content false diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-env.xml b/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-env.xml index 8e14716d05..18501c6d0a 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-env.xml +++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-env.xml @@ -112,7 +112,8 @@ export SPARK_HOME2=/opt/soft/spark2 export PYTHON_HOME=/opt/soft/python export JAVA_HOME=/opt/soft/java export HIVE_HOME=/opt/soft/hive -export FLINK_HOME=/opt/soft/flink +export FLINK_HOME=/opt/soft/flink +export DATAX_HOME=/opt/soft/datax content false diff --git a/docker/build/README.md b/docker/build/README.md index 4d3ac8767b..5a1b28a715 100644 --- a/docker/build/README.md +++ b/docker/build/README.md @@ -209,7 +209,7 @@ This environment variable sets `FLINK_HOME`. The default value is `/opt/soft/fli **`DATAX_HOME`** -This environment variable sets `DATAX_HOME`. The default value is `/opt/soft/datax/bin/datax.py`. +This environment variable sets `DATAX_HOME`. The default value is `/opt/soft/datax`. **`DOLPHINSCHEDULER_DATA_BASEDIR_PATH`** diff --git a/docker/build/README_zh_CN.md b/docker/build/README_zh_CN.md index 1dfc69c5ad..f3b8ea723e 100644 --- a/docker/build/README_zh_CN.md +++ b/docker/build/README_zh_CN.md @@ -209,7 +209,7 @@ DolphinScheduler Docker 容器通过环境变量进行配置,缺省时将会 **`DATAX_HOME`** -配置`dolphinscheduler`的`DATAX_HOME`,默认值 `/opt/soft/datax/bin/datax。py`。 +配置`dolphinscheduler`的`DATAX_HOME`,默认值 `/opt/soft/datax`。 **`DOLPHINSCHEDULER_DATA_BASEDIR_PATH`** diff --git a/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh.tpl b/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh.tpl index 4463a3f227..b9b04eb83d 100755 --- a/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh.tpl +++ b/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh.tpl @@ -25,4 +25,4 @@ export HIVE_HOME=$HIVE_HOME export FLINK_HOME=$FLINK_HOME export DATAX_HOME=$DATAX_HOME -export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH:$FLINK_HOME/bin:$DATAX_HOME:$PATH +export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH diff --git a/docker/build/startup-init-conf.sh b/docker/build/startup-init-conf.sh index b16100c405..80baa92d25 100755 --- a/docker/build/startup-init-conf.sh +++ b/docker/build/startup-init-conf.sh @@ -46,7 +46,7 @@ export PYTHON_HOME=${PYTHON_HOME:-"/usr"} export JAVA_HOME=${JAVA_HOME:-"/usr/lib/jvm/java-1.8-openjdk"} export HIVE_HOME=${HIVE_HOME:-"/opt/soft/hive"} export FLINK_HOME=${FLINK_HOME:-"/opt/soft/flink"} -export DATAX_HOME=${DATAX_HOME:-"/opt/soft/datax/bin/datax.py"} +export DATAX_HOME=${DATAX_HOME:-"/opt/soft/datax"} # common env export DOLPHINSCHEDULER_DATA_BASEDIR_PATH=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH:-"/tmp/dolphinscheduler"} export DOLPHINSCHEDULER_OPTS=${DOLPHINSCHEDULER_OPTS:-""} diff --git a/docker/docker-swarm/docker-compose.yml b/docker/docker-swarm/docker-compose.yml index 5b2cf46b12..8f6a0b85cc 100644 --- a/docker/docker-swarm/docker-compose.yml +++ b/docker/docker-swarm/docker-compose.yml @@ -181,7 +181,7 @@ services: JAVA_HOME: "/usr/lib/jvm/java-1.8-openjdk" HIVE_HOME: "/opt/soft/hive" FLINK_HOME: "/opt/soft/flink" - DATAX_HOME: "/opt/soft/datax/bin/datax.py" + DATAX_HOME: "/opt/soft/datax" DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler DOLPHINSCHEDULER_OPTS: "-Xms1g -Xmx1g -Xmn512m" DATABASE_TYPE: postgresql diff --git a/docker/docker-swarm/docker-stack.yml b/docker/docker-swarm/docker-stack.yml index aab44a388d..d78e7261e7 100644 --- a/docker/docker-swarm/docker-stack.yml +++ b/docker/docker-swarm/docker-stack.yml @@ -175,7 +175,7 @@ services: JAVA_HOME: "/usr/lib/jvm/java-1.8-openjdk" HIVE_HOME: "/opt/soft/hive" FLINK_HOME: "/opt/soft/flink" - DATAX_HOME: "/opt/soft/datax/bin/datax.py" + DATAX_HOME: "/opt/soft/datax" DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler DOLPHINSCHEDULER_OPTS: "-Xms1g -Xmx1g -Xmn512m" DATABASE_TYPE: postgresql diff --git a/docker/kubernetes/dolphinscheduler/values.yaml b/docker/kubernetes/dolphinscheduler/values.yaml index ba6b8eb8f8..af7da46235 100644 --- a/docker/kubernetes/dolphinscheduler/values.yaml +++ b/docker/kubernetes/dolphinscheduler/values.yaml @@ -80,8 +80,8 @@ common: - "export JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk" - "export HIVE_HOME=/opt/soft/hive" - "export FLINK_HOME=/opt/soft/flink" - - "export DATAX_HOME=/opt/soft/datax/bin/datax.py" - - "export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$PATH" + - "export DATAX_HOME=/opt/soft/datax" + - "export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH" DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler" RESOURCE_STORAGE_TYPE: "HDFS" RESOURCE_UPLOAD_PATH: "/dolphinscheduler" diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/DataxParametersTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/DataxParametersTest.java index d6e2f69882..dd11aab260 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/DataxParametersTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/DataxParametersTest.java @@ -27,7 +27,7 @@ public class DataxParametersTest { /** * jvm parameters */ - public static final String JVM_EVN = " --jvm=\"-Xms%sG -Xmx%sG\" "; + public static final String JVM_PARAM = " --jvm=\"-Xms%sG -Xmx%sG\" "; @Test public void testLoadJvmEnv() { @@ -88,6 +88,6 @@ public class DataxParametersTest { public String loadJvmEnvTest(DataxParameters dataXParameters) { int xms = dataXParameters.getXms() < 1 ? 1 : dataXParameters.getXms(); int xmx = dataXParameters.getXmx() < 1 ? 1 : dataXParameters.getXmx(); - return String.format(JVM_EVN, xms, xmx); + return String.format(JVM_PARAM, xms, xmx); } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java index caf487947b..8f0b20d17f 100755 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java @@ -81,15 +81,15 @@ public class DataxTask extends AbstractTask { /** * jvm parameters */ - public static final String JVM_EVN = " --jvm=\"-Xms%sG -Xmx%sG\" "; + public static final String JVM_PARAM = " --jvm=\"-Xms%sG -Xmx%sG\" "; /** * python process(datax only supports version 2.7 by default) */ private static final String DATAX_PYTHON = "python2.7"; /** - * datax home path + * datax path */ - private static final String DATAX_HOME_EVN = "${DATAX_HOME}"; + private static final String DATAX_PATH = "${DATAX_HOME}/bin/datax.py"; /** * datax channel count */ @@ -396,7 +396,7 @@ public class DataxTask extends AbstractTask { StringBuilder sbr = new StringBuilder(); sbr.append(DATAX_PYTHON); sbr.append(" "); - sbr.append(DATAX_HOME_EVN); + sbr.append(DATAX_PATH); sbr.append(" "); sbr.append(loadJvmEnv(dataXParameters)); sbr.append(jobConfigFilePath); @@ -424,7 +424,7 @@ public class DataxTask extends AbstractTask { public String loadJvmEnv(DataxParameters dataXParameters) { int xms = dataXParameters.getXms() < 1 ? 1 : dataXParameters.getXms(); int xmx = dataXParameters.getXmx() < 1 ? 1 : dataXParameters.getXmx(); - return String.format(JVM_EVN, xms, xmx); + return String.format(JVM_PARAM, xms, xmx); } /** diff --git a/script/env/dolphinscheduler_env.sh b/script/env/dolphinscheduler_env.sh index 9fb9f55d70..e62f54e396 100755 --- a/script/env/dolphinscheduler_env.sh +++ b/script/env/dolphinscheduler_env.sh @@ -23,6 +23,6 @@ export PYTHON_HOME=/opt/soft/python export JAVA_HOME=/opt/soft/java export HIVE_HOME=/opt/soft/hive export FLINK_HOME=/opt/soft/flink -export DATAX_HOME=/opt/soft/datax/bin/datax.py +export DATAX_HOME=/opt/soft/datax -export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH:$FLINK_HOME/bin:$DATAX_HOME:$PATH +export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH