[Improvement][Datax] Re-assign DATAX_HOME to /opt/soft/datax instead of /opt/soft/datax/bin/datax.py (#5019)

4 years ago · 1cd62b4a5e
12 changed files with 21 additions and 19 deletions
--- a/ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-env.xml
+++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-env.xml
@ -112,7 +112,8 @@ export SPARK_HOME2=/opt/soft/spark2
 export PYTHON_HOME=/opt/soft/python
 export JAVA_HOME=/opt/soft/java
 export HIVE_HOME=/opt/soft/hive
-export FLINK_HOME=/opt/soft/flink</value>
+export FLINK_HOME=/opt/soft/flink
+export DATAX_HOME=/opt/soft/datax</value>
 		<value-attributes>
 			<type>content</type>
 			<empty-value-valid>false</empty-value-valid>
--- a/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-env.xml
+++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-env.xml
@ -112,7 +112,8 @@ export SPARK_HOME2=/opt/soft/spark2
 export PYTHON_HOME=/opt/soft/python
 export JAVA_HOME=/opt/soft/java
 export HIVE_HOME=/opt/soft/hive
-export FLINK_HOME=/opt/soft/flink</value>
+export FLINK_HOME=/opt/soft/flink
+export DATAX_HOME=/opt/soft/datax</value>
 		<value-attributes>
 			<type>content</type>
 			<empty-value-valid>false</empty-value-valid>
--- a/docker/build/README.md
+++ b/docker/build/README.md
@ -209,7 +209,7 @@ This environment variable sets `FLINK_HOME`. The default value is `/opt/soft/fli

 **`DATAX_HOME`**

-This environment variable sets `DATAX_HOME`. The default value is `/opt/soft/datax/bin/datax.py`.
+This environment variable sets `DATAX_HOME`. The default value is `/opt/soft/datax`.

 **`DOLPHINSCHEDULER_DATA_BASEDIR_PATH`**

--- a/docker/build/README_zh_CN.md
+++ b/docker/build/README_zh_CN.md
@ -209,7 +209,7 @@ DolphinScheduler Docker 容器通过环境变量进行配置，缺省时将会

 **`DATAX_HOME`**

-配置`dolphinscheduler`的`DATAX_HOME`，默认值 `/opt/soft/datax/bin/datax。py`。
+配置`dolphinscheduler`的`DATAX_HOME`，默认值 `/opt/soft/datax`。

 **`DOLPHINSCHEDULER_DATA_BASEDIR_PATH`**

--- a/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh.tpl
+++ b/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh.tpl
@ -25,4 +25,4 @@ export HIVE_HOME=$HIVE_HOME
 export FLINK_HOME=$FLINK_HOME
 export DATAX_HOME=$DATAX_HOME

-export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH:$FLINK_HOME/bin:$DATAX_HOME:$PATH
+export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH
--- a/docker/build/startup-init-conf.sh
+++ b/docker/build/startup-init-conf.sh
@ -46,7 +46,7 @@ export PYTHON_HOME=${PYTHON_HOME:-"/usr"}
 export JAVA_HOME=${JAVA_HOME:-"/usr/lib/jvm/java-1.8-openjdk"}
 export HIVE_HOME=${HIVE_HOME:-"/opt/soft/hive"}
 export FLINK_HOME=${FLINK_HOME:-"/opt/soft/flink"}
-export DATAX_HOME=${DATAX_HOME:-"/opt/soft/datax/bin/datax.py"}
+export DATAX_HOME=${DATAX_HOME:-"/opt/soft/datax"}
 # common env
 export DOLPHINSCHEDULER_DATA_BASEDIR_PATH=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH:-"/tmp/dolphinscheduler"}
 export DOLPHINSCHEDULER_OPTS=${DOLPHINSCHEDULER_OPTS:-""}
--- a/docker/docker-swarm/docker-compose.yml
+++ b/docker/docker-swarm/docker-compose.yml
@ -181,7 +181,7 @@ services:
      JAVA_HOME: "/usr/lib/jvm/java-1.8-openjdk"
      HIVE_HOME: "/opt/soft/hive"
      FLINK_HOME: "/opt/soft/flink"
-      DATAX_HOME: "/opt/soft/datax/bin/datax.py"
+      DATAX_HOME: "/opt/soft/datax"
      DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler
      DOLPHINSCHEDULER_OPTS: "-Xms1g -Xmx1g -Xmn512m"
      DATABASE_TYPE: postgresql
--- a/docker/docker-swarm/docker-stack.yml
+++ b/docker/docker-swarm/docker-stack.yml
@ -175,7 +175,7 @@ services:
      JAVA_HOME: "/usr/lib/jvm/java-1.8-openjdk"
      HIVE_HOME: "/opt/soft/hive"
      FLINK_HOME: "/opt/soft/flink"
-      DATAX_HOME: "/opt/soft/datax/bin/datax.py"
+      DATAX_HOME: "/opt/soft/datax"
      DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler
      DOLPHINSCHEDULER_OPTS: "-Xms1g -Xmx1g -Xmn512m"
      DATABASE_TYPE: postgresql
--- a/docker/kubernetes/dolphinscheduler/values.yaml
+++ b/docker/kubernetes/dolphinscheduler/values.yaml
@ -80,8 +80,8 @@ common:
    - "export JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk"
    - "export HIVE_HOME=/opt/soft/hive"
    - "export FLINK_HOME=/opt/soft/flink"
-    - "export DATAX_HOME=/opt/soft/datax/bin/datax.py"
-    - "export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$PATH"
+    - "export DATAX_HOME=/opt/soft/datax"
+    - "export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH"
    DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler"
    RESOURCE_STORAGE_TYPE: "HDFS"
    RESOURCE_UPLOAD_PATH: "/dolphinscheduler"
--- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/DataxParametersTest.java
+++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/DataxParametersTest.java
@ -27,7 +27,7 @@ public class DataxParametersTest {
    /**
     * jvm parameters
     */
-    public static final String JVM_EVN = " --jvm=\"-Xms%sG -Xmx%sG\" ";
+    public static final String JVM_PARAM = " --jvm=\"-Xms%sG -Xmx%sG\" ";

    @Test
    public void testLoadJvmEnv()   {
@ -88,6 +88,6 @@ public class DataxParametersTest {
    public String loadJvmEnvTest(DataxParameters dataXParameters) {
        int xms = dataXParameters.getXms() < 1 ? 1 : dataXParameters.getXms();
        int xmx = dataXParameters.getXmx() < 1 ? 1 : dataXParameters.getXmx();
-        return String.format(JVM_EVN, xms, xmx);
+        return String.format(JVM_PARAM, xms, xmx);
    }
 }
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java
@ -81,15 +81,15 @@ public class DataxTask extends AbstractTask {
    /**
     * jvm parameters
     */
-    public static final String JVM_EVN = " --jvm=\"-Xms%sG -Xmx%sG\" ";
+    public static final String JVM_PARAM = " --jvm=\"-Xms%sG -Xmx%sG\" ";
    /**
     * python process(datax only supports version 2.7 by default)
     */
    private static final String DATAX_PYTHON = "python2.7";
    /**
-     * datax home path
+     * datax path
     */
-    private static final String DATAX_HOME_EVN = "${DATAX_HOME}";
+    private static final String DATAX_PATH = "${DATAX_HOME}/bin/datax.py";
    /**
     * datax channel count
     */
@ -396,7 +396,7 @@ public class DataxTask extends AbstractTask {
        StringBuilder sbr = new StringBuilder();
        sbr.append(DATAX_PYTHON);
        sbr.append(" ");
-        sbr.append(DATAX_HOME_EVN);
+        sbr.append(DATAX_PATH);
        sbr.append(" ");
        sbr.append(loadJvmEnv(dataXParameters));
        sbr.append(jobConfigFilePath);
@ -424,7 +424,7 @@ public class DataxTask extends AbstractTask {
    public String loadJvmEnv(DataxParameters dataXParameters) {
        int xms = dataXParameters.getXms() < 1 ? 1 : dataXParameters.getXms();
        int xmx = dataXParameters.getXmx() < 1 ? 1 : dataXParameters.getXmx();
-        return String.format(JVM_EVN, xms, xmx);
+        return String.format(JVM_PARAM, xms, xmx);
    }

    /**
--- a/script/env/dolphinscheduler_env.sh
+++ b/script/env/dolphinscheduler_env.sh
@ -23,6 +23,6 @@ export PYTHON_HOME=/opt/soft/python
 export JAVA_HOME=/opt/soft/java
 export HIVE_HOME=/opt/soft/hive
 export FLINK_HOME=/opt/soft/flink
-export DATAX_HOME=/opt/soft/datax/bin/datax.py
+export DATAX_HOME=/opt/soft/datax

-export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH:$FLINK_HOME/bin:$DATAX_HOME:$PATH
+export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH