diff --git a/deploy/docker/docker-compose.yml b/deploy/docker/docker-compose.yml
index 2c557c4dd7..e59320b958 100644
--- a/deploy/docker/docker-compose.yml
+++ b/deploy/docker/docker-compose.yml
@@ -140,6 +140,29 @@ services:
networks:
- dolphinscheduler
+ dolphinscheduler-python-gateway:
+ image: ${HUB}/dolphinscheduler-python-gateway:${TAG}
+ ports:
+ - "54321:54321"
+ - "25333:25333"
+ env_file: .env
+ healthcheck:
+ test: [ "CMD", "curl", "http://localhost:54321/actuator/health" ]
+ interval: 30s
+ timeout: 5s
+ retries: 3
+ depends_on:
+ dolphinscheduler-schema-initializer:
+ condition: service_completed_successfully
+ dolphinscheduler-zookeeper:
+ condition: service_healthy
+ volumes:
+ - dolphinscheduler-logs:/opt/dolphinscheduler/logs
+ - dolphinscheduler-shared-local:/opt/soft
+ - dolphinscheduler-resource-local:/dolphinscheduler
+ networks:
+ - dolphinscheduler
+
networks:
dolphinscheduler:
driver: bridge
diff --git a/deploy/docker/docker-stack.yml b/deploy/docker/docker-stack.yml
index 02ee37be02..8d0ae92553 100644
--- a/deploy/docker/docker-stack.yml
+++ b/deploy/docker/docker-stack.yml
@@ -118,6 +118,27 @@ services:
mode: replicated
replicas: 1
+ dolphinscheduler-python-gateway:
+ image: apache/dolphinscheduler-python-gateway
+ ports:
+ - 54321:54321
+ - 25333:25333
+ env_file: .env
+ healthcheck:
+ test: [ "CMD", "curl", "http://localhost:54321/actuator/health" ]
+ interval: 30s
+ timeout: 5s
+ retries: 3
+ volumes:
+ - dolphinscheduler-logs:/opt/dolphinscheduler/logs
+ - dolphinscheduler-shared-local:/opt/soft
+ - dolphinscheduler-resource-local:/dolphinscheduler
+ networks:
+ - dolphinscheduler
+ deploy:
+ mode: replicated
+ replicas: 1
+
networks:
dolphinscheduler:
driver: overlay
diff --git a/deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl b/deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl
index c2684e029a..f8fef1167a 100644
--- a/deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl
+++ b/deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl
@@ -44,6 +44,9 @@ Create default docker images' fullname.
{{- define "dolphinscheduler.image.fullname.tools" -}}
{{- .Values.image.registry }}/dolphinscheduler-tools:{{ .Values.image.tag | default .Chart.AppVersion -}}
{{- end -}}
+{{- define "dolphinscheduler.image.fullname.python-gateway" -}}
+{{- .Values.image.registry }}/dolphinscheduler-python-gateway:{{ .Values.image.tag | default .Chart.AppVersion -}}
+{{- end -}}
{{/*
Create a default common labels.
diff --git a/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-bin.xml b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-bin.xml
index 661dbdb826..769393db7e 100644
--- a/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-bin.xml
+++ b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-bin.xml
@@ -50,6 +50,11 @@
logger-server
+
+ ${basedir}/../dolphinscheduler-python/target/python-gateway-server
+ python-gateway-server
+
+
${basedir}/../dolphinscheduler-standalone-server/target/standalone-server
standalone-server
diff --git a/dolphinscheduler-python/pom.xml b/dolphinscheduler-python/pom.xml
index 7076689473..ecb3872ab9 100644
--- a/dolphinscheduler-python/pom.xml
+++ b/dolphinscheduler-python/pom.xml
@@ -57,4 +57,52 @@
+
+
+
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+
+
+ *.yaml
+ *.xml
+
+
+
+
+ maven-assembly-plugin
+
+
+ dolphinscheduler-python-gateway-server
+ package
+
+ single
+
+
+ python-gateway-server
+
+ src/main/assembly/dolphinscheduler-python-gateway-server.xml
+
+ false
+
+
+
+
+
+
+
+
+
+ docker
+
+
+
+ org.codehaus.mojo
+ exec-maven-plugin
+
+
+
+
+
diff --git a/dolphinscheduler-python/src/main/assembly/dolphinscheduler-python-gateway-server.xml b/dolphinscheduler-python/src/main/assembly/dolphinscheduler-python-gateway-server.xml
new file mode 100644
index 0000000000..49e03206ea
--- /dev/null
+++ b/dolphinscheduler-python/src/main/assembly/dolphinscheduler-python-gateway-server.xml
@@ -0,0 +1,64 @@
+
+
+
+ dolphinscheduler-python-gateway-server
+
+ dir
+
+ false
+ python-gateway-server
+
+
+ ${basedir}/src/main/resources
+
+ *.yaml
+ *.xml
+
+ conf
+
+
+ ${basedir}/src/main/bin
+ bin
+ 0755
+ 0755
+
+
+ ${basedir}/../script/env
+ bin
+
+ dolphinscheduler_env.sh
+
+ 0755
+ 0755
+
+
+ ${basedir}/../dolphinscheduler-common/src/main/resources
+
+ **/*.properties
+
+ conf
+
+
+
+
+ libs
+
+
+
diff --git a/dolphinscheduler-python/src/main/bin/start.sh b/dolphinscheduler-python/src/main/bin/start.sh
new file mode 100644
index 0000000000..f189684330
--- /dev/null
+++ b/dolphinscheduler-python/src/main/bin/start.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+BIN_DIR=$(dirname $0)
+DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/..; pwd)}
+
+source "$BIN_DIR/dolphinscheduler_env.sh"
+
+JAVA_OPTS=${JAVA_OPTS:-"-server -Xms1g -Xmx1g -Xmn512m -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"}
+
+if [[ "$DOCKER" == "true" ]]; then
+ JAVA_OPTS="${JAVA_OPTS} -XX:-UseContainerSupport"
+fi
+
+java $JAVA_OPTS \
+ -cp "$DOLPHINSCHEDULER_HOME/conf":"$DOLPHINSCHEDULER_HOME/libs/*" \
+ org.apache.dolphinscheduler.server.PythonGatewayServer
diff --git a/dolphinscheduler-python/src/main/docker/Dockerfile b/dolphinscheduler-python/src/main/docker/Dockerfile
new file mode 100644
index 0000000000..7f50a8fc39
--- /dev/null
+++ b/dolphinscheduler-python/src/main/docker/Dockerfile
@@ -0,0 +1,34 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+FROM openjdk:8-jre-slim-buster
+
+ENV DOCKER true
+ENV TZ Asia/Shanghai
+ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler
+
+RUN apt update ; \
+ apt install -y curl sudo ; \
+ rm -rf /var/lib/apt/lists/*
+
+WORKDIR $DOLPHINSCHEDULER_HOME
+
+ADD ./target/python-gateway-server $DOLPHINSCHEDULER_HOME
+
+EXPOSE 25333 54321
+
+CMD [ "/bin/bash", "./bin/start.sh" ]
diff --git a/dolphinscheduler-python/src/main/java/org/apache/dolphinscheduler/server/PythonGatewayServer.java b/dolphinscheduler-python/src/main/java/org/apache/dolphinscheduler/server/PythonGatewayServer.java
index ef6deb45da..73d66b17de 100644
--- a/dolphinscheduler-python/src/main/java/org/apache/dolphinscheduler/server/PythonGatewayServer.java
+++ b/dolphinscheduler-python/src/main/java/org/apache/dolphinscheduler/server/PythonGatewayServer.java
@@ -503,7 +503,7 @@ public class PythonGatewayServer extends SpringBootServletInitializer {
public void run() {
GatewayServer server = new GatewayServer(this);
GatewayServer.turnLoggingOn();
- // Start server to accept python client RPC
+ // Start server to accept python client socket
server.start();
}
diff --git a/dolphinscheduler-python/src/main/resources/application.yaml b/dolphinscheduler-python/src/main/resources/application.yaml
new file mode 100644
index 0000000000..af315b1f7f
--- /dev/null
+++ b/dolphinscheduler-python/src/main/resources/application.yaml
@@ -0,0 +1,61 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+spring:
+ application:
+ name: python-gateway-server
+ main:
+ banner-mode: off
+ datasource:
+ driver-class-name: org.h2.Driver
+ url: jdbc:h2:mem:dolphinscheduler;MODE=MySQL;DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=true
+ username: sa
+ password: ""
+ jackson:
+ time-zone: GMT+8
+ servlet:
+ multipart:
+ max-file-size: 1024MB
+ max-request-size: 1024MB
+ messages:
+ basename: i18n/messages
+ jpa:
+ hibernate:
+ ddl-auto: none
+
+server:
+ port: 54321
+
+management:
+ endpoints:
+ web:
+ exposure:
+ include: '*'
+ metrics:
+ tags:
+ application: ${spring.application.name}
+
+# Override by profile
+---
+spring:
+ config:
+ activate:
+ on-profile: postgresql
+ quartz:
+ properties:
+ org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.PostgreSQLDelegate
+
diff --git a/dolphinscheduler-python/src/main/resources/logback-spring.xml b/dolphinscheduler-python/src/main/resources/logback-spring.xml
new file mode 100644
index 0000000000..4f06d160d6
--- /dev/null
+++ b/dolphinscheduler-python/src/main/resources/logback-spring.xml
@@ -0,0 +1,57 @@
+
+
+
+
+
+
+
+
+
+ [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
+
+ UTF-8
+
+
+
+
+ ${log.base}/dolphinscheduler-python-gateway.log
+
+ INFO
+
+
+ ${log.base}/dolphinscheduler-python-gateway.%d{yyyy-MM-dd_HH}.%i.log
+ 168
+ 64MB
+
+
+
+ [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
+
+ UTF-8
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/dolphinscheduler-standalone-server/pom.xml b/dolphinscheduler-standalone-server/pom.xml
index 4c4b2fd902..61df690c4d 100644
--- a/dolphinscheduler-standalone-server/pom.xml
+++ b/dolphinscheduler-standalone-server/pom.xml
@@ -50,6 +50,11 @@
dolphinscheduler-log-server
+
+ org.apache.dolphinscheduler
+ dolphinscheduler-python
+
+
org.apache.curator
curator-test
@@ -61,10 +66,6 @@
-
- org.apache.dolphinscheduler
- dolphinscheduler-python
-
diff --git a/dolphinscheduler-standalone-server/src/main/assembly/dolphinscheduler-standalone-server.xml b/dolphinscheduler-standalone-server/src/main/assembly/dolphinscheduler-standalone-server.xml
index 8d767eec0c..90e100e5cf 100644
--- a/dolphinscheduler-standalone-server/src/main/assembly/dolphinscheduler-standalone-server.xml
+++ b/dolphinscheduler-standalone-server/src/main/assembly/dolphinscheduler-standalone-server.xml
@@ -60,6 +60,13 @@
.
+
+ ${basedir}/../dolphinscheduler-python-gateway/target/python-gateway
+
+ libs
+
+ .
+
${basedir}/src/main/resources
diff --git a/script/dolphinscheduler-daemon.sh b/script/dolphinscheduler-daemon.sh
index 37263e3c75..ddfdbe6540 100755
--- a/script/dolphinscheduler-daemon.sh
+++ b/script/dolphinscheduler-daemon.sh
@@ -16,7 +16,7 @@
# limitations under the License.
#
-usage="Usage: dolphinscheduler-daemon.sh (start|stop|status) "
+usage="Usage: dolphinscheduler-daemon.sh (start|stop|status) "
# if no args specified, show usage
if [ $# -le 1 ]; then
@@ -62,6 +62,8 @@ elif [ "$command" = "alert-server" ]; then
:
elif [ "$command" = "standalone-server" ]; then
:
+elif [ "$command" = "python-gateway-server" ]; then
+ :
else
echo "Error: No command named '$command' was found."
exit 1
diff --git a/script/env/install_env.sh b/script/env/install_env.sh
index 8b68c0ea00..2df11c14ae 100644
--- a/script/env/install_env.sh
+++ b/script/env/install_env.sh
@@ -48,6 +48,11 @@ alertServer=${alertServer:-"ds3"}
# Example for hostname: apiServers="ds1", Example for IP: apiServers="192.168.8.1"
apiServers=${apiServers:-"ds1"}
+# A comma separated list of machine hostname or IP would be installed Python gateway server, it
+# must be a subset of configuration `ips`.
+# Example for hostname: pythonGatewayServers="ds1", Example for IP: pythonGatewayServers="192.168.8.1"
+pythonGatewayServers=${pythonGatewayServers:-"ds1"}
+
# The directory to install DolphinScheduler for all machine we config above. It will automatically be created by `install.sh` script if not exists.
# Do not set this configuration same as the current path (pwd)
installPath=${installPath:-"/tmp/dolphinscheduler"}
diff --git a/script/scp-hosts.sh b/script/scp-hosts.sh
index 2397ede936..1b216ab41e 100755
--- a/script/scp-hosts.sh
+++ b/script/scp-hosts.sh
@@ -49,7 +49,7 @@ do
echo "scp dirs to $host/$installPath starting"
ssh -p $sshPort $host "cd $installPath/; rm -rf bin/ conf/ lib/ script/ sql/ ui/"
- for dsDir in bin master-server worker-server alert-server api-server ui
+ for dsDir in bin master-server worker-server alert-server api-server ui python-gateway-server
do
# if worker in workersGroupMap
if [[ "${workersGroupMap[${host}]}" ]]; then
diff --git a/script/start-all.sh b/script/start-all.sh
index 002f99d873..e7934da8c1 100755
--- a/script/start-all.sh
+++ b/script/start-all.sh
@@ -56,6 +56,13 @@ do
ssh -p $sshPort $apiServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start api-server;"
done
+pythonGatewayHost=(${pythonGatewayServers//,/ })
+for pythonGatewayServer in "${pythonGatewayHost[@]}"
+do
+ echo "$pythonGatewayServer python gateway server is starting"
+ ssh -p $sshPort $pythonGatewayServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start python-gateway-server;"
+done
+
# query server status
echo "query server status"
cd $installPath/; sh bin/status-all.sh
diff --git a/script/status-all.sh b/script/status-all.sh
index df19eb9c37..c3b4591a62 100755
--- a/script/status-all.sh
+++ b/script/status-all.sh
@@ -74,3 +74,11 @@ do
apiState=`ssh -p $sshPort $apiServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh status api-server;"`
echo "$apiServer $apiState"
done
+
+# python gateway server check state
+pythonGatewayHost=(${pythonGatewayServers//,/ })
+for pythonGatewayServer in "${pythonGatewayHost[@]}"
+do
+ pythonGatewayState=`ssh -p $sshPort $pythonGatewayServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh status python-gateway-server;"`
+ echo "$pythonGatewayServer $pythonGatewayState"
+done
diff --git a/script/stop-all.sh b/script/stop-all.sh
index 6592374659..856340c390 100755
--- a/script/stop-all.sh
+++ b/script/stop-all.sh
@@ -54,3 +54,10 @@ do
echo "$apiServer api server is stopping"
ssh -p $sshPort $apiServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop api-server;"
done
+
+pythonGatewayHost=(${pythonGatewayServers//,/ })
+for pythonGatewayServer in "${pythonGatewayHost[@]}"
+do
+ echo "$pythonGatewayServer python gateway server is stopping"
+ ssh -p $sshPort $pythonGatewayServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop python-gateway-server;"
+done