From 416c41465dd490bd0546ac79f2c993004482ffe0 Mon Sep 17 00:00:00 2001 From: Jay Chung Date: Mon, 14 Nov 2022 18:43:08 +0800 Subject: [PATCH] [fix] Add token as authentication for python gateway (#12893) separate from #6407. Authentication, add secret to ensure only trusted people could connect to gateway. fix: #8255 (cherry picked from commit 6d8befa0752c1e8005651c7b57b2301c7b9606fc) --- .../PythonGatewayConfiguration.java | 68 ++----------------- .../api/python/PythonGateway.java | 56 ++++++++------- .../src/main/resources/application.yaml | 3 + .../src/main/resources/application.yaml | 3 + 4 files changed, 43 insertions(+), 87 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/PythonGatewayConfiguration.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/PythonGatewayConfiguration.java index 5735e27fd2..8a3a2e521c 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/PythonGatewayConfiguration.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/PythonGatewayConfiguration.java @@ -17,13 +17,14 @@ package org.apache.dolphinscheduler.api.configuration; +import lombok.Data; + import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.stereotype.Component; +import org.springframework.context.annotation.Configuration; -@Component -@EnableConfigurationProperties -@ConfigurationProperties(value = "python-gateway", ignoreUnknownFields = false) +@Data +@Configuration +@ConfigurationProperties(value = "python-gateway") public class PythonGatewayConfiguration { private boolean enabled; private String gatewayServerAddress; @@ -32,60 +33,5 @@ public class PythonGatewayConfiguration { private int pythonPort; private int connectTimeout; private int readTimeout; - - public boolean getEnabled() { - return enabled; - } - - public void setEnabled(boolean enabled) { - this.enabled = enabled; - } - - public String getGatewayServerAddress() { - return gatewayServerAddress; - } - - public void setGatewayServerAddress(String gatewayServerAddress) { - this.gatewayServerAddress = gatewayServerAddress; - } - - public int getGatewayServerPort() { - return gatewayServerPort; - } - - public void setGatewayServerPort(int gatewayServerPort) { - this.gatewayServerPort = gatewayServerPort; - } - - public String getPythonAddress() { - return pythonAddress; - } - - public void setPythonAddress(String pythonAddress) { - this.pythonAddress = pythonAddress; - } - - public int getPythonPort() { - return pythonPort; - } - - public void setPythonPort(int pythonPort) { - this.pythonPort = pythonPort; - } - - public int getConnectTimeout() { - return connectTimeout; - } - - public void setConnectTimeout(int connectTimeout) { - this.connectTimeout = connectTimeout; - } - - public int getReadTimeout() { - return readTimeout; - } - - public void setReadTimeout(int readTimeout) { - this.readTimeout = readTimeout; - } + private String authToken; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/python/PythonGateway.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/python/PythonGateway.java index d9f0c78674..b79eaf307e 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/python/PythonGateway.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/python/PythonGateway.java @@ -17,18 +17,6 @@ package org.apache.dolphinscheduler.api.python; -import java.io.IOException; -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import javax.annotation.PostConstruct; - -import org.apache.commons.collections.CollectionUtils; import org.apache.dolphinscheduler.api.configuration.PythonGatewayConfiguration; import org.apache.dolphinscheduler.api.dto.EnvironmentDto; import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent; @@ -72,6 +60,24 @@ import org.apache.dolphinscheduler.dao.mapper.ProjectUserMapper; import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper; import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; import org.apache.dolphinscheduler.spi.enums.ResourceType; + +import py4j.GatewayServer; +import py4j.GatewayServer.GatewayServerBuilder; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import javax.annotation.PostConstruct; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -626,29 +632,27 @@ public class PythonGateway { @PostConstruct public void init() { - if (pythonGatewayConfiguration.getEnabled()) { + if (pythonGatewayConfiguration.isEnabled()) { this.start(); } } private void start() { - GatewayServer server; try { InetAddress gatewayHost = InetAddress.getByName(pythonGatewayConfiguration.getGatewayServerAddress()); - InetAddress pythonHost = InetAddress.getByName(pythonGatewayConfiguration.getPythonAddress()); - server = new GatewayServer( - this, - pythonGatewayConfiguration.getGatewayServerPort(), - pythonGatewayConfiguration.getPythonPort(), - gatewayHost, - pythonHost, - pythonGatewayConfiguration.getConnectTimeout(), - pythonGatewayConfiguration.getReadTimeout(), - null - ); + GatewayServerBuilder serverBuilder = new GatewayServer.GatewayServerBuilder() + .entryPoint(this) + .javaAddress(gatewayHost) + .javaPort(pythonGatewayConfiguration.getGatewayServerPort()) + .connectTimeout(pythonGatewayConfiguration.getConnectTimeout()) + .readTimeout(pythonGatewayConfiguration.getReadTimeout()); + if (!StringUtils.isEmpty(pythonGatewayConfiguration.getAuthToken())) { + serverBuilder.authToken(pythonGatewayConfiguration.getAuthToken()); + } + GatewayServer.turnLoggingOn(); logger.info("PythonGatewayService started on: " + gatewayHost.toString()); - server.start(); + serverBuilder.build().start(); } catch (UnknownHostException e) { logger.error("exception occurred while constructing PythonGatewayService().", e); } diff --git a/dolphinscheduler-api/src/main/resources/application.yaml b/dolphinscheduler-api/src/main/resources/application.yaml index 9a8381454b..2b51881e07 100644 --- a/dolphinscheduler-api/src/main/resources/application.yaml +++ b/dolphinscheduler-api/src/main/resources/application.yaml @@ -121,6 +121,9 @@ metrics: python-gateway: # Weather enable python gateway server or not. The default value is true. enabled: true + # Authentication token for connection from python api to python gateway server. Should be changed the default value + # when you deploy in public network. + auth-token: jwUDzpLsNKEFER4*a8gruBH_GsAurNxU7A@Xc # The address of Python gateway server start. Set its value to `0.0.0.0` if your Python API run in different # between Python gateway server. It could be be specific to other address like `127.0.0.1` or `localhost` gateway-server-address: 0.0.0.0 diff --git a/dolphinscheduler-standalone-server/src/main/resources/application.yaml b/dolphinscheduler-standalone-server/src/main/resources/application.yaml index a97d03e34c..66c8abd70d 100644 --- a/dolphinscheduler-standalone-server/src/main/resources/application.yaml +++ b/dolphinscheduler-standalone-server/src/main/resources/application.yaml @@ -187,6 +187,9 @@ alert: python-gateway: # Weather enable python gateway server or not. The default value is true. enabled: true + # Authentication token for connection from python api to python gateway server. Should be changed the default value + # when you deploy in public network. + auth-token: jwUDzpLsNKEFER4*a8gruBH_GsAurNxU7A@Xc # The address of Python gateway server start. Set its value to `0.0.0.0` if your Python API run in different # between Python gateway server. It could be be specific to other address like `127.0.0.1` or `localhost` gateway-server-address: 0.0.0.0