github
/
DolphinScheduler
mirror of https://gitee.com/dolphinscheduler/DolphinScheduler.git
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

[Improvement-5527][api-server] failed find any kerberos (#5533) 

* fix failed find any kerberos.

* increase code coverage.

* update common utils test.

* update common utils test.

* update common utils test.

* fix exception type.
pull/3/MERGE
zhuangchong 4 years ago committed by GitHub
parent
46660b58ed
commit
9ba4ffbe48
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 78 additions and 38 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 28
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java
  2. 15
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java
  3. 53
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CommonUtilsTest.java

28
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java
Unescape View File

@ -91,30 +91,48 @@ public class CommonUtils {
/** /**
* load kerberos configuration * load kerberos configuration
* *
* @throws Exception errors * @param configuration
* @return load kerberos config return true
* @throws IOException errors
*/ */
public static void loadKerberosConf() throws Exception { public static boolean loadKerberosConf(Configuration configuration) throws IOException {
loadKerberosConf(PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH), return loadKerberosConf(PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH),
PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME), PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME),
PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH)); PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH), configuration);
} }
/** /**
* load kerberos configuration * load kerberos configuration
*
* @param javaSecurityKrb5Conf javaSecurityKrb5Conf * @param javaSecurityKrb5Conf javaSecurityKrb5Conf
* @param loginUserKeytabUsername loginUserKeytabUsername * @param loginUserKeytabUsername loginUserKeytabUsername
* @param loginUserKeytabPath loginUserKeytabPath * @param loginUserKeytabPath loginUserKeytabPath
* @throws IOException errors * @throws IOException errors
*/ */
public static void loadKerberosConf(String javaSecurityKrb5Conf, String loginUserKeytabUsername, String loginUserKeytabPath) throws IOException { public static void loadKerberosConf(String javaSecurityKrb5Conf, String loginUserKeytabUsername, String loginUserKeytabPath) throws IOException {
loadKerberosConf(javaSecurityKrb5Conf, loginUserKeytabUsername, loginUserKeytabPath, new Configuration());
}
/**
* load kerberos configuration
*
* @param javaSecurityKrb5Conf javaSecurityKrb5Conf
* @param loginUserKeytabUsername loginUserKeytabUsername
* @param loginUserKeytabPath loginUserKeytabPath
* @param configuration configuration
* @return load kerberos config return true
* @throws IOException errors
*/
public static boolean loadKerberosConf(String javaSecurityKrb5Conf, String loginUserKeytabUsername, String loginUserKeytabPath, Configuration configuration) throws IOException {
if (CommonUtils.getKerberosStartupState()) { if (CommonUtils.getKerberosStartupState()) {
System.setProperty(Constants.JAVA_SECURITY_KRB5_CONF, StringUtils.defaultIfBlank(javaSecurityKrb5Conf, PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH))); System.setProperty(Constants.JAVA_SECURITY_KRB5_CONF, StringUtils.defaultIfBlank(javaSecurityKrb5Conf, PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH)));
Configuration configuration = new Configuration();
configuration.set(Constants.HADOOP_SECURITY_AUTHENTICATION, Constants.KERBEROS); configuration.set(Constants.HADOOP_SECURITY_AUTHENTICATION, Constants.KERBEROS);
UserGroupInformation.setConfiguration(configuration); UserGroupInformation.setConfiguration(configuration);
UserGroupInformation.loginUserFromKeytab(StringUtils.defaultIfBlank(loginUserKeytabUsername, PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME)), UserGroupInformation.loginUserFromKeytab(StringUtils.defaultIfBlank(loginUserKeytabUsername, PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME)),
StringUtils.defaultIfBlank(loginUserKeytabPath, PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH))); StringUtils.defaultIfBlank(loginUserKeytabPath, PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH)));
return true;
} }
return false;
} }
/** /**

15
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java
Unescape View File

@ -127,14 +127,8 @@ public class HadoopUtils implements Closeable {
ResUploadType resUploadType = ResUploadType.valueOf(resourceStorageType); ResUploadType resUploadType = ResUploadType.valueOf(resourceStorageType);
if (resUploadType == ResUploadType.HDFS) { if (resUploadType == ResUploadType.HDFS) {
if (PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false)) { if (CommonUtils.loadKerberosConf(configuration)) {
System.setProperty(Constants.JAVA_SECURITY_KRB5_CONF,
PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH));
configuration.set(Constants.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
hdfsUser = ""; hdfsUser = "";
UserGroupInformation.setConfiguration(configuration);
UserGroupInformation.loginUserFromKeytab(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME),
PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH));
} }
String defaultFS = configuration.get(Constants.FS_DEFAULTFS); String defaultFS = configuration.get(Constants.FS_DEFAULTFS);
@ -156,21 +150,16 @@ public class HadoopUtils implements Closeable {
logger.info("get property:{} -> {}, from core-site.xml hdfs-site.xml ", Constants.FS_DEFAULTFS, defaultFS); logger.info("get property:{} -> {}, from core-site.xml hdfs-site.xml ", Constants.FS_DEFAULTFS, defaultFS);
} }
if (fs == null) {
if (StringUtils.isNotEmpty(hdfsUser)) { if (StringUtils.isNotEmpty(hdfsUser)) {
UserGroupInformation ugi = UserGroupInformation.createRemoteUser(hdfsUser); UserGroupInformation ugi = UserGroupInformation.createRemoteUser(hdfsUser);
ugi.doAs(new PrivilegedExceptionAction<Boolean>() { ugi.doAs((PrivilegedExceptionAction<Boolean>) () -> {
@Override
public Boolean run() throws Exception {
fs = FileSystem.get(configuration); fs = FileSystem.get(configuration);
return true; return true;
}
}); });
} else { } else {
logger.warn("hdfs.root.user is not set value!"); logger.warn("hdfs.root.user is not set value!");
fs = FileSystem.get(configuration); fs = FileSystem.get(configuration);
} }
}
} else if (resUploadType == ResUploadType.S3) { } else if (resUploadType == ResUploadType.S3) {
System.setProperty(Constants.AWS_S3_V4, Constants.STRING_TRUE); System.setProperty(Constants.AWS_S3_V4, Constants.STRING_TRUE);
configuration.set(Constants.FS_DEFAULTFS, PropertyUtils.getString(Constants.FS_DEFAULTFS)); configuration.set(Constants.FS_DEFAULTFS, PropertyUtils.getString(Constants.FS_DEFAULTFS));

53
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CommonUtilsTest.java
Unescape View File

@ -14,20 +14,31 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import java.net.InetAddress;
import java.net.UnknownHostException;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
import org.junit.runner.RunWith;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.net.InetAddress;
import java.net.UnknownHostException;
/** /**
* configuration test * configuration test
*/ */
@RunWith(PowerMockRunner.class)
@PrepareForTest(value = { PropertyUtils.class, UserGroupInformation.class})
public class CommonUtilsTest { public class CommonUtilsTest {
private static final Logger logger = LoggerFactory.getLogger(CommonUtilsTest.class); private static final Logger logger = LoggerFactory.getLogger(CommonUtilsTest.class);
@Test @Test
@ -35,21 +46,43 @@ public class CommonUtilsTest {
logger.info(CommonUtils.getSystemEnvPath()); logger.info(CommonUtils.getSystemEnvPath());
Assert.assertTrue(true); Assert.assertTrue(true);
} }
@Test @Test
public void isDevelopMode() { public void isDevelopMode() {
logger.info("develop mode: {}",CommonUtils.isDevelopMode()); logger.info("develop mode: {}",CommonUtils.isDevelopMode());
Assert.assertTrue(true); Assert.assertTrue(true);
} }
@Test @Test
public void getKerberosStartupState() { public void getKerberosStartupState() {
logger.info("kerberos startup state: {}",CommonUtils.getKerberosStartupState()); boolean kerberosStartupState = CommonUtils.getKerberosStartupState();
Assert.assertTrue(true); logger.info("kerberos startup state: {}",kerberosStartupState);
Assert.assertFalse(kerberosStartupState);
PowerMockito.mockStatic(PropertyUtils.class);
PowerMockito.when(PropertyUtils.getUpperCaseString(Constants.RESOURCE_STORAGE_TYPE)).thenReturn("HDFS");
PowerMockito.when(PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false)).thenReturn(Boolean.TRUE);
kerberosStartupState = CommonUtils.getKerberosStartupState();
logger.info("kerberos startup state: {}",kerberosStartupState);
Assert.assertTrue(kerberosStartupState);
} }
@Test @Test
public void loadKerberosConf() { public void loadKerberosConf() {
try { try {
CommonUtils.loadKerberosConf(); PowerMockito.mockStatic(PropertyUtils.class);
Assert.assertTrue(true); PowerMockito.when(PropertyUtils.getUpperCaseString(Constants.RESOURCE_STORAGE_TYPE)).thenReturn("HDFS");
PowerMockito.when(PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false)).thenReturn(Boolean.TRUE);
PowerMockito.when(PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH)).thenReturn("/opt/krb5.conf");
PowerMockito.when(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME)).thenReturn("hdfs-mycluster@ESZ.COM");
PowerMockito.when(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH)).thenReturn("/opt/hdfs.headless.keytab");
PowerMockito.mockStatic(UserGroupInformation.class);
boolean result = CommonUtils.loadKerberosConf(new Configuration());
Assert.assertTrue(result);
CommonUtils.loadKerberosConf(null, null, null);
} catch (Exception e) { } catch (Exception e) {
Assert.fail("load Kerberos Conf failed"); Assert.fail("load Kerberos Conf failed");
} }
@ -81,10 +114,10 @@ public class CommonUtilsTest {
@Test @Test
public void test() { public void test() {
InetAddress IP = null; InetAddress ip;
try { try {
IP = InetAddress.getLocalHost(); ip = InetAddress.getLocalHost();
logger.info(IP.getHostAddress()); logger.info(ip.getHostAddress());
} catch (UnknownHostException e) { } catch (UnknownHostException e) {
e.printStackTrace(); e.printStackTrace();
} }

Write Preview
Loading…
Cancel
Save