diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java index 541281f793..6cb58a4324 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java @@ -32,7 +32,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.*; +import java.nio.file.Files; import java.security.PrivilegedExceptionAction; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -46,16 +48,14 @@ public class HadoopUtils implements Closeable { private static final Logger logger = LoggerFactory.getLogger(HadoopUtils.class); - private static String hdfsUser = PropertyUtils.getString(Constants.HDFS_ROOT_USER); - private static volatile HadoopUtils instance = new HadoopUtils(); - private static volatile Configuration configuration; + private static HadoopUtils instance = new HadoopUtils(); + private static Configuration configuration; private static FileSystem fs; + private String hdfsUser; private HadoopUtils(){ - if(StringUtils.isEmpty(hdfsUser)){ - hdfsUser = PropertyUtils.getString(Constants.HDFS_ROOT_USER); - } + hdfsUser = PropertyUtils.getString(Constants.HDFS_ROOT_USER); init(); initHdfsPath(); } @@ -129,7 +129,6 @@ public class HadoopUtils implements Closeable { if (fs == null) { if(StringUtils.isNotEmpty(hdfsUser)){ - //UserGroupInformation ugi = UserGroupInformation.createProxyUser(hdfsUser,UserGroupInformation.getLoginUser()); UserGroupInformation ugi = UserGroupInformation.createRemoteUser(hdfsUser); ugi.doAs(new PrivilegedExceptionAction() { @Override @@ -196,7 +195,7 @@ public class HadoopUtils implements Closeable { if(StringUtils.isBlank(hdfsFilePath)){ logger.error("hdfs file path:{} is blank",hdfsFilePath); - return null; + return new byte[0]; } FSDataInputStream fsDataInputStream = fs.open(new Path(hdfsFilePath)); @@ -218,7 +217,7 @@ public class HadoopUtils implements Closeable { if (StringUtils.isBlank(hdfsFilePath)){ logger.error("hdfs file path:{} is blank",hdfsFilePath); - return null; + return Collections.emptyList(); } try (FSDataInputStream in = fs.open(new Path(hdfsFilePath))){ @@ -293,7 +292,7 @@ public class HadoopUtils implements Closeable { if (dstPath.exists()) { if (dstPath.isFile()) { if (overwrite) { - dstPath.delete(); + Files.delete(dstPath.toPath()); } } else { logger.error("destination file must be a file"); @@ -378,7 +377,7 @@ public class HadoopUtils implements Closeable { String responseContent = HttpUtils.get(applicationUrl); - JSONObject jsonObject = JSONObject.parseObject(responseContent); + JSONObject jsonObject = JSON.parseObject(responseContent); String result = jsonObject.getJSONObject("app").getString("finalStatus"); switch (result) { @@ -525,8 +524,6 @@ public class HadoopUtils implements Closeable { */ private static final class YarnHAAdminUtils extends RMAdminCLI { - private static final Logger logger = LoggerFactory.getLogger(YarnHAAdminUtils.class); - /** * get active resourcemanager * @@ -585,8 +582,7 @@ public class HadoopUtils implements Closeable { JSONObject jsonObject = JSON.parseObject(retStr); //get ResourceManager state - String state = jsonObject.getJSONObject("clusterInfo").getString("haState"); - return state; + return jsonObject.getJSONObject("clusterInfo").getString("haState"); } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java index c163dcab3d..7de198f28b 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java @@ -81,17 +81,15 @@ public class HttpUtils { logger.error(e.getMessage(),e); } - if (httpget != null && !httpget.isAborted()) { + if (!httpget.isAborted()) { httpget.releaseConnection(); httpget.abort(); } - if (httpclient != null) { - try { - httpclient.close(); - } catch (IOException e) { - logger.error(e.getMessage(),e); - } + try { + httpclient.close(); + } catch (IOException e) { + logger.error(e.getMessage(),e); } } return responseContent; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java index c3e8197079..5c1011cd8b 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java @@ -125,7 +125,7 @@ public class PropertyUtils { * @param key property name * @return property value */ - public static Boolean getBoolean(String key) { + public static boolean getBoolean(String key) { String value = properties.getProperty(key.trim()); if(null != value){ return Boolean.parseBoolean(value); diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java index 41049ec335..17929f9344 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java @@ -38,7 +38,7 @@ public class HttpUtilsTest { String result = HttpUtils.get("https://github.com/manifest.json"); Assert.assertNotNull(result); JSONObject jsonObject = JSON.parseObject(result); - Assert.assertEquals(jsonObject.getString("name"), "GitHub"); + Assert.assertEquals("GitHub", jsonObject.getString("name")); result = HttpUtils.get("https://123.333.111.33/ccc"); Assert.assertNull(result); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java index 0a8f527536..4dfb8817a8 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java @@ -39,7 +39,7 @@ public class HiveDataSource extends BaseDataSource { @Override public String getJdbcUrl() { String jdbcUrl = getAddress(); - if (jdbcUrl.lastIndexOf("/") != (jdbcUrl.length() - 1)) { + if (jdbcUrl.lastIndexOf('/') != (jdbcUrl.length() - 1)) { jdbcUrl += "/"; } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java index c925f90b9e..85c8d2723c 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.server.worker.task.http; +import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import org.apache.commons.io.Charsets; import org.apache.dolphinscheduler.common.Constants; @@ -26,6 +27,7 @@ import org.apache.dolphinscheduler.common.process.HttpProperty; import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.http.HttpParameters; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; @@ -51,6 +53,7 @@ import org.slf4j.Logger; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -98,7 +101,7 @@ public class HttpTask extends AbstractTask { @Override public void init() { logger.info("http task params {}", taskProps.getTaskParams()); - this.httpParameters = JSONObject.parseObject(taskProps.getTaskParams(), HttpParameters.class); + this.httpParameters = JSON.parseObject(taskProps.getTaskParams(), HttpParameters.class); if (!httpParameters.checkParameters()) { throw new RuntimeException("http task params is not valid"); @@ -146,12 +149,12 @@ public class HttpTask extends AbstractTask { processInstance.getCmdTypeIfComplement(), processInstance.getScheduleTime()); List httpPropertyList = new ArrayList<>(); - if(httpParameters.getHttpParams() != null && httpParameters.getHttpParams().size() > 0){ + if(CollectionUtils.isNotEmpty(httpParameters.getHttpParams() )){ for (HttpProperty httpProperty: httpParameters.getHttpParams()) { - String jsonObject = JSONObject.toJSONString(httpProperty); + String jsonObject = JSON.toJSONString(httpProperty); String params = ParameterUtils.convertParameterPlaceholders(jsonObject,ParamUtils.convert(paramsMap)); logger.info("http request params：{}",params); - httpPropertyList.add(JSONObject.parseObject(params,HttpProperty.class)); + httpPropertyList.add(JSON.parseObject(params,HttpProperty.class)); } } addRequestParams(builder,httpPropertyList); @@ -176,8 +179,7 @@ public class HttpTask extends AbstractTask { if (entity == null) { return null; } - String webPage = EntityUtils.toString(entity, StandardCharsets.UTF_8.name()); - return webPage; + return EntityUtils.toString(entity, StandardCharsets.UTF_8.name()); } /** @@ -186,8 +188,7 @@ public class HttpTask extends AbstractTask { * @return status code */ protected int getStatusCode(CloseableHttpResponse httpResponse) { - int status = httpResponse.getStatusLine().getStatusCode(); - return status; + return httpResponse.getStatusLine().getStatusCode(); } /** @@ -252,7 +253,7 @@ public class HttpTask extends AbstractTask { * @param httpPropertyList http property list */ protected void addRequestParams(RequestBuilder builder,List httpPropertyList) { - if(httpPropertyList != null && httpPropertyList.size() > 0){ + if(CollectionUtils.isNotEmpty(httpPropertyList)){ JSONObject jsonParam = new JSONObject(); for (HttpProperty property: httpPropertyList){ if(property.getHttpParametersType() != null){ @@ -276,12 +277,10 @@ public class HttpTask extends AbstractTask { * @param httpPropertyList http property list */ protected void setHeaders(HttpUriRequest request,List httpPropertyList) { - if(httpPropertyList != null && httpPropertyList.size() > 0){ - for (HttpProperty property: httpPropertyList){ - if(property.getHttpParametersType() != null) { - if (property.getHttpParametersType().equals(HttpParametersType.HEADERS)) { - request.addHeader(property.getProp(), property.getValue()); - } + if(CollectionUtils.isNotEmpty(httpPropertyList)){ + for (HttpProperty property: httpPropertyList) { + if (HttpParametersType.HEADERS.equals(property.getHttpParametersType())) { + request.addHeader(property.getProp(), property.getValue()); } } }