Cache HadoopUtils instance with specific days expire time (#2181)

* Cache HadoopUtils instance with 7 days expire time

* solve sonar issue

* add kerberos expire time config

* move KERBEROS_EXPIRE_TIME to Constants.java

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java

@@ -813,6 +813,11 @@ public final class Constants {
      */
     public static final String KERBEROS = "kerberos";
 
+    /**
+     * kerberos expire time
+     */
+    public static final String KERBEROS_EXPIRE_TIME = "kerberos.expire.time";
+
     /**
      * java.security.krb5.conf
      */

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java

@@ -16,6 +16,9 @@
  */
 package org.apache.dolphinscheduler.common.utils;
 
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
 import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
 import org.apache.dolphinscheduler.common.enums.ResUploadType;
@@ -37,6 +40,7 @@ import java.security.PrivilegedExceptionAction;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -48,24 +52,31 @@ public class HadoopUtils implements Closeable {
 
     private static final Logger logger = LoggerFactory.getLogger(HadoopUtils.class);
 
-    private static HadoopUtils instance = new HadoopUtils();
+    private static final String HADOOP_UTILS_KEY = "HADOOP_UTILS_KEY";
+
+    private static final LoadingCache<String, HadoopUtils> cache = CacheBuilder
+            .newBuilder()
+            .expireAfterWrite(PropertyUtils.getInt(Constants.KERBEROS_EXPIRE_TIME, 7), TimeUnit.DAYS)
+            .build(new CacheLoader<String, HadoopUtils>() {
+                @Override
+                public HadoopUtils load(String key) throws Exception {
+                    return new HadoopUtils();
+                }
+            });
+
     private static Configuration configuration;
     private static FileSystem fs;
 
-    private String hdfsUser;
+    private static String hdfsUser = PropertyUtils.getString(Constants.HDFS_ROOT_USER);
 
     private HadoopUtils() {
-        hdfsUser = PropertyUtils.getString(Constants.HDFS_ROOT_USER);
         init();
         initHdfsPath();
     }
 
     public static HadoopUtils getInstance() {
-        // if kerberos startup , renew HadoopUtils
+
-        if (CommonUtils.getKerberosStartupState()){
+        return cache.getUnchecked(HADOOP_UTILS_KEY);
-            return new HadoopUtils();
-        }
-        return instance;
     }
 
     /**
@@ -88,10 +99,7 @@ public class HadoopUtils implements Closeable {
     /**
      * init hadoop configuration
      */
-    private void init() {
+    private static void init() {
-        if (configuration == null) {
-            synchronized (HadoopUtils.class) {
-                if (configuration == null) {
         try {
             configuration = new Configuration();
 
@@ -161,10 +169,6 @@ public class HadoopUtils implements Closeable {
         } catch (Exception e) {
             logger.error(e.getMessage(), e);
         }
-
-                }
-            }
-        }
     }
 
     /**
@@ -203,7 +207,6 @@ public class HadoopUtils implements Closeable {
     }
 
 
-
     /**
      * cat file on hdfs
      *
@@ -258,7 +261,7 @@ public class HadoopUtils implements Closeable {
     /**
      * the src file is on the local disk.  Add it to FS at
      * the given dst name.
-
+     *
      * @param srcFile      local file
      * @param dstHdfsPath  destination hdfs path
      * @param deleteSource whether to delete the src
@@ -307,7 +310,6 @@ public class HadoopUtils implements Closeable {
     }
 
     /**
-     *
      * delete a file
      *
      * @param hdfsFilePath the path to delete.
@@ -351,10 +353,11 @@ public class HadoopUtils implements Closeable {
     /**
      * Renames Path src to Path dst.  Can take place on local fs
      * or remote DFS.
+     *
      * @param src path to be renamed
      * @param dst new path after rename
-     * @throws IOException on failure
      * @return true if rename is successful
+     * @throws IOException on failure
      */
     public boolean rename(String src, String dst) throws IOException {
         return fs.rename(new Path(src), new Path(dst));
@@ -400,7 +403,6 @@ public class HadoopUtils implements Closeable {
     }
 
     /**
-     *
      * @return data hdfs path
      */
     public static String getHdfsDataBasePath() {

dolphinscheduler-common/src/main/resources/common.properties

@@ -91,4 +91,4 @@ yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx
 # If it is a single resourcemanager, you only need to configure one host name. If it is resourcemanager HA, the default configuration is fine
 yarn.application.status.address=http://ark1:8088/ws/v1/cluster/apps/%s
 
-
+kerberos.expire.time=7