[Improvement-14243][Metrics] Add user tag of response time of api server (#14248)

docs/docs/en/guide/metrics/metrics.md

@@ -113,7 +113,7 @@ For example, you can get the master metrics by `curl http://localhost:5679/actua
 
 - ds.api.request.count: (counter) the number of requests received by the api server
 - ds.api.response.count: (counter) the number of responses received by the api server, sliced by tag `code`
-- ds.api.response.time: (histogram) the response time distribution of the api server
+- ds.api.response.time: (timer) the response time distribution of the api server, sliced by tag `user_id`
 - ds.api.resource.upload.size: (histogram) size distribution of resource files uploaded by the api server (bytes)
 - ds.api.resource.download.size: (histogram) size distribution of resource files download by the api server (bytes)
 

docs/docs/zh/guide/metrics/metrics.md

@@ -113,7 +113,7 @@ metrics exporter端口`server.port`是在application.yaml里定义的: master: `
 
 - ds.api.request.count: (counter) api请求次数
 - ds.api.response.count: (counter) api响应次数，可由标签`code`切分
-- ds.api.response.time: (histogram) api响应时间分布
+- ds.api.response.time: (timer) api响应时间分布，可由标签`user_id`切分
 - ds.api.resource.upload.size: (histogram) api上传资源文件大小的分布（bytes）
 - ds.api.resource.download.size: (histogram) api下载资源文件大小的分布（bytes）
 

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/aspect/AccessLogAspect.java

@@ -82,6 +82,9 @@ public class AccessLogAspect {
 
         String traceId = String.valueOf(CodeGenerateUtils.getInstance().genCode());
 
+        int userId = -1;
+        String userName = "NOT LOGIN";
+
         // log request
         if (!annotation.ignoreRequest()) {
             if (attributes != null) {
@@ -91,7 +94,11 @@ public class AccessLogAspect {
                     traceId = traceIdFromHeader;
                 }
                 // handle login info
-                String userName = parseLoginInfo(request);
+                User loginUser = parseLoginInfo(request);
+                if (loginUser != null) {
+                    userName = loginUser.getUserName();
+                    userId = loginUser.getId();
+                }
 
                 // handle args
                 String argsString = parseArgs(proceedingJoinPoint, annotation);
@@ -113,7 +120,10 @@ public class AccessLogAspect {
 
         long costTime = System.currentTimeMillis() - startTime;
         log.info("Call {}:{} success, cost: {}ms", requestMethod, URI, costTime);
-        ApiServerMetrics.recordApiResponseTime(costTime);
+
+        if (userId != -1) {
+            ApiServerMetrics.recordApiResponseTime(costTime, userId);
+        }
 
         return ob;
     }
@@ -160,13 +170,9 @@ public class AccessLogAspect {
         return originalData;
     }
 
-    private String parseLoginInfo(HttpServletRequest request) {
-        String userName = "NOT LOGIN";
+    private User parseLoginInfo(HttpServletRequest request) {
         User loginUser = (User) (request.getAttribute(Constants.SESSION_USER));
-        if (loginUser != null) {
-            userName = loginUser.getUserName();
-        }
-        return userName;
+        return loginUser;
     }
 
 }

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/metrics/ApiServerMetrics.java

@@ -17,10 +17,13 @@
 
 package org.apache.dolphinscheduler.api.metrics;
 
+import java.util.concurrent.TimeUnit;
+
 import lombok.experimental.UtilityClass;
 import io.micrometer.core.instrument.Counter;
 import io.micrometer.core.instrument.DistributionSummary;
 import io.micrometer.core.instrument.Metrics;
+import io.micrometer.core.instrument.Timer;
 
 @UtilityClass
 public class ApiServerMetrics {
@@ -70,13 +73,12 @@ public class ApiServerMetrics {
                     .description("size of download resource files on api")
                     .register(Metrics.globalRegistry);
 
-    private final DistributionSummary apiResponseTimeDistribution =
-            DistributionSummary.builder("ds.api.response.time")
-                    .baseUnit("milliseconds")
-                    .publishPercentiles(0.5, 0.75, 0.95, 0.99)
-                    .publishPercentileHistogram()
-                    .description("response time on api")
-                    .register(Metrics.globalRegistry);
+    static {
+        Timer.builder("ds.api.response.time")
+                .tag("user.id", "dummy")
+                .description("response time on api")
+                .register(Metrics.globalRegistry);
+    }
 
     public void incApiRequestCount() {
         apiRequestCounter.increment();
@@ -106,7 +108,16 @@ public class ApiServerMetrics {
         apiResourceDownloadSizeDistribution.record(size);
     }
 
-    public void recordApiResponseTime(final long milliseconds) {
-        apiResponseTimeDistribution.record(milliseconds);
+    public void recordApiResponseTime(final long milliseconds, final int userId) {
+        Metrics.globalRegistry.timer(
+                "ds.api.response.time",
+                "user.id", String.valueOf(userId)).record(milliseconds, TimeUnit.MILLISECONDS);
+    }
+
+    public void cleanUpApiResponseTimeMetricsByUserId(final int userId) {
+        Metrics.globalRegistry.remove(
+                Metrics.globalRegistry.timer(
+                        "ds.api.response.time",
+                        "user.id", String.valueOf(userId)));
     }
 }

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MetricsCleanUpService.java

@@ -21,4 +21,6 @@ public interface MetricsCleanUpService {
 
     void cleanUpWorkflowMetricsByDefinitionCode(String workflowDefinitionCode);
 
+    void cleanUpApiResponseTimeMetricsByUserId(int userId);
+
 }

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/MetricsCleanUpServiceImpl.java

@@ -17,6 +17,7 @@
 
 package org.apache.dolphinscheduler.api.service.impl;
 
+import org.apache.dolphinscheduler.api.metrics.ApiServerMetrics;
 import org.apache.dolphinscheduler.api.rpc.ApiRpcClient;
 import org.apache.dolphinscheduler.api.service.MetricsCleanUpService;
 import org.apache.dolphinscheduler.common.model.Server;
@@ -59,4 +60,8 @@ public class MetricsCleanUpServiceImpl implements MetricsCleanUpService {
         }
     }
 
+    @Override
+    public void cleanUpApiResponseTimeMetricsByUserId(int userId) {
+        ApiServerMetrics.cleanUpApiResponseTimeMetricsByUserId(userId);
+    }
 }

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java

@@ -22,6 +22,7 @@ import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationCon
 import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent;
 import org.apache.dolphinscheduler.api.enums.Status;
 import org.apache.dolphinscheduler.api.exceptions.ServiceException;
+import org.apache.dolphinscheduler.api.service.MetricsCleanUpService;
 import org.apache.dolphinscheduler.api.service.UsersService;
 import org.apache.dolphinscheduler.api.utils.CheckUtils;
 import org.apache.dolphinscheduler.api.utils.PageInfo;
@@ -129,6 +130,9 @@ public class UsersServiceImpl extends BaseServiceImpl implements UsersService {
     @Autowired
     private K8sNamespaceUserMapper k8sNamespaceUserMapper;
 
+    @Autowired
+    private MetricsCleanUpService metricsCleanUpService;
+
     /**
      * create user, only system admin have permission
      *
@@ -522,6 +526,7 @@ public class UsersServiceImpl extends BaseServiceImpl implements UsersService {
         accessTokenMapper.deleteAccessTokenByUserId(id);
 
         if (userMapper.deleteById(id) > 0) {
+            metricsCleanUpService.cleanUpApiResponseTimeMetricsByUserId(id);
             log.info("User is deleted and id is :{}.", id);
             putMsg(result, Status.SUCCESS);
             return result;

dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java

@@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.api.service;
 import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.USER_MANAGER;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.when;
 
 import org.apache.dolphinscheduler.api.enums.Status;
@@ -112,6 +113,9 @@ public class UsersServiceTest {
     @Mock
     private ResourceUserMapper resourceUserMapper;
 
+    @Mock
+    private MetricsCleanUpService metricsCleanUpService;
+
     @Mock
     private UDFUserMapper udfUserMapper;
 
@@ -338,9 +342,11 @@ public class UsersServiceTest {
 
             // success
             Mockito.when(projectMapper.queryProjectCreatedByUser(1)).thenReturn(null);
+            Mockito.doNothing().when(metricsCleanUpService).cleanUpApiResponseTimeMetricsByUserId(Mockito.anyInt());
             result = usersService.deleteUserById(loginUser, 1);
             logger.info(result.toString());
             Assertions.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
+            Mockito.verify(metricsCleanUpService, times(1)).cleanUpApiResponseTimeMetricsByUserId(Mockito.anyInt());
         } catch (Exception e) {
             logger.error("delete user error", e);
             Assertions.assertTrue(false);