Browse Source

[Feature-6988][MasterServer] add cache manager for workflow (#7090)

* add cache manager for workflow

* [DS-6988][MasterServer] add cache manager for workflow

* cache evict code optimization

* test

Co-authored-by: caishunfeng <534328519@qq.com>
3.0.0/version-upgrade
wind 3 years ago committed by GitHub
parent
commit
ba2b2a67c2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 102
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/aspect/CacheEvictAspect.java
  2. 8
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/QueueServiceImpl.java
  3. 12
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java
  4. 14
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java
  5. 6
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/QueueServiceTest.java
  6. 4
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java
  7. 4
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java
  8. 19
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/CacheType.java
  9. 2
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SpringConnectionFactory.java
  10. 5
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionLogMapper.java
  11. 15
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.java
  12. 13
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessTaskRelationMapper.java
  13. 13
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionLogMapper.java
  14. 29
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TenantMapper.java
  15. 46
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UserMapper.java
  16. 132
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/CacheProcessor.java
  17. 13
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java
  18. 2
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThread.java
  19. 7
      dolphinscheduler-server/src/main/resources/application-master.yaml
  20. 19
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/processor/CacheProcessorTest.java
  21. 2
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cache/CacheNotifyService.java
  22. 20
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cache/impl/CacheKeyGenerator.java
  23. 9
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cache/impl/CacheNotifyServiceImpl.java
  24. 22
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cache/processor/BaseCacheProcessor.java
  25. 22
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cache/processor/QueueCacheProcessor.java
  26. 26
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cache/processor/TenantCacheProcessor.java
  27. 58
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cache/processor/impl/CacheProcessorFactory.java
  28. 50
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cache/processor/impl/QueueCacheProcessorImpl.java
  29. 64
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cache/processor/impl/TenantCacheProcessorImpl.java
  30. 59
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cache/processor/impl/UserCacheProcessorImpl.java
  31. 37
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
  32. 2
      dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/cache/CacheNotifyServiceTest.java
  33. 60
      dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/cache/processor/QueueCacheProcessorTest.java
  34. 78
      dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/cache/processor/TenantCacheProcessorTest.java
  35. 76
      dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/cache/processor/UserCacheProxyTest.java
  36. 30
      dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java

102
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/aspect/CacheEvictAspect.java

@ -0,0 +1,102 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.aspect;
import org.apache.dolphinscheduler.common.enums.CacheType;
import org.apache.dolphinscheduler.remote.command.CacheExpireCommand;
import org.apache.dolphinscheduler.service.cache.CacheNotifyService;
import org.apache.dolphinscheduler.service.cache.impl.CacheKeyGenerator;
import java.lang.reflect.Method;
import org.aspectj.lang.ProceedingJoinPoint;
import org.aspectj.lang.annotation.Around;
import org.aspectj.lang.annotation.Aspect;
import org.aspectj.lang.annotation.Pointcut;
import org.aspectj.lang.reflect.MethodSignature;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cache.annotation.CacheConfig;
import org.springframework.cache.annotation.CacheEvict;
import org.springframework.stereotype.Component;
/**
* aspect for cache evict
*/
@Aspect
@Component
public class CacheEvictAspect {
private static final String UPDATE_BY_ID = "updateById";
@Autowired
private CacheKeyGenerator cacheKeyGenerator;
@Autowired
private CacheNotifyService cacheNotifyService;
@Pointcut("@annotation(org.springframework.cache.annotation.CacheEvict)")
public void cacheEvictPointCut() {
// Do nothing because of it's a pointcut
}
@Around("cacheEvictPointCut()")
public Object doAround(ProceedingJoinPoint proceedingJoinPoint) throws Throwable {
MethodSignature sign = (MethodSignature) proceedingJoinPoint.getSignature();
Method method = sign.getMethod();
Object target = proceedingJoinPoint.getTarget();
Object[] args = proceedingJoinPoint.getArgs();
Object result = proceedingJoinPoint.proceed();
CacheConfig cacheConfig = method.getDeclaringClass().getAnnotation(CacheConfig.class);
CacheEvict cacheEvict = method.getAnnotation(CacheEvict.class);
CacheType cacheType = getCacheType(cacheConfig, cacheEvict);
if (cacheType != null) {
// todo use springEL is better
if (method.getName().equalsIgnoreCase(UPDATE_BY_ID) && args.length == 1) {
Object updateObj = args[0];
cacheNotifyService.notifyMaster(new CacheExpireCommand(cacheType, updateObj).convert2Command());
} else {
Object key = cacheKeyGenerator.generate(target, method, args);
cacheNotifyService.notifyMaster(new CacheExpireCommand(cacheType, key).convert2Command());
}
}
return result;
}
private CacheType getCacheType(CacheConfig cacheConfig, CacheEvict cacheEvict) {
String cacheName = null;
if (cacheEvict.cacheNames().length > 0) {
cacheName = cacheEvict.cacheNames()[0];
}
if (cacheConfig.cacheNames().length > 0) {
cacheName = cacheConfig.cacheNames()[0];
}
if (cacheName == null) {
return null;
}
for (CacheType cacheType : CacheType.values()) {
if (cacheType.getCacheName().equals(cacheName)) {
return cacheType;
}
}
return null;
}
}

8
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/QueueServiceImpl.java

@ -22,13 +22,10 @@ import org.apache.dolphinscheduler.api.service.QueueService;
import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.CacheType;
import org.apache.dolphinscheduler.dao.entity.Queue; import org.apache.dolphinscheduler.dao.entity.Queue;
import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.QueueMapper; import org.apache.dolphinscheduler.dao.mapper.QueueMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper; import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import org.apache.dolphinscheduler.remote.command.CacheExpireCommand;
import org.apache.dolphinscheduler.service.cache.service.CacheNotifyService;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
@ -59,9 +56,6 @@ public class QueueServiceImpl extends BaseServiceImpl implements QueueService {
@Autowired @Autowired
private UserMapper userMapper; private UserMapper userMapper;
@Autowired
private CacheNotifyService cacheNotifyService;
/** /**
* query queue list * query queue list
* *
@ -229,8 +223,6 @@ public class QueueServiceImpl extends BaseServiceImpl implements QueueService {
queueMapper.updateById(queueObj); queueMapper.updateById(queueObj);
cacheNotifyService.notifyMaster(new CacheExpireCommand(CacheType.QUEUE, queueObj).convert2Command());
putMsg(result, Status.SUCCESS); putMsg(result, Status.SUCCESS);
return result; return result;

12
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java

@ -23,7 +23,6 @@ import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.RegexUtils; import org.apache.dolphinscheduler.api.utils.RegexUtils;
import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.CacheType;
import org.apache.dolphinscheduler.common.utils.HadoopUtils; import org.apache.dolphinscheduler.common.utils.HadoopUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
@ -34,8 +33,6 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper;
import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper; import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper; import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import org.apache.dolphinscheduler.remote.command.CacheExpireCommand;
import org.apache.dolphinscheduler.service.cache.service.CacheNotifyService;
import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
@ -70,9 +67,6 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService
@Autowired @Autowired
private UserMapper userMapper; private UserMapper userMapper;
@Autowired
private CacheNotifyService cacheNotifyService;
/** /**
* create tenant * create tenant
* *
@ -220,9 +214,6 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService
tenant.setUpdateTime(now); tenant.setUpdateTime(now);
tenantMapper.updateById(tenant); tenantMapper.updateById(tenant);
// notify master to expire cache
cacheNotifyService.notifyMaster(new CacheExpireCommand(CacheType.TENANT, tenant).convert2Command());
result.put(Constants.STATUS, Status.SUCCESS); result.put(Constants.STATUS, Status.SUCCESS);
result.put(Constants.MSG, Status.SUCCESS.getMsg()); result.put(Constants.MSG, Status.SUCCESS.getMsg());
return result; return result;
@ -282,9 +273,6 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService
tenantMapper.deleteById(id); tenantMapper.deleteById(id);
processInstanceMapper.updateProcessInstanceByTenantId(id, -1); processInstanceMapper.updateProcessInstanceByTenantId(id, -1);
// notify master to expire cache
cacheNotifyService.notifyMaster(new CacheExpireCommand(CacheType.TENANT, tenant).convert2Command());
putMsg(result, Status.SUCCESS); putMsg(result, Status.SUCCESS);
return result; return result;
} }

14
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java

@ -26,7 +26,6 @@ import org.apache.dolphinscheduler.api.utils.CheckUtils;
import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.CacheType;
import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.common.enums.Flag;
import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.EncryptionUtils; import org.apache.dolphinscheduler.common.utils.EncryptionUtils;
@ -53,8 +52,6 @@ import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper; import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper; import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import org.apache.dolphinscheduler.dao.utils.ResourceProcessDefinitionUtils; import org.apache.dolphinscheduler.dao.utils.ResourceProcessDefinitionUtils;
import org.apache.dolphinscheduler.remote.command.CacheExpireCommand;
import org.apache.dolphinscheduler.service.cache.service.CacheNotifyService;
import org.apache.dolphinscheduler.spi.enums.ResourceType; import org.apache.dolphinscheduler.spi.enums.ResourceType;
import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.CollectionUtils;
@ -121,10 +118,6 @@ public class UsersServiceImpl extends BaseServiceImpl implements UsersService {
@Autowired @Autowired
private ProjectMapper projectMapper; private ProjectMapper projectMapper;
@Autowired
private CacheNotifyService cacheNotifyService;
/** /**
* create user, only system admin have permission * create user, only system admin have permission
* *
@ -479,7 +472,6 @@ public class UsersServiceImpl extends BaseServiceImpl implements UsersService {
// updateProcessInstance user // updateProcessInstance user
userMapper.updateById(user); userMapper.updateById(user);
cacheNotifyService.notifyMaster(new CacheExpireCommand(CacheType.USER, user).convert2Command());
putMsg(result, Status.SUCCESS); putMsg(result, Status.SUCCESS);
return result; return result;
@ -531,10 +523,6 @@ public class UsersServiceImpl extends BaseServiceImpl implements UsersService {
userMapper.deleteById(id); userMapper.deleteById(id);
if (user != null) {
cacheNotifyService.notifyMaster(new CacheExpireCommand(CacheType.USER, user).convert2Command());
}
putMsg(result, Status.SUCCESS); putMsg(result, Status.SUCCESS);
return result; return result;
@ -1079,8 +1067,6 @@ public class UsersServiceImpl extends BaseServiceImpl implements UsersService {
user.setUpdateTime(now); user.setUpdateTime(now);
userMapper.updateById(user); userMapper.updateById(user);
cacheNotifyService.notifyMaster(new CacheExpireCommand(CacheType.USER, user).convert2Command());
User responseUser = userMapper.queryByUserNameAccurately(userName); User responseUser = userMapper.queryByUserNameAccurately(userName);
putMsg(result, Status.SUCCESS); putMsg(result, Status.SUCCESS);
result.put(Constants.DATA_LIST, responseUser); result.put(Constants.DATA_LIST, responseUser);

6
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/QueueServiceTest.java

@ -27,7 +27,6 @@ import org.apache.dolphinscheduler.dao.entity.Queue;
import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.QueueMapper; import org.apache.dolphinscheduler.dao.mapper.QueueMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper; import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import org.apache.dolphinscheduler.service.cache.service.CacheNotifyService;
import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.CollectionUtils;
@ -61,9 +60,6 @@ public class QueueServiceTest {
@InjectMocks @InjectMocks
private QueueServiceImpl queueService; private QueueServiceImpl queueService;
@Mock
private CacheNotifyService cacheNotifyService;
@Mock @Mock
private QueueMapper queueMapper; private QueueMapper queueMapper;
@ -187,7 +183,6 @@ public class QueueServiceTest {
/** /**
* create admin user * create admin user
* @return
*/ */
private User getLoginUser() { private User getLoginUser() {
@ -205,7 +200,6 @@ public class QueueServiceTest {
/** /**
* get queue * get queue
* @return
*/ */
private Queue getQueue() { private Queue getQueue() {
Queue queue = new Queue(); Queue queue = new Queue();

4
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java

@ -31,7 +31,6 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper;
import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper; import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper; import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import org.apache.dolphinscheduler.service.cache.service.CacheNotifyService;
import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.CollectionUtils;
@ -63,9 +62,6 @@ public class TenantServiceTest {
@InjectMocks @InjectMocks
private TenantServiceImpl tenantService; private TenantServiceImpl tenantService;
@Mock
private CacheNotifyService cacheNotifyService;
@Mock @Mock
private TenantMapper tenantMapper; private TenantMapper tenantMapper;

4
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java

@ -43,7 +43,6 @@ import org.apache.dolphinscheduler.dao.mapper.ResourceUserMapper;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper; import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper; import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper; import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import org.apache.dolphinscheduler.service.cache.service.CacheNotifyService;
import org.apache.dolphinscheduler.spi.enums.ResourceType; import org.apache.dolphinscheduler.spi.enums.ResourceType;
import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.CollectionUtils;
@ -79,9 +78,6 @@ public class UsersServiceTest {
@InjectMocks @InjectMocks
private UsersServiceImpl usersService; private UsersServiceImpl usersService;
@Mock
private CacheNotifyService cacheNotifyService;
@Mock @Mock
private UserMapper userMapper; private UserMapper userMapper;

19
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/CacheType.java

@ -18,7 +18,20 @@
package org.apache.dolphinscheduler.common.enums; package org.apache.dolphinscheduler.common.enums;
public enum CacheType { public enum CacheType {
TENANT, TENANT("tenant"),
USER, USER("user"),
QUEUE; QUEUE("queue"),
PROCESS_DEFINITION("processDefinition"),
PROCESS_TASK_RELATION("processTaskRelation"),
TASK_DEFINITION("taskDefinition");
CacheType(String cacheName) {
this.cacheName = cacheName;
}
private final String cacheName;
public String getCacheName() {
return cacheName;
}
} }

2
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SpringConnectionFactory.java

@ -42,6 +42,7 @@ import com.baomidou.mybatisplus.extension.spring.MybatisSqlSessionFactoryBean;
@Configuration @Configuration
public class SpringConnectionFactory { public class SpringConnectionFactory {
@Bean @Bean
public PaginationInterceptor paginationInterceptor() { public PaginationInterceptor paginationInterceptor() {
return new PaginationInterceptor(); return new PaginationInterceptor();
@ -60,6 +61,7 @@ public class SpringConnectionFactory {
configuration.setCallSettersOnNulls(true); configuration.setCallSettersOnNulls(true);
configuration.setJdbcTypeForNull(JdbcType.NULL); configuration.setJdbcTypeForNull(JdbcType.NULL);
configuration.addInterceptor(paginationInterceptor()); configuration.addInterceptor(paginationInterceptor());
configuration.setGlobalConfig(new GlobalConfig().setBanner(false)); configuration.setGlobalConfig(new GlobalConfig().setBanner(false));
MybatisSqlSessionFactoryBean sqlSessionFactoryBean = new MybatisSqlSessionFactoryBean(); MybatisSqlSessionFactoryBean sqlSessionFactoryBean = new MybatisSqlSessionFactoryBean();
sqlSessionFactoryBean.setConfiguration(configuration); sqlSessionFactoryBean.setConfiguration(configuration);

5
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionLogMapper.java

@ -23,6 +23,9 @@ import org.apache.ibatis.annotations.Param;
import java.util.List; import java.util.List;
import org.springframework.cache.annotation.CacheConfig;
import org.springframework.cache.annotation.Cacheable;
import com.baomidou.mybatisplus.core.mapper.BaseMapper; import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
@ -30,6 +33,7 @@ import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
/** /**
* process definition log mapper interface * process definition log mapper interface
*/ */
@CacheConfig(cacheNames = "processDefinition")
public interface ProcessDefinitionLogMapper extends BaseMapper<ProcessDefinitionLog> { public interface ProcessDefinitionLogMapper extends BaseMapper<ProcessDefinitionLog> {
/** /**
@ -66,6 +70,7 @@ public interface ProcessDefinitionLogMapper extends BaseMapper<ProcessDefinition
* @param version version number * @param version version number
* @return the process definition version info * @return the process definition version info
*/ */
@Cacheable(sync = true, key = "#processDefinitionCode + '_' + #version")
ProcessDefinitionLog queryByDefinitionCodeAndVersion(@Param("code") long code, @Param("version") int version); ProcessDefinitionLog queryByDefinitionCodeAndVersion(@Param("code") long code, @Param("version") int version);
/** /**

15
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.java

@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.dao.mapper;
import org.apache.dolphinscheduler.dao.entity.DefinitionGroupByUser; import org.apache.dolphinscheduler.dao.entity.DefinitionGroupByUser;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog;
import org.apache.ibatis.annotations.MapKey; import org.apache.ibatis.annotations.MapKey;
import org.apache.ibatis.annotations.Param; import org.apache.ibatis.annotations.Param;
@ -28,13 +27,17 @@ import java.util.Collection;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import org.springframework.cache.annotation.CacheConfig;
import org.springframework.cache.annotation.CacheEvict;
import org.springframework.cache.annotation.Cacheable;
import com.baomidou.mybatisplus.core.mapper.BaseMapper; import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
/** /**
* process definition mapper interface * process definition mapper interface
*/ */
@CacheConfig(cacheNames = "processDefinition", keyGenerator = "cacheKeyGenerator")
public interface ProcessDefinitionMapper extends BaseMapper<ProcessDefinition> { public interface ProcessDefinitionMapper extends BaseMapper<ProcessDefinition> {
/** /**
@ -43,8 +46,15 @@ public interface ProcessDefinitionMapper extends BaseMapper<ProcessDefinition> {
* @param code code * @param code code
* @return process definition * @return process definition
*/ */
@Cacheable(sync = true)
ProcessDefinition queryByCode(@Param("code") long code); ProcessDefinition queryByCode(@Param("code") long code);
/**
* update
*/
@CacheEvict
int updateById(@Param("et") ProcessDefinition processDefinition);
/** /**
* query process definition by code list * query process definition by code list
* *
@ -59,6 +69,7 @@ public interface ProcessDefinitionMapper extends BaseMapper<ProcessDefinition> {
* @param code code * @param code code
* @return delete result * @return delete result
*/ */
@CacheEvict
int deleteByCode(@Param("code") long code); int deleteByCode(@Param("code") long code);
/** /**

13
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessTaskRelationMapper.java

@ -25,11 +25,16 @@ import org.apache.ibatis.annotations.Param;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import org.springframework.cache.annotation.CacheConfig;
import org.springframework.cache.annotation.CacheEvict;
import org.springframework.cache.annotation.Cacheable;
import com.baomidou.mybatisplus.core.mapper.BaseMapper; import com.baomidou.mybatisplus.core.mapper.BaseMapper;
/** /**
* process task relation mapper interface * process task relation mapper interface
*/ */
@CacheConfig(cacheNames = "processTaskRelation", keyGenerator = "cacheKeyGenerator")
public interface ProcessTaskRelationMapper extends BaseMapper<ProcessTaskRelation> { public interface ProcessTaskRelationMapper extends BaseMapper<ProcessTaskRelation> {
/** /**
@ -39,9 +44,16 @@ public interface ProcessTaskRelationMapper extends BaseMapper<ProcessTaskRelatio
* @param processCode processCode * @param processCode processCode
* @return ProcessTaskRelation list * @return ProcessTaskRelation list
*/ */
@Cacheable(sync = true)
List<ProcessTaskRelation> queryByProcessCode(@Param("projectCode") long projectCode, List<ProcessTaskRelation> queryByProcessCode(@Param("projectCode") long projectCode,
@Param("processCode") long processCode); @Param("processCode") long processCode);
/**
* update
*/
@CacheEvict
int updateById(@Param("et") ProcessTaskRelation processTaskRelation);
/** /**
* process task relation by taskCode * process task relation by taskCode
* *
@ -65,6 +77,7 @@ public interface ProcessTaskRelationMapper extends BaseMapper<ProcessTaskRelatio
* @param processCode processCode * @param processCode processCode
* @return int * @return int
*/ */
@CacheEvict
int deleteByCode(@Param("projectCode") long projectCode, int deleteByCode(@Param("projectCode") long projectCode,
@Param("processCode") long processCode); @Param("processCode") long processCode);

13
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionLogMapper.java

@ -25,6 +25,10 @@ import org.apache.ibatis.annotations.Param;
import java.util.Collection; import java.util.Collection;
import java.util.List; import java.util.List;
import org.springframework.cache.annotation.CacheConfig;
import org.springframework.cache.annotation.CacheEvict;
import org.springframework.cache.annotation.Cacheable;
import com.baomidou.mybatisplus.core.mapper.BaseMapper; import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
@ -32,6 +36,7 @@ import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
/** /**
* task definition log mapper interface * task definition log mapper interface
*/ */
@CacheConfig(cacheNames = "taskDefinition", keyGenerator = "cacheKeyGenerator")
public interface TaskDefinitionLogMapper extends BaseMapper<TaskDefinitionLog> { public interface TaskDefinitionLogMapper extends BaseMapper<TaskDefinitionLog> {
/** /**
@ -48,9 +53,16 @@ public interface TaskDefinitionLogMapper extends BaseMapper<TaskDefinitionLog> {
* @param version version * @param version version
* @return task definition log * @return task definition log
*/ */
@Cacheable(sync = true, key = "#taskCode + '_' + #taskDefinitionVersion")
TaskDefinitionLog queryByDefinitionCodeAndVersion(@Param("code") long code, TaskDefinitionLog queryByDefinitionCodeAndVersion(@Param("code") long code,
@Param("version") int version); @Param("version") int version);
/**
* update
*/
@CacheEvict
int updateById(@Param("et") TaskDefinitionLog taskDefinitionLog);
/** /**
* @param taskDefinitions taskDefinition list * @param taskDefinitions taskDefinition list
* @return list * @return list
@ -72,6 +84,7 @@ public interface TaskDefinitionLogMapper extends BaseMapper<TaskDefinitionLog> {
* @param version task definition version * @param version task definition version
* @return delete result * @return delete result
*/ */
@CacheEvict
int deleteByCodeAndVersion(@Param("code") long code, @Param("version") int version); int deleteByCodeAndVersion(@Param("code") long code, @Param("version") int version);
/** /**

29
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TenantMapper.java

@ -14,29 +14,50 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.dao.mapper; package org.apache.dolphinscheduler.dao.mapper;
import org.apache.dolphinscheduler.dao.entity.Tenant; import org.apache.dolphinscheduler.dao.entity.Tenant;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
import org.apache.ibatis.annotations.Param; import org.apache.ibatis.annotations.Param;
import java.util.List; import org.springframework.cache.annotation.CacheConfig;
import org.springframework.cache.annotation.CacheEvict;
import org.springframework.cache.annotation.Cacheable;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
/** /**
* tenant mapper interface * tenant mapper interface
*/ */
@CacheConfig(cacheNames = "tenant", keyGenerator = "cacheKeyGenerator")
public interface TenantMapper extends BaseMapper<Tenant> { public interface TenantMapper extends BaseMapper<Tenant> {
/** /**
* query tenant by id * query tenant by id
*
* @param tenantId tenantId * @param tenantId tenantId
* @return tenant * @return tenant
*/ */
@Cacheable(sync = true)
Tenant queryById(@Param("tenantId") int tenantId); Tenant queryById(@Param("tenantId") int tenantId);
/**
* delete by id
*/
@CacheEvict
int deleteById(int id);
/**
* update
*/
@CacheEvict
int updateById(@Param("et") Tenant tenant);
/** /**
* query tenant by code * query tenant by code
*
* @param tenantCode tenantCode * @param tenantCode tenantCode
* @return tenant * @return tenant
*/ */
@ -44,6 +65,7 @@ public interface TenantMapper extends BaseMapper<Tenant> {
/** /**
* tenant page * tenant page
*
* @param page page * @param page page
* @param searchVal searchVal * @param searchVal searchVal
* @return tenant IPage * @return tenant IPage
@ -53,6 +75,7 @@ public interface TenantMapper extends BaseMapper<Tenant> {
/** /**
* check tenant exist * check tenant exist
*
* @param tenantCode tenantCode * @param tenantCode tenantCode
* @return true if exist else return null * @return true if exist else return null
*/ */

46
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UserMapper.java

@ -14,29 +14,57 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.dao.mapper; package org.apache.dolphinscheduler.dao.mapper;
import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.entity.User;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import org.apache.ibatis.annotations.Param; import org.apache.ibatis.annotations.Param;
import java.util.List; import java.util.List;
import org.springframework.cache.annotation.CacheConfig;
import org.springframework.cache.annotation.CacheEvict;
import org.springframework.cache.annotation.Cacheable;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
/** /**
* user mapper interface * user mapper interface
*/ */
@CacheConfig(cacheNames = "user", keyGenerator = "cacheKeyGenerator")
public interface UserMapper extends BaseMapper<User> { public interface UserMapper extends BaseMapper<User> {
/**
* select by user id
*/
@Cacheable(sync = true)
User selectById(int id);
/**
* delete by id
*/
@CacheEvict
int deleteById(int id);
/**
* update
*/
@CacheEvict
int updateById(@Param("et") User user);
/** /**
* query all general user * query all general user
*
* @return user list * @return user list
*/ */
List<User> queryAllGeneralUser(); List<User> queryAllGeneralUser();
/** /**
* query user by name * query user by name
*
* @param userName userName * @param userName userName
* @return user * @return user
*/ */
@ -44,6 +72,7 @@ public interface UserMapper extends BaseMapper<User> {
/** /**
* query user by userName and password * query user by userName and password
*
* @param userName userName * @param userName userName
* @param password password * @param password password
* @return user * @return user
@ -53,6 +82,7 @@ public interface UserMapper extends BaseMapper<User> {
/** /**
* user page * user page
*
* @param page page * @param page page
* @param userName userName * @param userName userName
* @return user IPage * @return user IPage
@ -62,6 +92,7 @@ public interface UserMapper extends BaseMapper<User> {
/** /**
* query user detail by id * query user detail by id
*
* @param userId userId * @param userId userId
* @return user * @return user
*/ */
@ -69,6 +100,7 @@ public interface UserMapper extends BaseMapper<User> {
/** /**
* query user list by alertgroupId * query user list by alertgroupId
*
* @param alertgroupId alertgroupId * @param alertgroupId alertgroupId
* @return user list * @return user list
*/ */
@ -76,6 +108,7 @@ public interface UserMapper extends BaseMapper<User> {
/** /**
* query user list by tenantId * query user list by tenantId
*
* @param tenantId tenantId * @param tenantId tenantId
* @return user list * @return user list
*/ */
@ -83,6 +116,7 @@ public interface UserMapper extends BaseMapper<User> {
/** /**
* query user by userId * query user by userId
*
* @param userId userId * @param userId userId
* @return user * @return user
*/ */
@ -90,6 +124,7 @@ public interface UserMapper extends BaseMapper<User> {
/** /**
* query user by token * query user by token
*
* @param token token * @param token token
* @return user * @return user
*/ */
@ -97,6 +132,7 @@ public interface UserMapper extends BaseMapper<User> {
/** /**
* query user by queue name * query user by queue name
*
* @param queueName queue name * @param queueName queue name
* @return user list * @return user list
*/ */
@ -104,13 +140,15 @@ public interface UserMapper extends BaseMapper<User> {
/** /**
* check the user exist * check the user exist
* @param queueName queue name *
* @param queue queue name
* @return true if exist else return null * @return true if exist else return null
*/ */
Boolean existUser(@Param("queue") String queue); Boolean existUser(@Param("queue") String queue);
/** /**
* update user with old queue * update user with old queue
*
* @param oldQueue old queue name * @param oldQueue old queue name
* @param newQueue new queue name * @param newQueue new queue name
* @return update rows * @return update rows

132
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/CacheProcessor.java

@ -17,16 +17,24 @@
package org.apache.dolphinscheduler.server.master.processor; package org.apache.dolphinscheduler.server.master.processor;
import org.apache.dolphinscheduler.common.enums.CacheType;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelation;
import org.apache.dolphinscheduler.dao.entity.Queue;
import org.apache.dolphinscheduler.dao.entity.TaskDefinition;
import org.apache.dolphinscheduler.dao.entity.Tenant;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.remote.command.CacheExpireCommand; import org.apache.dolphinscheduler.remote.command.CacheExpireCommand;
import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.Command;
import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.CommandType;
import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor;
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
import org.apache.dolphinscheduler.service.cache.processor.impl.CacheProcessorFactory;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.cache.Cache;
import org.springframework.cache.CacheManager;
import com.google.common.base.Preconditions; import com.google.common.base.Preconditions;
@ -39,11 +47,7 @@ public class CacheProcessor implements NettyRequestProcessor {
private final Logger logger = LoggerFactory.getLogger(CacheProcessor.class); private final Logger logger = LoggerFactory.getLogger(CacheProcessor.class);
private CacheProcessorFactory cacheProcessorFactory; private CacheManager cacheManager;
public CacheProcessor() {
this.cacheProcessorFactory = SpringApplicationContext.getBean(CacheProcessorFactory.class);
}
@Override @Override
public void process(Channel channel, Command command) { public void process(Channel channel, Command command) {
@ -53,6 +57,120 @@ public class CacheProcessor implements NettyRequestProcessor {
logger.info("received command : {}", cacheExpireCommand); logger.info("received command : {}", cacheExpireCommand);
cacheProcessorFactory.getCacheProcessor(cacheExpireCommand.getCacheType()).cacheExpire(cacheExpireCommand.getUpdateObjClass(), cacheExpireCommand.getUpdateObjJson()); this.cacheExpire(cacheExpireCommand);
}
private void cacheExpire(CacheExpireCommand cacheExpireCommand) {
if (cacheManager == null) {
cacheManager = SpringApplicationContext.getBean(CacheManager.class);
}
Object object = JSONUtils.parseObject(cacheExpireCommand.getUpdateObjJson(), cacheExpireCommand.getUpdateObjClass());
if (object == null) {
return;
}
CacheType cacheType = cacheExpireCommand.getCacheType();
switch (cacheType) {
case TENANT:
if (object instanceof Tenant) {
Tenant tenant = (Tenant) object;
tenantCacheExpire(tenant);
}
break;
case USER:
if (object instanceof User) {
User user = (User) object;
userCacheExpire(user);
}
break;
case QUEUE:
if (object instanceof Queue) {
Queue queue = (Queue) object;
queueCacheExpire(queue);
}
break;
case PROCESS_DEFINITION:
if (object instanceof ProcessDefinition) {
ProcessDefinition processDefinition = (ProcessDefinition) object;
processDefinitionCacheExpire(processDefinition);
}
break;
case TASK_DEFINITION:
if (object instanceof TaskDefinition) {
TaskDefinition taskDefinition = (TaskDefinition) object;
taskDefinitionCacheExpire(taskDefinition);
}
break;
case PROCESS_TASK_RELATION:
if (object instanceof ProcessTaskRelation) {
ProcessTaskRelation processTaskRelation = (ProcessTaskRelation) object;
processTaskRelationCacheExpire(processTaskRelation);
}
break;
default:
logger.error("no support cache type:{}", cacheType);
}
// if delete operation, just send key
if (object instanceof String) {
Cache cache = cacheManager.getCache(cacheType.getCacheName());
if (cache != null) {
cache.evict(object);
logger.info("cache evict, type:{}, key:{}", cacheType.getCacheName(), object);
}
}
}
private void tenantCacheExpire(Tenant tenant) {
Cache cache = cacheManager.getCache(CacheType.TENANT.getCacheName());
if (cache != null) {
cache.evict(tenant.getId());
logger.info("cache evict, type:{}, key:{}", CacheType.TENANT.getCacheName(), tenant.getId());
}
}
private void userCacheExpire(User user) {
Cache cache = cacheManager.getCache(CacheType.USER.getCacheName());
if (cache != null) {
cache.evict(user.getId());
logger.info("cache evict, type:{}, key:{}", CacheType.USER.getCacheName(), user.getId());
}
}
private void queueCacheExpire(Queue queue) {
Cache cache = cacheManager.getCache(CacheType.USER.getCacheName());
if (cache != null) {
cache.clear();
logger.info("cache evict, type:{}, clear", CacheType.USER.getCacheName());
}
}
private void processDefinitionCacheExpire(ProcessDefinition processDefinition) {
Cache cache = cacheManager.getCache(CacheType.PROCESS_DEFINITION.getCacheName());
if (cache != null) {
cache.evict(processDefinition.getCode());
cache.evict(processDefinition.getCode() + "_" + processDefinition.getVersion());
logger.info("cache evict, type:{}, key:{}",
CacheType.PROCESS_DEFINITION.getCacheName(), processDefinition.getCode() + "_" + processDefinition.getVersion());
}
}
private void processTaskRelationCacheExpire(ProcessTaskRelation processTaskRelation) {
Cache cache = cacheManager.getCache(CacheType.PROCESS_TASK_RELATION.getCacheName());
if (cache != null) {
cache.evict(processTaskRelation.getProjectCode() + "_" + processTaskRelation.getProcessDefinitionCode());
logger.info("cache evict, type:{}, key:{}",
CacheType.PROCESS_TASK_RELATION.getCacheName(), processTaskRelation.getProjectCode() + "_" + processTaskRelation.getProcessDefinitionCode());
}
}
private void taskDefinitionCacheExpire(TaskDefinition taskDefinition) {
Cache cache = cacheManager.getCache(CacheType.TASK_DEFINITION.getCacheName());
if (cache != null) {
cache.evict(taskDefinition.getCode() + "_" + taskDefinition.getVersion());
logger.info("cache evict, type:{}, key:{}",
CacheType.TASK_DEFINITION.getCacheName(), taskDefinition.getCode() + "_" + taskDefinition.getVersion());
}
} }
} }

13
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java

@ -118,12 +118,6 @@ public class MasterSchedulerService extends Thread {
*/ */
ConcurrentHashMap<Integer, TaskInstance> taskTimeoutCheckList = new ConcurrentHashMap<>(); ConcurrentHashMap<Integer, TaskInstance> taskTimeoutCheckList = new ConcurrentHashMap<>();
/**
* key:code-version
* value: processDefinition
*/
HashMap<String, ProcessDefinition> processDefinitionCacheMaps = new HashMap<>();
private StateWheelExecuteThread stateWheelExecuteThread; private StateWheelExecuteThread stateWheelExecuteThread;
/** /**
@ -195,10 +189,6 @@ public class MasterSchedulerService extends Thread {
return; return;
} }
if (!masterConfig.isCacheProcessDefinition() && processDefinitionCacheMaps.size() > 0) {
processDefinitionCacheMaps.clear();
}
List<ProcessInstance> processInstances = command2ProcessInstance(commands); List<ProcessInstance> processInstances = command2ProcessInstance(commands);
if (CollectionUtils.isEmpty(processInstances)) { if (CollectionUtils.isEmpty(processInstances)) {
return; return;
@ -245,8 +235,7 @@ public class MasterSchedulerService extends Thread {
try { try {
ProcessInstance processInstance = processService.handleCommand(logger, ProcessInstance processInstance = processService.handleCommand(logger,
getLocalAddress(), getLocalAddress(),
command, command);
processDefinitionCacheMaps);
if (processInstance != null) { if (processInstance != null) {
processInstances[index] = processInstance; processInstances[index] = processInstance;
logger.info("handle command command {} end, create process instance {}", logger.info("handle command command {} end, create process instance {}",

2
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThread.java

@ -737,6 +737,7 @@ public class WorkflowExecuteThread implements Runnable {
completeTaskMap.clear(); completeTaskMap.clear();
errorTaskMap.clear(); errorTaskMap.clear();
if (ExecutionStatus.SUBMITTED_SUCCESS != processInstance.getState()) {
List<TaskInstance> validTaskInstanceList = processService.findValidTaskListByProcessId(processInstance.getId()); List<TaskInstance> validTaskInstanceList = processService.findValidTaskListByProcessId(processInstance.getId());
for (TaskInstance task : validTaskInstanceList) { for (TaskInstance task : validTaskInstanceList) {
validTaskMap.put(Long.toString(task.getTaskCode()), task.getId()); validTaskMap.put(Long.toString(task.getTaskCode()), task.getId());
@ -752,6 +753,7 @@ public class WorkflowExecuteThread implements Runnable {
errorTaskMap.put(Long.toString(task.getTaskCode()), task.getId()); errorTaskMap.put(Long.toString(task.getTaskCode()), task.getId());
} }
} }
}
if (processInstance.isComplementData() && complementListDate.size() == 0) { if (processInstance.isComplementData() && complementListDate.size() == 0) {
Map<String, String> cmdParam = JSONUtils.toMap(processInstance.getCommandParam()); Map<String, String> cmdParam = JSONUtils.toMap(processInstance.getCommandParam());

7
dolphinscheduler-server/src/main/resources/application-master.yaml

@ -19,12 +19,15 @@ spring:
name: master-server name: master-server
cache: cache:
# default enable cache, you can disable by `type: none` # default enable cache, you can disable by `type: none`
type: caffeine type: none
cache-names: cache-names:
- tenant - tenant
- user - user
- processDefinition
- processTaskRelation
- taskDefinition
caffeine: caffeine:
spec: maximumSize=100,expireAfterWrite=60s,recordStats spec: maximumSize=100,expireAfterWrite=300s,recordStats
master: master:
listen-port: 5678 listen-port: 5678

19
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/processor/CacheProcessorTest.java

@ -22,19 +22,17 @@ import org.apache.dolphinscheduler.dao.entity.Tenant;
import org.apache.dolphinscheduler.remote.command.CacheExpireCommand; import org.apache.dolphinscheduler.remote.command.CacheExpireCommand;
import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.Command;
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
import org.apache.dolphinscheduler.service.cache.processor.TenantCacheProcessor;
import org.apache.dolphinscheduler.service.cache.processor.impl.CacheProcessorFactory;
import org.apache.dolphinscheduler.service.cache.processor.impl.TenantCacheProcessorImpl;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock; import org.mockito.Mock;
import org.mockito.Mockito; import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito; import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner; import org.powermock.modules.junit4.PowerMockRunner;
import org.springframework.cache.Cache;
import org.springframework.cache.CacheManager;
import io.netty.channel.Channel; import io.netty.channel.Channel;
@ -47,21 +45,20 @@ public class CacheProcessorTest {
private CacheProcessor cacheProcessor; private CacheProcessor cacheProcessor;
@InjectMocks
private TenantCacheProcessorImpl tenantCacheProcessor;
@Mock @Mock
private Channel channel; private Channel channel;
@Mock @Mock
private CacheProcessorFactory cacheProcessorFactory; private CacheManager cacheManager;
@Mock
private Cache cache;
@Before @Before
public void before() { public void before() {
PowerMockito.mockStatic(SpringApplicationContext.class); PowerMockito.mockStatic(SpringApplicationContext.class);
PowerMockito.when(SpringApplicationContext.getBean(TenantCacheProcessor.class)).thenReturn(tenantCacheProcessor); PowerMockito.when(SpringApplicationContext.getBean(CacheManager.class)).thenReturn(cacheManager);
PowerMockito.when(SpringApplicationContext.getBean(CacheProcessorFactory.class)).thenReturn(cacheProcessorFactory); Mockito.when(cacheManager.getCache(CacheType.TENANT.getCacheName())).thenReturn(cache);
Mockito.when(cacheProcessorFactory.getCacheProcessor(CacheType.TENANT)).thenReturn(tenantCacheProcessor);
cacheProcessor = new CacheProcessor(); cacheProcessor = new CacheProcessor();
} }

2
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cache/service/CacheNotifyService.java → dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cache/CacheNotifyService.java vendored

@ -15,7 +15,7 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.service.cache.service; package org.apache.dolphinscheduler.service.cache;
import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.Command;

20
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cache/processor/UserCacheProcessor.java → dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cache/impl/CacheKeyGenerator.java vendored

@ -15,12 +15,22 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.service.cache.processor; package org.apache.dolphinscheduler.service.cache.impl;
import org.apache.dolphinscheduler.dao.entity.User; import java.lang.reflect.Method;
public interface UserCacheProcessor extends BaseCacheProcessor { import org.springframework.cache.interceptor.KeyGenerator;
void update(int userId); import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils;
User selectById(int userId); /**
* custom cache key generator
*/
@Component
public class CacheKeyGenerator implements KeyGenerator {
@Override
public Object generate(Object target, Method method, Object... params) {
return StringUtils.arrayToDelimitedString(params, "_");
}
} }

9
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cache/service/impl/CacheNotifyServiceImpl.java → dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cache/impl/CacheNotifyServiceImpl.java vendored

@ -15,7 +15,7 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.service.cache.service.impl; package org.apache.dolphinscheduler.service.cache.impl;
import org.apache.dolphinscheduler.common.enums.NodeType; import org.apache.dolphinscheduler.common.enums.NodeType;
import org.apache.dolphinscheduler.common.model.Server; import org.apache.dolphinscheduler.common.model.Server;
@ -24,7 +24,7 @@ import org.apache.dolphinscheduler.remote.command.Command;
import org.apache.dolphinscheduler.remote.config.NettyClientConfig; import org.apache.dolphinscheduler.remote.config.NettyClientConfig;
import org.apache.dolphinscheduler.remote.processor.NettyRemoteChannel; import org.apache.dolphinscheduler.remote.processor.NettyRemoteChannel;
import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.remote.utils.Host;
import org.apache.dolphinscheduler.service.cache.service.CacheNotifyService; import org.apache.dolphinscheduler.service.cache.CacheNotifyService;
import org.apache.dolphinscheduler.service.registry.RegistryClient; import org.apache.dolphinscheduler.service.registry.RegistryClient;
import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.CollectionUtils;
@ -114,7 +114,7 @@ public class CacheNotifyServiceImpl implements CacheNotifyService {
@Override @Override
public void notifyMaster(Command command) { public void notifyMaster(Command command) {
logger.info("send result, command:{}", command.toString()); logger.info("send result, command:{}", command.toString());
try {
List<Server> serverList = registryClient.getServerList(NodeType.MASTER); List<Server> serverList = registryClient.getServerList(NodeType.MASTER);
if (CollectionUtils.isEmpty(serverList)) { if (CollectionUtils.isEmpty(serverList)) {
return; return;
@ -128,5 +128,8 @@ public class CacheNotifyServiceImpl implements CacheNotifyService {
} }
nettyRemoteChannel.writeAndFlush(command); nettyRemoteChannel.writeAndFlush(command);
} }
} catch (Exception e) {
logger.error("notify master error", e);
}
} }
} }

22
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cache/processor/BaseCacheProcessor.java vendored

@ -1,22 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.service.cache.processor;
public interface BaseCacheProcessor {
void cacheExpire(Class updateObjClass, String updateObjJson);
}

22
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cache/processor/QueueCacheProcessor.java vendored

@ -1,22 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.service.cache.processor;
public interface QueueCacheProcessor extends BaseCacheProcessor {
public void expireAllUserCache();
}

26
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cache/processor/TenantCacheProcessor.java vendored

@ -1,26 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.service.cache.processor;
import org.apache.dolphinscheduler.dao.entity.Tenant;
public interface TenantCacheProcessor extends BaseCacheProcessor {
void update(int tenantId);
Tenant queryById(int tenantId);
}

58
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cache/processor/impl/CacheProcessorFactory.java vendored

@ -1,58 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.service.cache.processor.impl;
import org.apache.dolphinscheduler.common.enums.CacheType;
import org.apache.dolphinscheduler.service.cache.processor.BaseCacheProcessor;
import org.apache.dolphinscheduler.service.cache.processor.QueueCacheProcessor;
import org.apache.dolphinscheduler.service.cache.processor.TenantCacheProcessor;
import org.apache.dolphinscheduler.service.cache.processor.UserCacheProcessor;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import javax.annotation.PostConstruct;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@Component
public class CacheProcessorFactory {
@Autowired
private TenantCacheProcessor tenantCacheProcessor;
@Autowired
private UserCacheProcessor userCacheProcessor;
@Autowired
private QueueCacheProcessor queueCacheProcessor;
Map<CacheType, BaseCacheProcessor> cacheProcessorMap = new ConcurrentHashMap<>();
@PostConstruct
private void init() {
cacheProcessorMap.put(CacheType.TENANT, tenantCacheProcessor);
cacheProcessorMap.put(CacheType.USER, userCacheProcessor);
cacheProcessorMap.put(CacheType.QUEUE, queueCacheProcessor);
}
public BaseCacheProcessor getCacheProcessor(CacheType cacheType) {
return cacheProcessorMap.get(cacheType);
}
}

50
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cache/processor/impl/QueueCacheProcessorImpl.java vendored

@ -1,50 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.service.cache.processor.impl;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.entity.Queue;
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
import org.apache.dolphinscheduler.service.cache.processor.QueueCacheProcessor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.cache.annotation.CacheEvict;
import org.springframework.stereotype.Component;
@Component
public class QueueCacheProcessorImpl implements QueueCacheProcessor {
private final Logger logger = LoggerFactory.getLogger(getClass());
@Override
@CacheEvict(cacheNames = "user", allEntries = true)
public void expireAllUserCache() {
// just evict cache
logger.debug("expire all user cache");
}
@Override
public void cacheExpire(Class updateObjClass, String updateObjJson) {
Queue updateQueue = (Queue) JSONUtils.parseObject(updateObjJson, updateObjClass);
if (updateQueue == null) {
return;
}
SpringApplicationContext.getBean(QueueCacheProcessor.class).expireAllUserCache();
}
}

64
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cache/processor/impl/TenantCacheProcessorImpl.java vendored

@ -1,64 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.service.cache.processor.impl;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.entity.Tenant;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
import org.apache.dolphinscheduler.service.cache.processor.TenantCacheProcessor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cache.annotation.CacheConfig;
import org.springframework.cache.annotation.CacheEvict;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.stereotype.Component;
@Component
@CacheConfig(cacheNames = "tenant")
public class TenantCacheProcessorImpl implements TenantCacheProcessor {
private final Logger logger = LoggerFactory.getLogger(getClass());
@Autowired
private TenantMapper tenantMapper;
@Override
@CacheEvict
public void update(int tenantId) {
// just evict cache
}
@Override
@Cacheable(sync = true)
public Tenant queryById(int tenantId) {
logger.debug("tenant cache proxy, tenantId:{}", tenantId);
return tenantMapper.queryById(tenantId);
}
@Override
public void cacheExpire(Class updateObjClass, String updateObjJson) {
Tenant updateTenant = (Tenant) JSONUtils.parseObject(updateObjJson, updateObjClass);
if (updateTenant == null) {
return;
}
SpringApplicationContext.getBean(TenantCacheProcessor.class).update(updateTenant.getId());
}
}

59
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cache/processor/impl/UserCacheProcessorImpl.java vendored

@ -1,59 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.service.cache.processor.impl;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
import org.apache.dolphinscheduler.service.cache.processor.UserCacheProcessor;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cache.annotation.CacheConfig;
import org.springframework.cache.annotation.CacheEvict;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.stereotype.Component;
@Component
@CacheConfig(cacheNames = "user")
public class UserCacheProcessorImpl implements UserCacheProcessor {
@Autowired
private UserMapper userMapper;
@Override
@CacheEvict
public void update(int userId) {
// just evict cache
}
@Override
@Cacheable(sync = true)
public User selectById(int userId) {
return userMapper.selectById(userId);
}
@Override
public void cacheExpire(Class updateObjClass, String updateObjJson) {
User user = (User) JSONUtils.parseObject(updateObjJson, updateObjClass);
if (user == null) {
return;
}
SpringApplicationContext.getBean(UserCacheProcessor.class).update(user.getId());
}
}

37
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java

@ -100,15 +100,15 @@ import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskGroupMapper; import org.apache.dolphinscheduler.dao.mapper.TaskGroupMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskGroupQueueMapper; import org.apache.dolphinscheduler.dao.mapper.TaskGroupQueueMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper; import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import org.apache.dolphinscheduler.dao.utils.DagHelper; import org.apache.dolphinscheduler.dao.utils.DagHelper;
import org.apache.dolphinscheduler.remote.command.StateEventChangeCommand; import org.apache.dolphinscheduler.remote.command.StateEventChangeCommand;
import org.apache.dolphinscheduler.remote.command.TaskEventChangeCommand; import org.apache.dolphinscheduler.remote.command.TaskEventChangeCommand;
import org.apache.dolphinscheduler.remote.processor.StateEventCallbackService; import org.apache.dolphinscheduler.remote.processor.StateEventCallbackService;
import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.remote.utils.Host;
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
import org.apache.dolphinscheduler.service.cache.processor.TenantCacheProcessor;
import org.apache.dolphinscheduler.service.cache.processor.UserCacheProcessor;
import org.apache.dolphinscheduler.service.exceptions.ServiceException; import org.apache.dolphinscheduler.service.exceptions.ServiceException;
import org.apache.dolphinscheduler.service.log.LogClientService; import org.apache.dolphinscheduler.service.log.LogClientService;
import org.apache.dolphinscheduler.service.quartz.cron.CronUtils; import org.apache.dolphinscheduler.service.quartz.cron.CronUtils;
@ -155,7 +155,7 @@ public class ProcessService {
ExecutionStatus.READY_STOP.ordinal()}; ExecutionStatus.READY_STOP.ordinal()};
@Autowired @Autowired
private UserCacheProcessor userCacheProcessor; private UserMapper userMapper;
@Autowired @Autowired
private ProcessDefinitionMapper processDefineMapper; private ProcessDefinitionMapper processDefineMapper;
@ -194,7 +194,7 @@ public class ProcessService {
private ErrorCommandMapper errorCommandMapper; private ErrorCommandMapper errorCommandMapper;
@Autowired @Autowired
private TenantCacheProcessor tenantCacheProcessor; private TenantMapper tenantMapper;
@Autowired @Autowired
private ProjectMapper projectMapper; private ProjectMapper projectMapper;
@ -233,8 +233,8 @@ public class ProcessService {
* @return process instance * @return process instance
*/ */
@Transactional @Transactional
public ProcessInstance handleCommand(Logger logger, String host, Command command, HashMap<String, ProcessDefinition> processDefinitionCacheMaps) { public ProcessInstance handleCommand(Logger logger, String host, Command command) {
ProcessInstance processInstance = constructProcessInstance(command, host, processDefinitionCacheMaps); ProcessInstance processInstance = constructProcessInstance(command, host);
// cannot construct process instance, return null // cannot construct process instance, return null
if (processInstance == null) { if (processInstance == null) {
logger.error("scan command, command parameter is error: {}", command); logger.error("scan command, command parameter is error: {}", command);
@ -739,7 +739,7 @@ public class ProcessService {
public Tenant getTenantForProcess(int tenantId, int userId) { public Tenant getTenantForProcess(int tenantId, int userId) {
Tenant tenant = null; Tenant tenant = null;
if (tenantId >= 0) { if (tenantId >= 0) {
tenant = tenantCacheProcessor.queryById(tenantId); tenant = tenantMapper.queryById(tenantId);
} }
if (userId == 0) { if (userId == 0) {
@ -747,8 +747,8 @@ public class ProcessService {
} }
if (tenant == null) { if (tenant == null) {
User user = userCacheProcessor.selectById(userId); User user = userMapper.selectById(userId);
tenant = tenantCacheProcessor.queryById(user.getTenantId()); tenant = tenantMapper.queryById(user.getTenantId());
} }
return tenant; return tenant;
} }
@ -794,19 +794,12 @@ public class ProcessService {
* @param host host * @param host host
* @return process instance * @return process instance
*/ */
private ProcessInstance constructProcessInstance(Command command, String host, HashMap<String, ProcessDefinition> processDefinitionCacheMaps) { private ProcessInstance constructProcessInstance(Command command, String host) {
ProcessInstance processInstance; ProcessInstance processInstance;
ProcessDefinition processDefinition; ProcessDefinition processDefinition;
CommandType commandType = command.getCommandType(); CommandType commandType = command.getCommandType();
String key = String.format("%d-%d", command.getProcessDefinitionCode(), command.getProcessDefinitionVersion());
if (processDefinitionCacheMaps.containsKey(key)) {
processDefinition = processDefinitionCacheMaps.get(key);
} else {
processDefinition = this.findProcessDefinition(command.getProcessDefinitionCode(), command.getProcessDefinitionVersion()); processDefinition = this.findProcessDefinition(command.getProcessDefinitionCode(), command.getProcessDefinitionVersion());
if (processDefinition != null) {
processDefinitionCacheMaps.put(key, processDefinition);
}
}
if (processDefinition == null) { if (processDefinition == null) {
logger.error("cannot find the work process define! define code : {}", command.getProcessDefinitionCode()); logger.error("cannot find the work process define! define code : {}", command.getProcessDefinitionCode());
return null; return null;
@ -1964,11 +1957,11 @@ public class ProcessService {
return StringUtils.EMPTY; return StringUtils.EMPTY;
} }
int userId = resourceList.get(0).getUserId(); int userId = resourceList.get(0).getUserId();
User user = userCacheProcessor.selectById(userId); User user = userMapper.selectById(userId);
if (Objects.isNull(user)) { if (Objects.isNull(user)) {
return StringUtils.EMPTY; return StringUtils.EMPTY;
} }
Tenant tenant = tenantCacheProcessor.queryById(user.getTenantId()); Tenant tenant = tenantMapper.queryById(user.getTenantId());
if (Objects.isNull(tenant)) { if (Objects.isNull(tenant)) {
return StringUtils.EMPTY; return StringUtils.EMPTY;
} }
@ -2038,7 +2031,7 @@ public class ProcessService {
if (processInstance == null) { if (processInstance == null) {
return queue; return queue;
} }
User executor = userCacheProcessor.selectById(processInstance.getExecutorId()); User executor = userMapper.selectById(processInstance.getExecutorId());
if (executor != null) { if (executor != null) {
queue = executor.getQueue(); queue = executor.getQueue();
} }
@ -2149,7 +2142,7 @@ public class ProcessService {
* @return User * @return User
*/ */
public User getUserById(int userId) { public User getUserById(int userId) {
return userCacheProcessor.selectById(userId); return userMapper.selectById(userId);
} }
/** /**

2
dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/cache/CacheNotifyServiceTest.java vendored

@ -26,7 +26,7 @@ import org.apache.dolphinscheduler.remote.command.CacheExpireCommand;
import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.Command;
import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.CommandType;
import org.apache.dolphinscheduler.remote.config.NettyServerConfig; import org.apache.dolphinscheduler.remote.config.NettyServerConfig;
import org.apache.dolphinscheduler.service.cache.service.impl.CacheNotifyServiceImpl; import org.apache.dolphinscheduler.service.cache.impl.CacheNotifyServiceImpl;
import org.apache.dolphinscheduler.service.registry.RegistryClient; import org.apache.dolphinscheduler.service.registry.RegistryClient;
import java.util.ArrayList; import java.util.ArrayList;

60
dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/cache/processor/QueueCacheProcessorTest.java vendored

@ -1,60 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.service.cache.processor;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.entity.Queue;
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
import org.apache.dolphinscheduler.service.cache.processor.impl.QueueCacheProcessorImpl;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
/**
* tenant cache proxy test
*/
@RunWith(PowerMockRunner.class)
@PrepareForTest({SpringApplicationContext.class})
public class QueueCacheProcessorTest {
@Rule
public final ExpectedException exception = ExpectedException.none();
@InjectMocks
private QueueCacheProcessorImpl queueCacheProcessor;
@Before
public void before() {
PowerMockito.mockStatic(SpringApplicationContext.class);
PowerMockito.when(SpringApplicationContext.getBean(QueueCacheProcessor.class)).thenReturn(queueCacheProcessor);
}
@Test
public void testCacheExpire() {
Queue queue = new Queue();
queue.setId(100);
queueCacheProcessor.cacheExpire(Queue.class, JSONUtils.toJsonString(queue));
}
}

78
dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/cache/processor/TenantCacheProcessorTest.java vendored

@ -1,78 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.service.cache.processor;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.entity.Tenant;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
import org.apache.dolphinscheduler.service.cache.processor.impl.TenantCacheProcessorImpl;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
/**
* tenant cache proxy test
*/
@RunWith(PowerMockRunner.class)
@PrepareForTest({SpringApplicationContext.class})
public class TenantCacheProcessorTest {
@Rule
public final ExpectedException exception = ExpectedException.none();
@InjectMocks
private TenantCacheProcessorImpl tenantCacheProcessor;
@Mock
private TenantMapper tenantMapper;
@Before
public void before() {
PowerMockito.mockStatic(SpringApplicationContext.class);
PowerMockito.when(SpringApplicationContext.getBean(TenantCacheProcessor.class)).thenReturn(tenantCacheProcessor);
}
@Test
public void testQueryById() {
Tenant tenant1 = new Tenant();
tenant1.setId(100);
tenant1.setDescription("test1");
Mockito.when(tenantMapper.queryById(100)).thenReturn(tenant1);
Assert.assertEquals(tenant1, tenantCacheProcessor.queryById(100));
}
@Test
public void testCacheExpire() {
Tenant tenant1 = new Tenant();
tenant1.setId(100);
tenant1.setDescription("test1");
tenantCacheProcessor.cacheExpire(Tenant.class, JSONUtils.toJsonString(tenant1));
}
}

76
dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/cache/processor/UserCacheProxyTest.java vendored

@ -1,76 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.service.cache.processor;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
import org.apache.dolphinscheduler.service.cache.processor.impl.UserCacheProcessorImpl;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
/**
* tenant cache proxy test
*/
@RunWith(PowerMockRunner.class)
@PrepareForTest({SpringApplicationContext.class})
public class UserCacheProxyTest {
@Rule
public final ExpectedException exception = ExpectedException.none();
@InjectMocks
private UserCacheProcessorImpl userCacheProcessor;
@Mock
private UserMapper userMapper;
@Before
public void before() {
PowerMockito.mockStatic(SpringApplicationContext.class);
PowerMockito.when(SpringApplicationContext.getBean(UserCacheProcessor.class)).thenReturn(userCacheProcessor);
}
@Test
public void testQueryById() {
User user1 = new User();
user1.setId(100);
Mockito.when(userMapper.selectById(100)).thenReturn(user1);
Assert.assertEquals(user1, userCacheProcessor.selectById(100));
}
@Test
public void testCacheExpire() {
User user = new User();
user.setId(100);
userCacheProcessor.cacheExpire(User.class, JSONUtils.toJsonString(user));
}
}

30
dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java

@ -64,7 +64,7 @@ import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskGroupMapper; import org.apache.dolphinscheduler.dao.mapper.TaskGroupMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskGroupQueueMapper; import org.apache.dolphinscheduler.dao.mapper.TaskGroupQueueMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper;
import org.apache.dolphinscheduler.service.cache.processor.impl.UserCacheProcessorImpl; import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import org.apache.dolphinscheduler.service.exceptions.ServiceException; import org.apache.dolphinscheduler.service.exceptions.ServiceException;
import org.apache.dolphinscheduler.service.quartz.cron.CronUtilsTest; import org.apache.dolphinscheduler.service.quartz.cron.CronUtilsTest;
@ -116,7 +116,7 @@ public class ProcessServiceTest {
@Mock @Mock
private ProcessInstanceMapper processInstanceMapper; private ProcessInstanceMapper processInstanceMapper;
@Mock @Mock
private UserCacheProcessorImpl userCacheProcessor; private UserMapper userMapper;
@Mock @Mock
private TaskInstanceMapper taskInstanceMapper; private TaskInstanceMapper taskInstanceMapper;
@Mock @Mock
@ -134,8 +134,6 @@ public class ProcessServiceTest {
@Mock @Mock
private TaskGroupQueueMapper taskGroupQueueMapper; private TaskGroupQueueMapper taskGroupQueueMapper;
private HashMap<String, ProcessDefinition> processDefinitionCacheMaps = new HashMap<>();
@Test @Test
public void testCreateSubCommand() { public void testCreateSubCommand() {
ProcessInstance parentInstance = new ProcessInstance(); ProcessInstance parentInstance = new ProcessInstance();
@ -263,7 +261,7 @@ public class ProcessServiceTest {
command.setCommandType(CommandType.REPEAT_RUNNING); command.setCommandType(CommandType.REPEAT_RUNNING);
command.setCommandParam("{\"" + CMD_PARAM_RECOVER_PROCESS_ID_STRING + "\":\"111\",\"" command.setCommandParam("{\"" + CMD_PARAM_RECOVER_PROCESS_ID_STRING + "\":\"111\",\""
+ CMD_PARAM_SUB_PROCESS_DEFINE_CODE + "\":\"222\"}"); + CMD_PARAM_SUB_PROCESS_DEFINE_CODE + "\":\"222\"}");
Assert.assertNull(processService.handleCommand(logger, host, command, processDefinitionCacheMaps)); Assert.assertNull(processService.handleCommand(logger, host, command));
int definitionVersion = 1; int definitionVersion = 1;
long definitionCode = 123; long definitionCode = 123;
@ -298,7 +296,7 @@ public class ProcessServiceTest {
Mockito.when(processDefineLogMapper.queryByDefinitionCodeAndVersion(processInstance.getProcessDefinitionCode(), Mockito.when(processDefineLogMapper.queryByDefinitionCodeAndVersion(processInstance.getProcessDefinitionCode(),
processInstance.getProcessDefinitionVersion())).thenReturn(new ProcessDefinitionLog(processDefinition)); processInstance.getProcessDefinitionVersion())).thenReturn(new ProcessDefinitionLog(processDefinition));
Mockito.when(processInstanceMapper.queryDetailById(222)).thenReturn(processInstance); Mockito.when(processInstanceMapper.queryDetailById(222)).thenReturn(processInstance);
Assert.assertNotNull(processService.handleCommand(logger, host, command1, processDefinitionCacheMaps)); Assert.assertNotNull(processService.handleCommand(logger, host, command1));
Command command2 = new Command(); Command command2 = new Command();
command2.setId(2); command2.setId(2);
@ -308,7 +306,7 @@ public class ProcessServiceTest {
command2.setCommandType(CommandType.RECOVER_SUSPENDED_PROCESS); command2.setCommandType(CommandType.RECOVER_SUSPENDED_PROCESS);
command2.setProcessInstanceId(processInstanceId); command2.setProcessInstanceId(processInstanceId);
Mockito.when(commandMapper.deleteById(2)).thenReturn(1); Mockito.when(commandMapper.deleteById(2)).thenReturn(1);
Assert.assertNotNull(processService.handleCommand(logger, host, command2, processDefinitionCacheMaps)); Assert.assertNotNull(processService.handleCommand(logger, host, command2));
Command command3 = new Command(); Command command3 = new Command();
command3.setId(3); command3.setId(3);
@ -318,7 +316,7 @@ public class ProcessServiceTest {
command3.setCommandParam("{\"WaitingThreadInstanceId\":222}"); command3.setCommandParam("{\"WaitingThreadInstanceId\":222}");
command3.setCommandType(CommandType.START_FAILURE_TASK_PROCESS); command3.setCommandType(CommandType.START_FAILURE_TASK_PROCESS);
Mockito.when(commandMapper.deleteById(3)).thenReturn(1); Mockito.when(commandMapper.deleteById(3)).thenReturn(1);
Assert.assertNotNull(processService.handleCommand(logger, host, command3, processDefinitionCacheMaps)); Assert.assertNotNull(processService.handleCommand(logger, host, command3));
Command command4 = new Command(); Command command4 = new Command();
command4.setId(4); command4.setId(4);
@ -328,7 +326,7 @@ public class ProcessServiceTest {
command4.setCommandType(CommandType.REPEAT_RUNNING); command4.setCommandType(CommandType.REPEAT_RUNNING);
command4.setProcessInstanceId(processInstanceId); command4.setProcessInstanceId(processInstanceId);
Mockito.when(commandMapper.deleteById(4)).thenReturn(1); Mockito.when(commandMapper.deleteById(4)).thenReturn(1);
Assert.assertNotNull(processService.handleCommand(logger, host, command4, processDefinitionCacheMaps)); Assert.assertNotNull(processService.handleCommand(logger, host, command4));
Command command5 = new Command(); Command command5 = new Command();
command5.setId(5); command5.setId(5);
@ -342,7 +340,7 @@ public class ProcessServiceTest {
command5.setCommandType(CommandType.START_PROCESS); command5.setCommandType(CommandType.START_PROCESS);
command5.setDryRun(Constants.DRY_RUN_FLAG_NO); command5.setDryRun(Constants.DRY_RUN_FLAG_NO);
Mockito.when(commandMapper.deleteById(5)).thenReturn(1); Mockito.when(commandMapper.deleteById(5)).thenReturn(1);
ProcessInstance processInstance1 = processService.handleCommand(logger, host, command5, processDefinitionCacheMaps); ProcessInstance processInstance1 = processService.handleCommand(logger, host, command5);
Assert.assertTrue(processInstance1.getGlobalParams().contains("\"testStartParam1\"")); Assert.assertTrue(processInstance1.getGlobalParams().contains("\"testStartParam1\""));
ProcessDefinition processDefinition1 = new ProcessDefinition(); ProcessDefinition processDefinition1 = new ProcessDefinition();
@ -367,7 +365,7 @@ public class ProcessServiceTest {
Mockito.when(processInstanceMapper.queryDetailById(223)).thenReturn(processInstance2); Mockito.when(processInstanceMapper.queryDetailById(223)).thenReturn(processInstance2);
Mockito.when(processDefineMapper.queryByCode(11L)).thenReturn(processDefinition1); Mockito.when(processDefineMapper.queryByCode(11L)).thenReturn(processDefinition1);
Mockito.when(commandMapper.deleteById(1)).thenReturn(1); Mockito.when(commandMapper.deleteById(1)).thenReturn(1);
Assert.assertNotNull(processService.handleCommand(logger, host, command1, processDefinitionCacheMaps)); Assert.assertNotNull(processService.handleCommand(logger, host, command1));
Command command6 = new Command(); Command command6 = new Command();
command6.setId(6); command6.setId(6);
@ -378,7 +376,7 @@ public class ProcessServiceTest {
Mockito.when(processInstanceMapper.queryByProcessDefineCodeAndStatusAndNextId(11L, Constants.RUNNING_PROCESS_STATE, 223)).thenReturn(lists); Mockito.when(processInstanceMapper.queryByProcessDefineCodeAndStatusAndNextId(11L, Constants.RUNNING_PROCESS_STATE, 223)).thenReturn(lists);
Mockito.when(processInstanceMapper.updateNextProcessIdById(223, 222)).thenReturn(true); Mockito.when(processInstanceMapper.updateNextProcessIdById(223, 222)).thenReturn(true);
Mockito.when(commandMapper.deleteById(6)).thenReturn(1); Mockito.when(commandMapper.deleteById(6)).thenReturn(1);
ProcessInstance processInstance6 = processService.handleCommand(logger, host, command6, processDefinitionCacheMaps); ProcessInstance processInstance6 = processService.handleCommand(logger, host, command6);
Assert.assertTrue(processInstance6 != null); Assert.assertTrue(processInstance6 != null);
processDefinition1.setExecutionType(ProcessExecutionTypeEnum.SERIAL_DISCARD); processDefinition1.setExecutionType(ProcessExecutionTypeEnum.SERIAL_DISCARD);
@ -397,7 +395,7 @@ public class ProcessServiceTest {
command7.setProcessDefinitionVersion(1); command7.setProcessDefinitionVersion(1);
Mockito.when(commandMapper.deleteById(7)).thenReturn(1); Mockito.when(commandMapper.deleteById(7)).thenReturn(1);
Mockito.when(processInstanceMapper.queryByProcessDefineCodeAndStatusAndNextId(11L, Constants.RUNNING_PROCESS_STATE, 224)).thenReturn(null); Mockito.when(processInstanceMapper.queryByProcessDefineCodeAndStatusAndNextId(11L, Constants.RUNNING_PROCESS_STATE, 224)).thenReturn(null);
ProcessInstance processInstance8 = processService.handleCommand(logger, host, command7, processDefinitionCacheMaps); ProcessInstance processInstance8 = processService.handleCommand(logger, host, command7);
Assert.assertTrue(processInstance8 == null); Assert.assertTrue(processInstance8 == null);
ProcessDefinition processDefinition2 = new ProcessDefinition(); ProcessDefinition processDefinition2 = new ProcessDefinition();
@ -421,7 +419,7 @@ public class ProcessServiceTest {
Mockito.when(processInstanceMapper.queryByProcessDefineCodeAndStatusAndNextId(12L, Constants.RUNNING_PROCESS_STATE, 0)).thenReturn(lists); Mockito.when(processInstanceMapper.queryByProcessDefineCodeAndStatusAndNextId(12L, Constants.RUNNING_PROCESS_STATE, 0)).thenReturn(lists);
Mockito.when(processInstanceMapper.updateById(processInstance)).thenReturn(1); Mockito.when(processInstanceMapper.updateById(processInstance)).thenReturn(1);
Mockito.when(commandMapper.deleteById(9)).thenReturn(1); Mockito.when(commandMapper.deleteById(9)).thenReturn(1);
ProcessInstance processInstance10 = processService.handleCommand(logger, host, command9, processDefinitionCacheMaps); ProcessInstance processInstance10 = processService.handleCommand(logger, host, command9);
Assert.assertTrue(processInstance10 == null); Assert.assertTrue(processInstance10 == null);
} }
@ -462,14 +460,14 @@ public class ProcessServiceTest {
Mockito.when(processInstanceMapper.queryDetailById(222)).thenReturn(processInstance); Mockito.when(processInstanceMapper.queryDetailById(222)).thenReturn(processInstance);
// will throw exception when command id is 0 and delete fail // will throw exception when command id is 0 and delete fail
processService.handleCommand(logger, host, command1, processDefinitionCacheMaps); processService.handleCommand(logger, host, command1);
} }
@Test @Test
public void testGetUserById() { public void testGetUserById() {
User user = new User(); User user = new User();
user.setId(123); user.setId(123);
Mockito.when(userCacheProcessor.selectById(123)).thenReturn(user); Mockito.when(userMapper.selectById(123)).thenReturn(user);
Assert.assertEquals(user, processService.getUserById(123)); Assert.assertEquals(user, processService.getUserById(123));
} }

Loading…
Cancel
Save