Browse Source

[Improvement][API] ignore noNodeException when get worker groups (#4120)

* ignore noNodeException when get worker groups

* add ut
pull/3/MERGE
felix.wang 4 years ago committed by GitHub
parent
commit
b46044d592
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 76
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkerGroupService.java
  2. 42
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkerGroupServiceTest.java

76
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkerGroupService.java

@ -14,8 +14,11 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.api.service; package org.apache.dolphinscheduler.api.service;
import static org.apache.dolphinscheduler.common.Constants.DEFAULT_WORKER_GROUP;
import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
@ -26,11 +29,17 @@ import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.entity.WorkerGroup; import org.apache.dolphinscheduler.dao.entity.WorkerGroup;
import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
import org.apache.dolphinscheduler.service.zk.ZookeeperCachedOperator; import org.apache.dolphinscheduler.service.zk.ZookeeperCachedOperator;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import java.util.*;
import java.util.stream.Collectors;
/** /**
* work group service * work group service
@ -38,14 +47,11 @@ import java.util.stream.Collectors;
@Service @Service
public class WorkerGroupService extends BaseService { public class WorkerGroupService extends BaseService {
private static final String NO_NODE_EXCEPTION_REGEX = "KeeperException$NoNodeException";
@Autowired
ProcessInstanceMapper processInstanceMapper;
@Autowired @Autowired
protected ZookeeperCachedOperator zookeeperCachedOperator; protected ZookeeperCachedOperator zookeeperCachedOperator;
@Autowired
ProcessInstanceMapper processInstanceMapper;
/** /**
* query worker group paging * query worker group paging
@ -56,7 +62,7 @@ public class WorkerGroupService extends BaseService {
* @param pageSize page size * @param pageSize page size
* @return worker group list page * @return worker group list page
*/ */
public Map<String,Object> queryAllGroupPaging(User loginUser, Integer pageNo, Integer pageSize, String searchVal) { public Map<String, Object> queryAllGroupPaging(User loginUser, Integer pageNo, Integer pageSize, String searchVal) {
// list from index // list from index
Integer fromIndex = (pageNo - 1) * pageSize; Integer fromIndex = (pageNo - 1) * pageSize;
@ -72,20 +78,20 @@ public class WorkerGroupService extends BaseService {
List<WorkerGroup> resultDataList = new ArrayList<>(); List<WorkerGroup> resultDataList = new ArrayList<>();
if (CollectionUtils.isNotEmpty(workerGroups)){ if (CollectionUtils.isNotEmpty(workerGroups)) {
List<WorkerGroup> searchValDataList = new ArrayList<>(); List<WorkerGroup> searchValDataList = new ArrayList<>();
if (StringUtils.isNotEmpty(searchVal)){ if (StringUtils.isNotEmpty(searchVal)) {
for (WorkerGroup workerGroup : workerGroups){ for (WorkerGroup workerGroup : workerGroups) {
if (workerGroup.getName().contains(searchVal)){ if (workerGroup.getName().contains(searchVal)) {
searchValDataList.add(workerGroup); searchValDataList.add(workerGroup);
} }
} }
}else { } else {
searchValDataList = workerGroups; searchValDataList = workerGroups;
} }
if (searchValDataList.size() < pageSize){ if (searchValDataList.size() < pageSize) {
toIndex = (pageNo - 1) * pageSize + searchValDataList.size(); toIndex = (pageNo - 1) * pageSize + searchValDataList.size();
} }
resultDataList = searchValDataList.subList(fromIndex, toIndex); resultDataList = searchValDataList.subList(fromIndex, toIndex);
@ -100,14 +106,12 @@ public class WorkerGroupService extends BaseService {
return result; return result;
} }
/** /**
* query all worker group * query all worker group
* *
* @return all worker group list * @return all worker group list
*/ */
public Map<String,Object> queryAllGroup() { public Map<String, Object> queryAllGroup() {
Map<String, Object> result = new HashMap<>(); Map<String, Object> result = new HashMap<>();
List<WorkerGroup> workerGroups = getWorkerGroups(false); List<WorkerGroup> workerGroups = getWorkerGroups(false);
@ -120,30 +124,46 @@ public class WorkerGroupService extends BaseService {
return result; return result;
} }
/** /**
* get worker groups * get worker groups
* *
* @param isPaging whether paging * @param isPaging whether paging
* @return WorkerGroup list * @return WorkerGroup list
*/ */
private List<WorkerGroup> getWorkerGroups(boolean isPaging) { private List<WorkerGroup> getWorkerGroups(boolean isPaging) {
String workerPath = zookeeperCachedOperator.getZookeeperConfig().getDsRoot()+"/nodes" +"/worker";
List<String> workerGroupList = zookeeperCachedOperator.getChildrenKeys(workerPath); String workerPath = zookeeperCachedOperator.getZookeeperConfig().getDsRoot() + Constants.ZOOKEEPER_DOLPHINSCHEDULER_WORKERS;
List<WorkerGroup> workerGroups = new ArrayList<>();
List<String> workerGroupList;
try {
workerGroupList = zookeeperCachedOperator.getChildrenKeys(workerPath);
} catch (Exception e) {
if (e.getMessage().contains(NO_NODE_EXCEPTION_REGEX)) {
if (isPaging) {
return workerGroups;
} else {
//ignore noNodeException return Default
WorkerGroup wg = new WorkerGroup();
wg.setName(DEFAULT_WORKER_GROUP);
workerGroups.add(wg);
return workerGroups;
}
} else {
throw e;
}
}
// available workerGroup list // available workerGroup list
List<String> availableWorkerGroupList = new ArrayList<>(); List<String> availableWorkerGroupList = new ArrayList<>();
List<WorkerGroup> workerGroups = new ArrayList<>(); for (String workerGroup : workerGroupList) {
String workerGroupPath = workerPath + "/" + workerGroup;
for (String workerGroup : workerGroupList){
String workerGroupPath= workerPath + "/" + workerGroup;
List<String> childrenNodes = zookeeperCachedOperator.getChildrenKeys(workerGroupPath); List<String> childrenNodes = zookeeperCachedOperator.getChildrenKeys(workerGroupPath);
if (CollectionUtils.isNotEmpty(childrenNodes)){ if (CollectionUtils.isNotEmpty(childrenNodes)) {
availableWorkerGroupList.add(workerGroup); availableWorkerGroupList.add(workerGroup);
WorkerGroup wg = new WorkerGroup(); WorkerGroup wg = new WorkerGroup();
wg.setName(workerGroup); wg.setName(workerGroup);
if (isPaging){ if (isPaging) {
wg.setIpList(childrenNodes); wg.setIpList(childrenNodes);
String registeredIpValue = zookeeperCachedOperator.get(workerGroupPath + "/" + childrenNodes.get(0)); String registeredIpValue = zookeeperCachedOperator.get(workerGroupPath + "/" + childrenNodes.get(0));
wg.setCreateTime(DateUtils.stringToDate(registeredIpValue.split(",")[6])); wg.setCreateTime(DateUtils.stringToDate(registeredIpValue.split(",")[6]));

42
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkerGroupServiceTest.java

@ -14,20 +14,24 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.api.service; package org.apache.dolphinscheduler.api.service;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.entity.WorkerGroup; import org.apache.dolphinscheduler.dao.entity.WorkerGroup;
import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
import org.apache.dolphinscheduler.service.zk.ZookeeperCachedOperator; import org.apache.dolphinscheduler.service.zk.ZookeeperCachedOperator;
import org.apache.dolphinscheduler.service.zk.ZookeeperConfig; import org.apache.dolphinscheduler.service.zk.ZookeeperConfig;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
@ -35,16 +39,10 @@ import org.junit.runner.RunWith;
import org.mockito.InjectMocks; import org.mockito.InjectMocks;
import org.mockito.Mock; import org.mockito.Mock;
import org.mockito.Mockito; import org.mockito.Mockito;
import org.mockito.internal.matchers.Any;
import org.mockito.junit.MockitoJUnitRunner; import org.mockito.junit.MockitoJUnitRunner;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
@RunWith(MockitoJUnitRunner.class) @RunWith(MockitoJUnitRunner.class)
public class WorkerGroupServiceTest { public class WorkerGroupServiceTest {
@ -59,14 +57,13 @@ public class WorkerGroupServiceTest {
@Mock @Mock
private ZookeeperCachedOperator zookeeperCachedOperator; private ZookeeperCachedOperator zookeeperCachedOperator;
@Before @Before
public void init(){ public void init() {
ZookeeperConfig zookeeperConfig = new ZookeeperConfig(); ZookeeperConfig zookeeperConfig = new ZookeeperConfig();
zookeeperConfig.setDsRoot("/dolphinscheduler_qzw"); zookeeperConfig.setDsRoot("/dolphinscheduler_qzw");
Mockito.when(zookeeperCachedOperator.getZookeeperConfig()).thenReturn(zookeeperConfig); Mockito.when(zookeeperCachedOperator.getZookeeperConfig()).thenReturn(zookeeperConfig);
String workerPath = zookeeperCachedOperator.getZookeeperConfig().getDsRoot()+"/nodes" +"/worker"; String workerPath = zookeeperCachedOperator.getZookeeperConfig().getDsRoot() + Constants.ZOOKEEPER_DOLPHINSCHEDULER_WORKERS;
List<String> workerGroupStrList = new ArrayList<>(); List<String> workerGroupStrList = new ArrayList<>();
workerGroupStrList.add("default"); workerGroupStrList.add("default");
@ -83,19 +80,18 @@ public class WorkerGroupServiceTest {
} }
/** /**
* query worker group paging * query worker group paging
*/ */
@Test @Test
public void testQueryAllGroupPaging(){ public void testQueryAllGroupPaging() {
User user = new User(); User user = new User();
// general user add // general user add
user.setUserType(UserType.ADMIN_USER); user.setUserType(UserType.ADMIN_USER);
Map<String, Object> result = workerGroupService.queryAllGroupPaging(user, 1, 10, null); Map<String, Object> result = workerGroupService.queryAllGroupPaging(user, 1, 10, null);
PageInfo<WorkerGroup> pageInfo = (PageInfo) result.get(Constants.DATA_LIST); PageInfo<WorkerGroup> pageInfo = (PageInfo) result.get(Constants.DATA_LIST);
Assert.assertEquals(pageInfo.getLists().size(),1); Assert.assertEquals(pageInfo.getLists().size(), 1);
} }
@Test @Test
public void testQueryAllGroup() throws Exception { public void testQueryAllGroup() throws Exception {
Map<String, Object> result = workerGroupService.queryAllGroup(); Map<String, Object> result = workerGroupService.queryAllGroup();
@ -103,16 +99,24 @@ public class WorkerGroupServiceTest {
Assert.assertEquals(workerGroups.size(), 1); Assert.assertEquals(workerGroups.size(), 1);
} }
/** /**
* get processInstances * get processInstances
* @return
*/ */
private List<ProcessInstance> getProcessInstanceList(){ private List<ProcessInstance> getProcessInstanceList() {
List<ProcessInstance> processInstances = new ArrayList<>(); List<ProcessInstance> processInstances = new ArrayList<>();
processInstances.add(new ProcessInstance()); processInstances.add(new ProcessInstance());
return processInstances; return processInstances;
} }
@Test
public void testQueryAllGroupWithNoNodeException() {
String workerPath = zookeeperCachedOperator.getZookeeperConfig().getDsRoot() + Constants.ZOOKEEPER_DOLPHINSCHEDULER_WORKERS;
Mockito.when(zookeeperCachedOperator.getChildrenKeys(workerPath)).thenThrow(new RuntimeException("KeeperException$NoNodeException"));
Map<String, Object> result = workerGroupService.queryAllGroup();
Set<String> workerGroups = (Set<String>) result.get(Constants.DATA_LIST);
Assert.assertEquals(1, workerGroups.size());
Assert.assertEquals("default", workerGroups.toArray()[0]);
}
} }

Loading…
Cancel
Save