Browse Source

[Feature#3234][cluster]enhanced load balancing (#3235)

* [Future#3234][cluster]enhanced load balancing
weight-based load balancing algorithm
this close # 3234

* remove useless parameter

* code smell

* load balancing according to work group

* add smooth weight round robin

* remove unused constants

* perfect unit test

* code smell

* code smell

* add work weight config

* fix config error

* add weight docs to readme.md
pull/3/MERGE
CalvinKirs 4 years ago committed by GitHub
parent
commit
d6a32ac652
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 4
      docker/build/README.md
  2. 4
      docker/build/README_zh_CN.md
  3. 5
      docker/build/conf/dolphinscheduler/worker.properties.tpl
  4. 1
      docker/build/startup-init-conf.sh
  5. 1
      docker/docker-swarm/docker-compose.yml
  6. 1
      docker/docker-swarm/docker-stack.yml
  7. 1
      docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml
  8. 5
      docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml
  9. 1
      docker/kubernetes/dolphinscheduler/values.yaml
  10. 61
      dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Host.java
  11. 7
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/CommonHostManager.java
  12. 2
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/RandomHostManager.java
  13. 2
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManager.java
  14. 41
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RandomSelector.java
  15. 120
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelector.java
  16. 12
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java
  17. 14
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistry.java
  18. 3
      dolphinscheduler-server/src/main/resources/worker.properties
  19. 16
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RandomSelectorTest.java
  20. 51
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelectorTest.java

4
docker/build/README.md

@ -238,6 +238,10 @@ This environment variable sets max cpu load avg for `worker-server`. The default
This environment variable sets reserved memory for `worker-server`. The default value is `0.1`. This environment variable sets reserved memory for `worker-server`. The default value is `0.1`.
**`WORKER_WEIGHT`**
This environment variable sets port for `worker-server`. The default value is `100`.
**`WORKER_LISTEN_PORT`** **`WORKER_LISTEN_PORT`**
This environment variable sets port for `worker-server`. The default value is `1234`. This environment variable sets port for `worker-server`. The default value is `1234`.

4
docker/build/README_zh_CN.md

@ -238,6 +238,10 @@ Dolphin Scheduler映像使用了几个容易遗漏的环境变量。虽然这些
配置`worker-server`的保留内存,默认值 `0.1` 配置`worker-server`的保留内存,默认值 `0.1`
**`WORKER_WEIGHT`**
配置`worker-server`的权重,默认之`100`。
**`WORKER_LISTEN_PORT`** **`WORKER_LISTEN_PORT`**
配置`worker-server`的端口,默认值 `1234` 配置`worker-server`的端口,默认值 `1234`

5
docker/build/conf/dolphinscheduler/worker.properties.tpl

@ -34,4 +34,7 @@ worker.reserved.memory=${WORKER_RESERVED_MEMORY}
#worker.listen.port=${WORKER_LISTEN_PORT} #worker.listen.port=${WORKER_LISTEN_PORT}
# default worker group # default worker group
#worker.group=${WORKER_GROUP} #worker.groups=${WORKER_GROUP}
# default worker weight
#worker.weight=${WORKER_WEIGHT}

1
docker/build/startup-init-conf.sh

@ -74,6 +74,7 @@ export WORKER_MAX_CPULOAD_AVG=${WORKER_MAX_CPULOAD_AVG:-"100"}
export WORKER_RESERVED_MEMORY=${WORKER_RESERVED_MEMORY:-"0.1"} export WORKER_RESERVED_MEMORY=${WORKER_RESERVED_MEMORY:-"0.1"}
export WORKER_LISTEN_PORT=${WORKER_LISTEN_PORT:-"1234"} export WORKER_LISTEN_PORT=${WORKER_LISTEN_PORT:-"1234"}
export WORKER_GROUP=${WORKER_GROUP:-"default"} export WORKER_GROUP=${WORKER_GROUP:-"default"}
export WORKER_WEIGHT=${WORKER_WEIGHT:-"100"}
#============================================================================ #============================================================================
# Alert Server # Alert Server

1
docker/docker-swarm/docker-compose.yml

@ -187,6 +187,7 @@ services:
WORKER_MAX_CPULOAD_AVG: "100" WORKER_MAX_CPULOAD_AVG: "100"
WORKER_RESERVED_MEMORY: "0.1" WORKER_RESERVED_MEMORY: "0.1"
WORKER_GROUP: "default" WORKER_GROUP: "default"
WORKER_WEIGHT: "100"
DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler" DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler"
DATABASE_HOST: dolphinscheduler-postgresql DATABASE_HOST: dolphinscheduler-postgresql
DATABASE_PORT: 5432 DATABASE_PORT: 5432

1
docker/docker-swarm/docker-stack.yml

@ -187,6 +187,7 @@ services:
WORKER_MAX_CPULOAD_AVG: "100" WORKER_MAX_CPULOAD_AVG: "100"
WORKER_RESERVED_MEMORY: "0.1" WORKER_RESERVED_MEMORY: "0.1"
WORKER_GROUP: "default" WORKER_GROUP: "default"
WORKER_WEIGHT: "100"
DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler" DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler"
DATABASE_HOST: dolphinscheduler-postgresql DATABASE_HOST: dolphinscheduler-postgresql
DATABASE_PORT: 5432 DATABASE_PORT: 5432

1
docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml

@ -31,6 +31,7 @@ data:
WORKER_RESERVED_MEMORY: {{ .Values.worker.configmap.WORKER_RESERVED_MEMORY | quote }} WORKER_RESERVED_MEMORY: {{ .Values.worker.configmap.WORKER_RESERVED_MEMORY | quote }}
WORKER_LISTEN_PORT: {{ .Values.worker.configmap.WORKER_LISTEN_PORT | quote }} WORKER_LISTEN_PORT: {{ .Values.worker.configmap.WORKER_LISTEN_PORT | quote }}
WORKER_GROUP: {{ .Values.worker.configmap.WORKER_GROUP | quote }} WORKER_GROUP: {{ .Values.worker.configmap.WORKER_GROUP | quote }}
WORKER_WEIGHT: {{ .Values.worker.configmap.WORKER_WEIGHT | quote }}
DOLPHINSCHEDULER_DATA_BASEDIR_PATH: {{ include "dolphinscheduler.worker.base.dir" . | quote }} DOLPHINSCHEDULER_DATA_BASEDIR_PATH: {{ include "dolphinscheduler.worker.base.dir" . | quote }}
dolphinscheduler_env.sh: |- dolphinscheduler_env.sh: |-
{{- range .Values.worker.configmap.DOLPHINSCHEDULER_ENV }} {{- range .Values.worker.configmap.DOLPHINSCHEDULER_ENV }}

5
docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml

@ -162,6 +162,11 @@ spec:
configMapKeyRef: configMapKeyRef:
name: {{ include "dolphinscheduler.fullname" . }}-worker name: {{ include "dolphinscheduler.fullname" . }}-worker
key: WORKER_GROUP key: WORKER_GROUP
- name: WORKER_WEUGHT
valueFrom:
configMapKeyRef:
name: {{ include "dolphinscheduler.fullname" . }}-worker
key: WORKER_WEIGHT
- name: DOLPHINSCHEDULER_DATA_BASEDIR_PATH - name: DOLPHINSCHEDULER_DATA_BASEDIR_PATH
valueFrom: valueFrom:
configMapKeyRef: configMapKeyRef:

1
docker/kubernetes/dolphinscheduler/values.yaml

@ -201,6 +201,7 @@ worker:
WORKER_RESERVED_MEMORY: "0.1" WORKER_RESERVED_MEMORY: "0.1"
WORKER_LISTEN_PORT: "1234" WORKER_LISTEN_PORT: "1234"
WORKER_GROUP: "default" WORKER_GROUP: "default"
WORKER_WEIGHT: "100"
DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler" DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler"
DOLPHINSCHEDULER_ENV: DOLPHINSCHEDULER_ENV:
- "export HADOOP_HOME=/opt/soft/hadoop" - "export HADOOP_HOME=/opt/soft/hadoop"

61
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Host.java

@ -39,6 +39,16 @@ public class Host implements Serializable {
*/ */
private int port; private int port;
/**
* weight
*/
private int weight;
/**
* workGroup
*/
private String workGroup;
public Host() { public Host() {
} }
@ -48,6 +58,21 @@ public class Host implements Serializable {
this.address = ip + ":" + port; this.address = ip + ":" + port;
} }
public Host(String ip, int port, int weight) {
this.ip = ip;
this.port = port;
this.address = ip + ":" + port;
this.weight = weight;
}
public Host(String ip, int port, int weight,String workGroup) {
this.ip = ip;
this.port = port;
this.address = ip + ":" + port;
this.weight = weight;
this.workGroup=workGroup;
}
public String getAddress() { public String getAddress() {
return address; return address;
} }
@ -65,6 +90,14 @@ public class Host implements Serializable {
this.address = ip + ":" + port; this.address = ip + ":" + port;
} }
public int getWeight() {
return weight;
}
public void setWeight(int weight) {
this.weight = weight;
}
public int getPort() { public int getPort() {
return port; return port;
} }
@ -74,31 +107,47 @@ public class Host implements Serializable {
this.address = ip + ":" + port; this.address = ip + ":" + port;
} }
public String getWorkGroup() {
return workGroup;
}
public void setWorkGroup(String workGroup) {
this.workGroup = workGroup;
}
/** /**
* address convert host * address convert host
*
* @param address address * @param address address
* @return host * @return host
*/ */
public static Host of(String address){ public static Host of(String address) {
if(address == null) { if (address == null) {
throw new IllegalArgumentException("Host : address is null."); throw new IllegalArgumentException("Host : address is null.");
} }
String[] parts = address.split(":"); String[] parts = address.split(":");
if (parts.length != 2) { if (parts.length < 2) {
throw new IllegalArgumentException(String.format("Host : %s illegal.", address)); throw new IllegalArgumentException(String.format("Host : %s illegal.", address));
} }
Host host = new Host(parts[0], Integer.parseInt(parts[1])); Host host = null;
if (parts.length == 2) {
host = new Host(parts[0], Integer.parseInt(parts[1]));
}
if (parts.length == 3) {
host = new Host(parts[0], Integer.parseInt(parts[1]), Integer.parseInt(parts[2]));
}
return host; return host;
} }
/** /**
* whether old version * whether old version
*
* @param address address * @param address address
* @return old version is true , otherwise is false * @return old version is true , otherwise is false
*/ */
public static Boolean isOldVersion(String address){ public static Boolean isOldVersion(String address) {
String[] parts = address.split(":"); String[] parts = address.split(":");
return parts.length != 2 ? true : false; return parts.length != 2 && parts.length != 3;
} }
@Override @Override

7
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/CommonHostManager.java

@ -71,7 +71,12 @@ public abstract class CommonHostManager implements HostManager {
return host; return host;
} }
List<Host> candidateHosts = new ArrayList<>(nodes.size()); List<Host> candidateHosts = new ArrayList<>(nodes.size());
nodes.stream().forEach(node -> candidateHosts.add(Host.of(node))); nodes.forEach(node -> {
Host nodeHost=Host.of(node);
nodeHost.setWorkGroup(context.getWorkerGroup());
candidateHosts.add(nodeHost);
});
return select(candidateHosts); return select(candidateHosts);
} }

2
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/RandomHostManager.java

@ -38,7 +38,7 @@ public class RandomHostManager extends CommonHostManager {
* set round robin * set round robin
*/ */
public RandomHostManager(){ public RandomHostManager(){
this.selector = new RandomSelector<>(); this.selector = new RandomSelector();
} }
@Override @Override

2
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManager.java

@ -38,7 +38,7 @@ public class RoundRobinHostManager extends CommonHostManager {
* set round robin * set round robin
*/ */
public RoundRobinHostManager(){ public RoundRobinHostManager(){
this.selector = new RoundRobinSelector<>(); this.selector = new RoundRobinSelector();
} }
@Override @Override

41
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RandomSelector.java

@ -17,27 +17,44 @@
package org.apache.dolphinscheduler.server.master.dispatch.host.assign; package org.apache.dolphinscheduler.server.master.dispatch.host.assign;
import org.apache.dolphinscheduler.remote.utils.Host;
import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.Random; import java.util.List;
import java.util.concurrent.ThreadLocalRandom;
/** /**
* random selector * random selector
* @param <T> T
*/ */
public class RandomSelector<T> extends AbstractSelector<T> { public class RandomSelector extends AbstractSelector<Host> {
private final Random random = new Random();
@Override @Override
public T doSelect(final Collection<T> source) { public Host doSelect(final Collection<Host> source) {
List<Host> hosts = new ArrayList<>(source);
int size = hosts.size();
int[] weights = new int[size];
int totalWeight = 0;
int index = 0;
for (Host host : hosts) {
totalWeight += host.getWeight();
weights[index] = host.getWeight();
index++;
}
int size = source.size(); if (totalWeight > 0) {
/** int offset = ThreadLocalRandom.current().nextInt(totalWeight);
* random select
*/
int randomIndex = random.nextInt(size);
return (T) source.toArray()[randomIndex]; for (int i = 0; i < size; i++) {
offset -= weights[i];
if (offset < 0) {
return hosts.get(i);
}
}
}
return hosts.get(ThreadLocalRandom.current().nextInt(size));
} }
} }

120
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelector.java

@ -16,27 +16,123 @@
*/ */
package org.apache.dolphinscheduler.server.master.dispatch.host.assign; package org.apache.dolphinscheduler.server.master.dispatch.host.assign;
import org.apache.dolphinscheduler.remote.utils.Host;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import java.util.Collection; import java.util.*;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
/** /**
* round robin selector * Smooth Weight Round Robin
* @param <T> T
*/ */
@Service @Service
public class RoundRobinSelector<T> extends AbstractSelector<T> { public class RoundRobinSelector extends AbstractSelector<Host> {
private ConcurrentMap<String, ConcurrentMap<String, WeightedRoundRobin>> workGroupWeightMap = new ConcurrentHashMap<>();
private static final int RECYCLE_PERIOD = 100000;
private AtomicBoolean updateLock = new AtomicBoolean();
protected static class WeightedRoundRobin {
private int weight;
private AtomicLong current = new AtomicLong(0);
private long lastUpdate;
int getWeight() {
return weight;
}
void setWeight(int weight) {
this.weight = weight;
current.set(0);
}
long increaseCurrent() {
return current.addAndGet(weight);
}
void sel(int total) {
current.addAndGet(-1L * total);
}
long getLastUpdate() {
return lastUpdate;
}
void setLastUpdate(long lastUpdate) {
this.lastUpdate = lastUpdate;
}
}
private final AtomicInteger index = new AtomicInteger(0);
@Override @Override
public T doSelect(Collection<T> source) { public Host doSelect(Collection<Host> source) {
int size = source.size(); List<Host> hosts = new ArrayList<>(source);
/** String key = hosts.get(0).getWorkGroup();
* round robin ConcurrentMap<String, WeightedRoundRobin> map = workGroupWeightMap.get(key);
*/ if (map == null) {
return (T) source.toArray()[index.getAndIncrement() % size]; workGroupWeightMap.putIfAbsent(key, new ConcurrentHashMap<>());
map = workGroupWeightMap.get(key);
}
int totalWeight = 0;
long maxCurrent = Long.MIN_VALUE;
long now = System.currentTimeMillis();
Host selectedHost = null;
WeightedRoundRobin selectWeightRoundRobin = null;
for (Host host : hosts) {
String workGroupHost = host.getWorkGroup() + host.getAddress();
WeightedRoundRobin weightedRoundRobin = map.get(workGroupHost);
int weight = host.getWeight();
if (weight < 0) {
weight = 0;
}
if (weightedRoundRobin == null) {
weightedRoundRobin = new WeightedRoundRobin();
// set weight
weightedRoundRobin.setWeight(weight);
map.putIfAbsent(workGroupHost, weightedRoundRobin);
weightedRoundRobin = map.get(workGroupHost);
}
if (weight != weightedRoundRobin.getWeight()) {
weightedRoundRobin.setWeight(weight);
}
long cur = weightedRoundRobin.increaseCurrent();
weightedRoundRobin.setLastUpdate(now);
if (cur > maxCurrent) {
maxCurrent = cur;
selectedHost = host;
selectWeightRoundRobin = weightedRoundRobin;
}
totalWeight += weight;
}
if (!updateLock.get() && hosts.size() != map.size() && updateLock.compareAndSet(false, true)) {
try {
ConcurrentMap<String, WeightedRoundRobin> newMap = new ConcurrentHashMap<>(map);
newMap.entrySet().removeIf(item -> now - item.getValue().getLastUpdate() > RECYCLE_PERIOD);
workGroupWeightMap.put(key, newMap);
} finally {
updateLock.set(false);
}
}
if (selectedHost != null) {
selectWeightRoundRobin.sel(totalWeight);
return selectedHost;
}
return hosts.get(0);
} }
} }

12
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java

@ -49,6 +49,9 @@ public class WorkerConfig {
@Value("${worker.listen.port: 1234}") @Value("${worker.listen.port: 1234}")
private int listenPort; private int listenPort;
@Value("${worker.weight:100}")
private int weight;
public int getListenPort() { public int getListenPort() {
return listenPort; return listenPort;
} }
@ -107,4 +110,13 @@ public class WorkerConfig {
public void setWorkerMaxCpuloadAvg(int workerMaxCpuloadAvg) { public void setWorkerMaxCpuloadAvg(int workerMaxCpuloadAvg) {
this.workerMaxCpuloadAvg = workerMaxCpuloadAvg; this.workerMaxCpuloadAvg = workerMaxCpuloadAvg;
} }
public int getWeight() {
return weight;
}
public void setWeight(int weight) {
this.weight = weight;
}
} }

14
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistry.java

@ -16,9 +16,6 @@
*/ */
package org.apache.dolphinscheduler.server.worker.registry; package org.apache.dolphinscheduler.server.worker.registry;
import static org.apache.dolphinscheduler.common.Constants.DEFAULT_WORKER_GROUP;
import static org.apache.dolphinscheduler.common.Constants.SLASH;
import java.util.Date; import java.util.Date;
import java.util.Set; import java.util.Set;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
@ -44,6 +41,8 @@ import org.springframework.stereotype.Service;
import com.google.common.collect.Sets; import com.google.common.collect.Sets;
import static org.apache.dolphinscheduler.common.Constants.*;
/** /**
* worker registry * worker registry
@ -142,6 +141,7 @@ public class WorkerRegistry {
String address = getLocalAddress(); String address = getLocalAddress();
String workerZkPathPrefix = this.zookeeperRegistryCenter.getWorkerPath(); String workerZkPathPrefix = this.zookeeperRegistryCenter.getWorkerPath();
String weight = getWorkerWeight();
for (String workGroup : this.workerGroups) { for (String workGroup : this.workerGroups) {
StringBuilder workerZkPathBuilder = new StringBuilder(100); StringBuilder workerZkPathBuilder = new StringBuilder(100);
@ -152,6 +152,7 @@ public class WorkerRegistry {
// trim and lower case is need // trim and lower case is need
workerZkPathBuilder.append(workGroup.trim().toLowerCase()).append(SLASH); workerZkPathBuilder.append(workGroup.trim().toLowerCase()).append(SLASH);
workerZkPathBuilder.append(address); workerZkPathBuilder.append(address);
workerZkPathBuilder.append(weight).append(SLASH);
workerZkPaths.add(workerZkPathBuilder.toString()); workerZkPaths.add(workerZkPathBuilder.toString());
} }
return workerZkPaths; return workerZkPaths;
@ -163,4 +164,11 @@ public class WorkerRegistry {
private String getLocalAddress() { private String getLocalAddress() {
return NetUtils.getHost() + ":" + workerConfig.getListenPort(); return NetUtils.getHost() + ":" + workerConfig.getListenPort();
} }
/**
* get Worker Weight
*/
private String getWorkerWeight() {
return ":" + workerConfig.getWeight();
}
} }

3
dolphinscheduler-server/src/main/resources/worker.properties

@ -32,3 +32,6 @@
# default worker group # default worker group
#worker.groups=default #worker.groups=default
# default worker weight
#work.weight=100

16
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RandomSelectorTest.java

@ -16,7 +16,9 @@
*/ */
package org.apache.dolphinscheduler.server.master.dispatch.host.assign; package org.apache.dolphinscheduler.server.master.dispatch.host.assign;
import org.apache.commons.lang.ObjectUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.remote.utils.Host;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
@ -36,16 +38,16 @@ public class RandomSelectorTest {
@Test @Test
public void testSelect1(){ public void testSelect1(){
RandomSelector<String> selector = new RandomSelector(); RandomSelector selector = new RandomSelector();
String result = selector.select(Arrays.asList("1")); Host result = selector.select(Arrays.asList(new Host("192.168.1.1",80,100),new Host("192.168.1.2",80,20)));
Assert.assertTrue(StringUtils.isNotEmpty(result)); Assert.assertNotNull(result);
Assert.assertTrue(result.equalsIgnoreCase("1"));
} }
@Test @Test
public void testSelect(){ public void testSelect(){
RandomSelector<Integer> selector = new RandomSelector(); RandomSelector selector = new RandomSelector();
int result = selector.select(Arrays.asList(1,2,3,4,5,6,7)); Host result = selector.select(Arrays.asList(new Host("192.168.1.1",80,100),new Host("192.168.1.1",80,20)));
Assert.assertTrue(result >= 1 && result <= 7); Assert.assertNotNull(result);
} }
} }

51
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelectorTest.java

@ -17,6 +17,7 @@
package org.apache.dolphinscheduler.server.master.dispatch.host.assign; package org.apache.dolphinscheduler.server.master.dispatch.host.assign;
import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.remote.utils.Host;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
@ -30,26 +31,46 @@ import java.util.List;
public class RoundRobinSelectorTest { public class RoundRobinSelectorTest {
@Test(expected = IllegalArgumentException.class) @Test(expected = IllegalArgumentException.class)
public void testSelectWithIllegalArgumentException(){ public void testSelectWithIllegalArgumentException() {
RoundRobinSelector selector = new RoundRobinSelector(); RoundRobinSelector selector = new RoundRobinSelector();
selector.select(Collections.EMPTY_LIST); selector.select(Collections.EMPTY_LIST);
} }
@Test @Test
public void testSelect1(){ public void testSelect1() {
RoundRobinSelector<String> selector = new RoundRobinSelector(); RoundRobinSelector selector = new RoundRobinSelector();
String result = selector.select(Arrays.asList("1")); Host result = null;
Assert.assertTrue(StringUtils.isNotEmpty(result)); result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, "kris"), new Host("192.168.1.2", 80, 10, "kris")));
Assert.assertTrue(result.equalsIgnoreCase("1")); Assert.assertEquals("192.168.1.1", result.getIp());
} result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, "kris"), new Host("192.168.1.2", 80, 10, "kris")));
Assert.assertEquals("192.168.1.2", result.getIp());
result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, "kris"), new Host("192.168.1.2", 80, 10, "kris")));
Assert.assertEquals("192.168.1.1", result.getIp());
// add new host
result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, "kris"), new Host("192.168.1.2", 80, 10, "kris")));
Assert.assertEquals("192.168.1.1", result.getIp());
result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, "kris"), new Host("192.168.1.2", 80, 10, "kris")));
Assert.assertEquals("192.168.1.2", result.getIp());
result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, "kris"), new Host("192.168.1.2", 80, 10, "kris"), new Host("192.168.1.3", 80, 10, "kris")));
Assert.assertEquals("192.168.1.1",result.getIp());
result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, "kris"), new Host("192.168.1.2", 80, 10, "kris"), new Host("192.168.1.3", 80, 10, "kris")));
Assert.assertEquals("192.168.1.3",result.getIp());
result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, "kris"), new Host("192.168.1.2", 80, 10, "kris"), new Host("192.168.1.3", 80, 10, "kris")));
Assert.assertEquals("192.168.1.1",result.getIp());
result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, "kris"), new Host("192.168.1.2", 80, 10, "kris"), new Host("192.168.1.3", 80, 10, "kris")));
Assert.assertEquals("192.168.1.2",result.getIp());
result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, "kris"), new Host("192.168.1.2", 80, 10, "kris"), new Host("192.168.1.3", 80, 10, "kris")));
Assert.assertEquals("192.168.1.1",result.getIp());
result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, "kris"), new Host("192.168.1.2", 80, 10, "kris"), new Host("192.168.1.3", 80, 10, "kris")));
Assert.assertEquals("192.168.1.3",result.getIp());
// remove host3
result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, "kris"), new Host("192.168.1.2", 80, 10, "kris")));
Assert.assertEquals("192.168.1.1",result.getIp());
result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, "kris"), new Host("192.168.1.2", 80, 10, "kris")));
Assert.assertEquals("192.168.1.2",result.getIp());
result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, "kris"), new Host("192.168.1.2", 80, 10, "kris")));
Assert.assertEquals("192.168.1.1",result.getIp());
@Test
public void testSelect(){
RoundRobinSelector<Integer> selector = new RoundRobinSelector();
List<Integer> sources = Arrays.asList(1, 2, 3, 4, 5, 6, 7);
int result = selector.select(sources);
Assert.assertTrue(result == 1);
int result2 = selector.select(Arrays.asList(1,2,3,4,5,6,7));
Assert.assertTrue(result2 == 2);
} }
} }

Loading…
Cancel
Save