Wenjun Ruan
6 months ago
committed by
GitHub
14 changed files with 281 additions and 276 deletions
@ -0,0 +1,119 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.plugin.task.api.utils; |
||||
|
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils; |
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; |
||||
import org.apache.dolphinscheduler.plugin.task.api.model.Property; |
||||
|
||||
import org.apache.commons.collections4.CollectionUtils; |
||||
|
||||
import java.util.ArrayList; |
||||
import java.util.HashMap; |
||||
import java.util.List; |
||||
import java.util.Map; |
||||
import java.util.stream.Collectors; |
||||
|
||||
import lombok.experimental.UtilityClass; |
||||
import lombok.extern.slf4j.Slf4j; |
||||
|
||||
@Slf4j |
||||
@UtilityClass |
||||
public class VarPoolUtils { |
||||
|
||||
public List<Property> deserializeVarPool(String varPoolJson) { |
||||
return JSONUtils.toList(varPoolJson, Property.class); |
||||
} |
||||
|
||||
/** |
||||
* @see #mergeVarPool(List) |
||||
*/ |
||||
public String mergeVarPoolJsonString(List<String> varPoolJsons) { |
||||
if (CollectionUtils.isEmpty(varPoolJsons)) { |
||||
return null; |
||||
} |
||||
List<List<Property>> varPools = varPoolJsons.stream() |
||||
.map(VarPoolUtils::deserializeVarPool) |
||||
.collect(Collectors.toList()); |
||||
List<Property> finalVarPool = mergeVarPool(varPools); |
||||
return JSONUtils.toJsonString(finalVarPool); |
||||
} |
||||
|
||||
/** |
||||
* Merge the given two varpools, and return the merged varpool. |
||||
* If the two varpools have the same property({@link Property#getProp()} and {@link Property#getDirect()} is same), the value of the property in varpool2 will be used. |
||||
* // todo: we may need to consider the datatype of the property
|
||||
*/ |
||||
public List<Property> mergeVarPool(List<List<Property>> varPools) { |
||||
if (CollectionUtils.isEmpty(varPools)) { |
||||
return null; |
||||
} |
||||
if (varPools.size() == 1) { |
||||
return varPools.get(0); |
||||
} |
||||
Map<String, Property> result = new HashMap<>(); |
||||
for (List<Property> varPool : varPools) { |
||||
if (CollectionUtils.isEmpty(varPool)) { |
||||
continue; |
||||
} |
||||
for (Property property : varPool) { |
||||
if (!Direct.OUT.equals(property.getDirect())) { |
||||
log.info("The direct should be OUT in varPool, but got {}", property.getDirect()); |
||||
continue; |
||||
} |
||||
result.put(property.getProp(), property); |
||||
} |
||||
} |
||||
return new ArrayList<>(result.values()); |
||||
} |
||||
|
||||
public String subtractVarPoolJson(String varPool, List<String> subtractVarPool) { |
||||
List<Property> varPoolList = deserializeVarPool(varPool); |
||||
List<List<Property>> subtractVarPoolList = subtractVarPool.stream() |
||||
.map(VarPoolUtils::deserializeVarPool) |
||||
.collect(Collectors.toList()); |
||||
List<Property> finalVarPool = subtractVarPool(varPoolList, subtractVarPoolList); |
||||
return JSONUtils.toJsonString(finalVarPool); |
||||
} |
||||
|
||||
/** |
||||
* Return the subtracted varpool, which key is in varPool but not in subtractVarPool. |
||||
*/ |
||||
public List<Property> subtractVarPool(List<Property> varPool, List<List<Property>> subtractVarPool) { |
||||
if (CollectionUtils.isEmpty(varPool)) { |
||||
return null; |
||||
} |
||||
if (CollectionUtils.isEmpty(subtractVarPool)) { |
||||
return varPool; |
||||
} |
||||
Map<String, Property> subtractVarPoolMap = new HashMap<>(); |
||||
for (List<Property> properties : subtractVarPool) { |
||||
for (Property property : properties) { |
||||
subtractVarPoolMap.put(property.getProp(), property); |
||||
} |
||||
} |
||||
List<Property> result = new ArrayList<>(); |
||||
for (Property property : varPool) { |
||||
if (!subtractVarPoolMap.containsKey(property.getProp())) { |
||||
result.add(property); |
||||
} |
||||
} |
||||
return result; |
||||
} |
||||
|
||||
} |
||||
@ -0,0 +1,62 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.plugin.task.api.utils; |
||||
|
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; |
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; |
||||
import org.apache.dolphinscheduler.plugin.task.api.model.Property; |
||||
|
||||
import java.util.List; |
||||
|
||||
import org.junit.jupiter.api.Test; |
||||
|
||||
import com.google.common.collect.Lists; |
||||
import com.google.common.truth.Truth; |
||||
|
||||
class VarPoolUtilsTest { |
||||
|
||||
@Test |
||||
void mergeVarPool() { |
||||
Truth.assertThat(VarPoolUtils.mergeVarPool(null)).isNull(); |
||||
|
||||
// Override the value of the same property
|
||||
// Merge the property with different key.
|
||||
List<Property> varpool1 = Lists.newArrayList(new Property("name", Direct.OUT, DataType.VARCHAR, "tom")); |
||||
List<Property> varpool2 = Lists.newArrayList( |
||||
new Property("name", Direct.OUT, DataType.VARCHAR, "tim"), |
||||
new Property("age", Direct.OUT, DataType.INTEGER, "10")); |
||||
|
||||
Truth.assertThat(VarPoolUtils.mergeVarPool(Lists.newArrayList(varpool1, varpool2))) |
||||
.containsExactly( |
||||
new Property("name", Direct.OUT, DataType.VARCHAR, "tim"), |
||||
new Property("age", Direct.OUT, DataType.INTEGER, "10")); |
||||
|
||||
} |
||||
|
||||
@Test |
||||
void subtractVarPool() { |
||||
Truth.assertThat(VarPoolUtils.subtractVarPool(null, null)).isNull(); |
||||
List<Property> varpool1 = Lists.newArrayList(new Property("name", Direct.OUT, DataType.VARCHAR, "tom"), |
||||
new Property("age", Direct.OUT, DataType.INTEGER, "10")); |
||||
List<Property> varpool2 = Lists.newArrayList(new Property("name", Direct.OUT, DataType.VARCHAR, "tom")); |
||||
List<Property> varpool3 = Lists.newArrayList(new Property("location", Direct.OUT, DataType.VARCHAR, "china")); |
||||
|
||||
Truth.assertThat(VarPoolUtils.subtractVarPool(varpool1, Lists.newArrayList(varpool2, varpool3))) |
||||
.containsExactly(new Property("age", Direct.OUT, DataType.INTEGER, "10")); |
||||
} |
||||
} |
||||
Loading…
Reference in new issue