Browse Source
* refactor zk client & zk config module * refactor zk client & zk config module * add licensepull/2/head
DK.Pino
5 years ago
committed by
qiaozhanwei
17 changed files with 923 additions and 320 deletions
@ -0,0 +1,288 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package org.apache.dolphinscheduler.common.utils; |
||||||
|
|
||||||
|
import org.springframework.lang.Nullable; |
||||||
|
|
||||||
|
/** |
||||||
|
* A collection of static utility methods to validate input. |
||||||
|
* |
||||||
|
* <p>This class is modelled after Google Guava's Preconditions class, and partly takes code |
||||||
|
* from that class. We add this code to here base in order to reduce external |
||||||
|
* dependencies. |
||||||
|
*/ |
||||||
|
public final class Preconditions { |
||||||
|
|
||||||
|
// ------------------------------------------------------------------------
|
||||||
|
// Null checks
|
||||||
|
// ------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/** |
||||||
|
* Ensures that the given object reference is not null. |
||||||
|
* Upon violation, a {@code NullPointerException} with no message is thrown. |
||||||
|
* |
||||||
|
* @param reference The object reference |
||||||
|
* @return The object reference itself (generically typed). |
||||||
|
* |
||||||
|
* @throws NullPointerException Thrown, if the passed reference was null. |
||||||
|
*/ |
||||||
|
public static <T> T checkNotNull(T reference) { |
||||||
|
if (reference == null) { |
||||||
|
throw new NullPointerException(); |
||||||
|
} |
||||||
|
return reference; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Ensures that the given object reference is not null. |
||||||
|
* Upon violation, a {@code NullPointerException} with the given message is thrown. |
||||||
|
* |
||||||
|
* @param reference The object reference |
||||||
|
* @param errorMessage The message for the {@code NullPointerException} that is thrown if the check fails. |
||||||
|
* @return The object reference itself (generically typed). |
||||||
|
* |
||||||
|
* @throws NullPointerException Thrown, if the passed reference was null. |
||||||
|
*/ |
||||||
|
public static <T> T checkNotNull(T reference, @Nullable String errorMessage) { |
||||||
|
if (reference == null) { |
||||||
|
throw new NullPointerException(String.valueOf(errorMessage)); |
||||||
|
} |
||||||
|
return reference; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Ensures that the given object reference is not null. |
||||||
|
* Upon violation, a {@code NullPointerException} with the given message is thrown. |
||||||
|
* |
||||||
|
* <p>The error message is constructed from a template and an arguments array, after |
||||||
|
* a similar fashion as {@link String#format(String, Object...)}, but supporting only |
||||||
|
* {@code %s} as a placeholder. |
||||||
|
* |
||||||
|
* @param reference The object reference |
||||||
|
* @param errorMessageTemplate The message template for the {@code NullPointerException} |
||||||
|
* that is thrown if the check fails. The template substitutes its |
||||||
|
* {@code %s} placeholders with the error message arguments. |
||||||
|
* @param errorMessageArgs The arguments for the error message, to be inserted into the |
||||||
|
* message template for the {@code %s} placeholders. |
||||||
|
* |
||||||
|
* @return The object reference itself (generically typed). |
||||||
|
* |
||||||
|
* @throws NullPointerException Thrown, if the passed reference was null. |
||||||
|
*/ |
||||||
|
public static <T> T checkNotNull(T reference, |
||||||
|
@Nullable String errorMessageTemplate, |
||||||
|
@Nullable Object... errorMessageArgs) { |
||||||
|
|
||||||
|
if (reference == null) { |
||||||
|
throw new NullPointerException(format(errorMessageTemplate, errorMessageArgs)); |
||||||
|
} |
||||||
|
return reference; |
||||||
|
} |
||||||
|
|
||||||
|
// ------------------------------------------------------------------------
|
||||||
|
// Boolean Condition Checking (Argument)
|
||||||
|
// ------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/** |
||||||
|
* Checks the given boolean condition, and throws an {@code IllegalArgumentException} if |
||||||
|
* the condition is not met (evaluates to {@code false}). |
||||||
|
* |
||||||
|
* @param condition The condition to check |
||||||
|
* |
||||||
|
* @throws IllegalArgumentException Thrown, if the condition is violated. |
||||||
|
*/ |
||||||
|
public static void checkArgument(boolean condition) { |
||||||
|
if (!condition) { |
||||||
|
throw new IllegalArgumentException(); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Checks the given boolean condition, and throws an {@code IllegalArgumentException} if |
||||||
|
* the condition is not met (evaluates to {@code false}). The exception will have the |
||||||
|
* given error message. |
||||||
|
* |
||||||
|
* @param condition The condition to check |
||||||
|
* @param errorMessage The message for the {@code IllegalArgumentException} that is thrown if the check fails. |
||||||
|
* |
||||||
|
* @throws IllegalArgumentException Thrown, if the condition is violated. |
||||||
|
*/ |
||||||
|
public static void checkArgument(boolean condition, @Nullable Object errorMessage) { |
||||||
|
if (!condition) { |
||||||
|
throw new IllegalArgumentException(String.valueOf(errorMessage)); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Checks the given boolean condition, and throws an {@code IllegalArgumentException} if |
||||||
|
* the condition is not met (evaluates to {@code false}). |
||||||
|
* |
||||||
|
* @param condition The condition to check |
||||||
|
* @param errorMessageTemplate The message template for the {@code IllegalArgumentException} |
||||||
|
* that is thrown if the check fails. The template substitutes its |
||||||
|
* {@code %s} placeholders with the error message arguments. |
||||||
|
* @param errorMessageArgs The arguments for the error message, to be inserted into the |
||||||
|
* message template for the {@code %s} placeholders. |
||||||
|
* |
||||||
|
* @throws IllegalArgumentException Thrown, if the condition is violated. |
||||||
|
*/ |
||||||
|
public static void checkArgument(boolean condition, |
||||||
|
@Nullable String errorMessageTemplate, |
||||||
|
@Nullable Object... errorMessageArgs) { |
||||||
|
|
||||||
|
if (!condition) { |
||||||
|
throw new IllegalArgumentException(format(errorMessageTemplate, errorMessageArgs)); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// ------------------------------------------------------------------------
|
||||||
|
// Boolean Condition Checking (State)
|
||||||
|
// ------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/** |
||||||
|
* Checks the given boolean condition, and throws an {@code IllegalStateException} if |
||||||
|
* the condition is not met (evaluates to {@code false}). |
||||||
|
* |
||||||
|
* @param condition The condition to check |
||||||
|
* |
||||||
|
* @throws IllegalStateException Thrown, if the condition is violated. |
||||||
|
*/ |
||||||
|
public static void checkState(boolean condition) { |
||||||
|
if (!condition) { |
||||||
|
throw new IllegalStateException(); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Checks the given boolean condition, and throws an {@code IllegalStateException} if |
||||||
|
* the condition is not met (evaluates to {@code false}). The exception will have the |
||||||
|
* given error message. |
||||||
|
* |
||||||
|
* @param condition The condition to check |
||||||
|
* @param errorMessage The message for the {@code IllegalStateException} that is thrown if the check fails. |
||||||
|
* |
||||||
|
* @throws IllegalStateException Thrown, if the condition is violated. |
||||||
|
*/ |
||||||
|
public static void checkState(boolean condition, @Nullable Object errorMessage) { |
||||||
|
if (!condition) { |
||||||
|
throw new IllegalStateException(String.valueOf(errorMessage)); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Checks the given boolean condition, and throws an {@code IllegalStateException} if |
||||||
|
* the condition is not met (evaluates to {@code false}). |
||||||
|
* |
||||||
|
* @param condition The condition to check |
||||||
|
* @param errorMessageTemplate The message template for the {@code IllegalStateException} |
||||||
|
* that is thrown if the check fails. The template substitutes its |
||||||
|
* {@code %s} placeholders with the error message arguments. |
||||||
|
* @param errorMessageArgs The arguments for the error message, to be inserted into the |
||||||
|
* message template for the {@code %s} placeholders. |
||||||
|
* |
||||||
|
* @throws IllegalStateException Thrown, if the condition is violated. |
||||||
|
*/ |
||||||
|
public static void checkState(boolean condition, |
||||||
|
@Nullable String errorMessageTemplate, |
||||||
|
@Nullable Object... errorMessageArgs) { |
||||||
|
|
||||||
|
if (!condition) { |
||||||
|
throw new IllegalStateException(format(errorMessageTemplate, errorMessageArgs)); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Ensures that the given index is valid for an array, list or string of the given size. |
||||||
|
* |
||||||
|
* @param index index to check |
||||||
|
* @param size size of the array, list or string |
||||||
|
* |
||||||
|
* @throws IllegalArgumentException Thrown, if size is negative. |
||||||
|
* @throws IndexOutOfBoundsException Thrown, if the index negative or greater than or equal to size |
||||||
|
*/ |
||||||
|
public static void checkElementIndex(int index, int size) { |
||||||
|
checkArgument(size >= 0, "Size was negative."); |
||||||
|
if (index < 0 || index >= size) { |
||||||
|
throw new IndexOutOfBoundsException("Index: " + index + ", Size: " + size); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Ensures that the given index is valid for an array, list or string of the given size. |
||||||
|
* |
||||||
|
* @param index index to check |
||||||
|
* @param size size of the array, list or string |
||||||
|
* @param errorMessage The message for the {@code IndexOutOfBoundsException} that is thrown if the check fails. |
||||||
|
* |
||||||
|
* @throws IllegalArgumentException Thrown, if size is negative. |
||||||
|
* @throws IndexOutOfBoundsException Thrown, if the index negative or greater than or equal to size |
||||||
|
*/ |
||||||
|
public static void checkElementIndex(int index, int size, @Nullable String errorMessage) { |
||||||
|
checkArgument(size >= 0, "Size was negative."); |
||||||
|
if (index < 0 || index >= size) { |
||||||
|
throw new IndexOutOfBoundsException(String.valueOf(errorMessage) + " Index: " + index + ", Size: " + size); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// ------------------------------------------------------------------------
|
||||||
|
// Utilities
|
||||||
|
// ------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/** |
||||||
|
* A simplified formatting method. Similar to {@link String#format(String, Object...)}, but |
||||||
|
* with lower overhead (only String parameters, no locale, no format validation). |
||||||
|
* |
||||||
|
* <p>This method is taken quasi verbatim from the Guava Preconditions class. |
||||||
|
*/ |
||||||
|
private static String format(@Nullable String template, @Nullable Object... args) { |
||||||
|
final int numArgs = args == null ? 0 : args.length; |
||||||
|
template = String.valueOf(template); // null -> "null"
|
||||||
|
|
||||||
|
// start substituting the arguments into the '%s' placeholders
|
||||||
|
StringBuilder builder = new StringBuilder(template.length() + 16 * numArgs); |
||||||
|
int templateStart = 0; |
||||||
|
int i = 0; |
||||||
|
while (i < numArgs) { |
||||||
|
int placeholderStart = template.indexOf("%s", templateStart); |
||||||
|
if (placeholderStart == -1) { |
||||||
|
break; |
||||||
|
} |
||||||
|
builder.append(template.substring(templateStart, placeholderStart)); |
||||||
|
builder.append(args[i++]); |
||||||
|
templateStart = placeholderStart + 2; |
||||||
|
} |
||||||
|
builder.append(template.substring(templateStart)); |
||||||
|
|
||||||
|
// if we run out of placeholders, append the extra args in square braces
|
||||||
|
if (i < numArgs) { |
||||||
|
builder.append(" ["); |
||||||
|
builder.append(args[i++]); |
||||||
|
while (i < numArgs) { |
||||||
|
builder.append(", "); |
||||||
|
builder.append(args[i++]); |
||||||
|
} |
||||||
|
builder.append(']'); |
||||||
|
} |
||||||
|
|
||||||
|
return builder.toString(); |
||||||
|
} |
||||||
|
|
||||||
|
// ------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/** Private constructor to prevent instantiation. */ |
||||||
|
private Preconditions() {} |
||||||
|
} |
||||||
@ -0,0 +1,35 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package org.apache.dolphinscheduler.common.zk; |
||||||
|
|
||||||
|
import org.apache.curator.framework.CuratorFramework; |
||||||
|
import org.apache.curator.framework.recipes.cache.TreeCacheEvent; |
||||||
|
import org.apache.curator.framework.recipes.cache.TreeCacheListener; |
||||||
|
|
||||||
|
public abstract class AbstractListener implements TreeCacheListener { |
||||||
|
|
||||||
|
@Override |
||||||
|
public final void childEvent(final CuratorFramework client, final TreeCacheEvent event) throws Exception { |
||||||
|
String path = null == event.getData() ? "" : event.getData().getPath(); |
||||||
|
if (path.isEmpty()) { |
||||||
|
return; |
||||||
|
} |
||||||
|
dataChanged(client, event, path); |
||||||
|
} |
||||||
|
|
||||||
|
protected abstract void dataChanged(final CuratorFramework client, final TreeCacheEvent event, final String path); |
||||||
|
} |
||||||
@ -0,0 +1,48 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package org.apache.dolphinscheduler.common.zk; |
||||||
|
|
||||||
|
import org.apache.curator.ensemble.EnsembleProvider; |
||||||
|
|
||||||
|
import java.io.IOException; |
||||||
|
|
||||||
|
/** |
||||||
|
* default conf provider |
||||||
|
*/ |
||||||
|
public class DefaultEnsembleProvider implements EnsembleProvider { |
||||||
|
|
||||||
|
private final String serverList; |
||||||
|
|
||||||
|
public DefaultEnsembleProvider(String serverList){ |
||||||
|
this.serverList = serverList; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public void start() throws Exception { |
||||||
|
//NOP
|
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public String getConnectionString() { |
||||||
|
return serverList; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public void close() throws IOException { |
||||||
|
//NOP
|
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,82 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package org.apache.dolphinscheduler.common.zk; |
||||||
|
|
||||||
|
import org.apache.curator.framework.recipes.cache.ChildData; |
||||||
|
import org.apache.curator.framework.recipes.cache.TreeCache; |
||||||
|
import org.apache.curator.framework.recipes.cache.TreeCacheListener; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
import org.springframework.stereotype.Component; |
||||||
|
|
||||||
|
import java.nio.charset.StandardCharsets; |
||||||
|
import java.util.concurrent.ConcurrentHashMap; |
||||||
|
|
||||||
|
import static org.apache.dolphinscheduler.common.utils.Preconditions.*; |
||||||
|
import static org.apache.dolphinscheduler.common.utils.Preconditions.checkNotNull; |
||||||
|
|
||||||
|
@Component |
||||||
|
public class ZookeeperCachedOperator extends ZookeeperOperator { |
||||||
|
|
||||||
|
private final Logger logger = LoggerFactory.getLogger(ZookeeperCachedOperator.class); |
||||||
|
|
||||||
|
//kay is zk path, value is TreeCache
|
||||||
|
private ConcurrentHashMap<String, TreeCache> allCaches = new ConcurrentHashMap<>(); |
||||||
|
|
||||||
|
/** |
||||||
|
* @param cachePath zk path |
||||||
|
* @param listener operator |
||||||
|
*/ |
||||||
|
public void registerListener(final String cachePath, final TreeCacheListener listener) { |
||||||
|
TreeCache newCache = new TreeCache(zkClient, cachePath); |
||||||
|
logger.info("add listener to zk path: {}", cachePath); |
||||||
|
try { |
||||||
|
newCache.start(); |
||||||
|
} catch (Exception e) { |
||||||
|
logger.error("add listener to zk path: {} failed", cachePath); |
||||||
|
throw new RuntimeException(e); |
||||||
|
} |
||||||
|
|
||||||
|
newCache.getListenable().addListener(listener); |
||||||
|
|
||||||
|
allCaches.put(cachePath, newCache); |
||||||
|
} |
||||||
|
|
||||||
|
public String getFromCache(final String cachePath, final String key) { |
||||||
|
ChildData resultInCache = allCaches.get(checkNotNull(cachePath)).getCurrentData(key); |
||||||
|
if (null != resultInCache) { |
||||||
|
return null == resultInCache.getData() ? null : new String(resultInCache.getData(), StandardCharsets.UTF_8); |
||||||
|
} |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
public TreeCache getTreeCache(final String cachePath) { |
||||||
|
return allCaches.get(checkNotNull(cachePath)); |
||||||
|
} |
||||||
|
|
||||||
|
public void close() { |
||||||
|
|
||||||
|
allCaches.forEach((path, cache) -> { |
||||||
|
cache.close(); |
||||||
|
try { |
||||||
|
Thread.sleep(500); |
||||||
|
} catch (InterruptedException ignore) { |
||||||
|
} |
||||||
|
}); |
||||||
|
super.close(); |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,71 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package org.apache.dolphinscheduler.common.zk; |
||||||
|
|
||||||
|
import lombok.AllArgsConstructor; |
||||||
|
import lombok.Builder; |
||||||
|
import lombok.Data; |
||||||
|
import lombok.NoArgsConstructor; |
||||||
|
import org.apache.commons.configuration.Configuration; |
||||||
|
import org.springframework.beans.factory.annotation.Value; |
||||||
|
import org.springframework.context.annotation.PropertySource; |
||||||
|
import org.springframework.stereotype.Component; |
||||||
|
|
||||||
|
/** |
||||||
|
* zookeeper conf |
||||||
|
*/ |
||||||
|
@Component |
||||||
|
@Data |
||||||
|
@AllArgsConstructor |
||||||
|
@NoArgsConstructor |
||||||
|
@Builder |
||||||
|
@PropertySource("classpath:zookeeper.properties") |
||||||
|
public class ZookeeperConfig { |
||||||
|
|
||||||
|
//zk connect config
|
||||||
|
@Value("${zookeeper.quorum}") |
||||||
|
private String serverList; |
||||||
|
|
||||||
|
@Value("${zookeeper.retry.base.sleep:100}") |
||||||
|
private int baseSleepTimeMs; |
||||||
|
|
||||||
|
@Value("${zookeeper.retry.max.sleep:30000}") |
||||||
|
private int maxSleepMs; |
||||||
|
|
||||||
|
@Value("${zookeeper.retry.maxtime:10}") |
||||||
|
private int maxRetries; |
||||||
|
|
||||||
|
@Value("${zookeeper.session.timeout:60000}") |
||||||
|
private int sessionTimeoutMs; |
||||||
|
|
||||||
|
@Value("${zookeeper.connection.timeout:30000}") |
||||||
|
private int connectionTimeoutMs; |
||||||
|
|
||||||
|
@Value("${zookeeper.connection.digest: }") |
||||||
|
private String digest; |
||||||
|
|
||||||
|
//ds scheduler install config
|
||||||
|
@Value("${zookeeper.dolphinscheduler.root:/dolphinscheduler}") |
||||||
|
private String dsRoot; |
||||||
|
|
||||||
|
public static ZookeeperConfig getFromConf(Configuration conf){ |
||||||
|
return ZookeeperConfig.builder().serverList(conf.getString("zookeeper.quorum")).baseSleepTimeMs(conf.getInt("zookeeper.retry.base.sleep")) |
||||||
|
.maxSleepMs(conf.getInt("zookeeper.retry.max.sleep")).maxRetries(conf.getInt("zookeeper.retry.maxtime")) |
||||||
|
.sessionTimeoutMs(conf.getInt("zookeeper.session.timeout")).connectionTimeoutMs(conf.getInt("zookeeper.connection.timeout")) |
||||||
|
.dsRoot(conf.getString("zookeeper.dolphinscheduler.root")).build(); |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,232 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package org.apache.dolphinscheduler.common.zk; |
||||||
|
|
||||||
|
import org.apache.commons.lang.StringUtils; |
||||||
|
import org.apache.curator.framework.CuratorFramework; |
||||||
|
import org.apache.curator.framework.CuratorFrameworkFactory; |
||||||
|
import org.apache.curator.framework.api.ACLProvider; |
||||||
|
import org.apache.curator.framework.state.ConnectionState; |
||||||
|
import org.apache.curator.retry.ExponentialBackoffRetry; |
||||||
|
import org.apache.curator.utils.CloseableUtils; |
||||||
|
import org.apache.zookeeper.CreateMode; |
||||||
|
import org.apache.zookeeper.KeeperException; |
||||||
|
import org.apache.zookeeper.ZooDefs; |
||||||
|
import org.apache.zookeeper.data.ACL; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
import org.springframework.beans.factory.InitializingBean; |
||||||
|
import org.springframework.beans.factory.annotation.Autowired; |
||||||
|
import org.springframework.stereotype.Component; |
||||||
|
import org.springframework.util.CollectionUtils; |
||||||
|
|
||||||
|
import java.nio.charset.StandardCharsets; |
||||||
|
import java.util.List; |
||||||
|
|
||||||
|
import static org.apache.dolphinscheduler.common.utils.Preconditions.*; |
||||||
|
import static org.apache.dolphinscheduler.common.utils.Preconditions.checkNotNull; |
||||||
|
|
||||||
|
/** |
||||||
|
* zk base operator |
||||||
|
*/ |
||||||
|
@Component |
||||||
|
public class ZookeeperOperator implements InitializingBean { |
||||||
|
|
||||||
|
private final Logger logger = LoggerFactory.getLogger(ZookeeperOperator.class); |
||||||
|
|
||||||
|
@Autowired |
||||||
|
private ZookeeperConfig zookeeperConfig; |
||||||
|
|
||||||
|
protected CuratorFramework zkClient; |
||||||
|
|
||||||
|
@Override |
||||||
|
public void afterPropertiesSet() throws Exception { |
||||||
|
this.zkClient = buildClient(); |
||||||
|
initStateLister(); |
||||||
|
//init();
|
||||||
|
} |
||||||
|
|
||||||
|
//for subclass
|
||||||
|
//protected void init(){}
|
||||||
|
|
||||||
|
public void initStateLister() { |
||||||
|
checkNotNull(zkClient); |
||||||
|
|
||||||
|
zkClient.getConnectionStateListenable().addListener((client, newState) -> { |
||||||
|
if(newState == ConnectionState.LOST){ |
||||||
|
logger.error("connection lost from zookeeper"); |
||||||
|
} else if(newState == ConnectionState.RECONNECTED){ |
||||||
|
logger.info("reconnected to zookeeper"); |
||||||
|
} else if(newState == ConnectionState.SUSPENDED){ |
||||||
|
logger.warn("connection SUSPENDED to zookeeper"); |
||||||
|
} |
||||||
|
}); |
||||||
|
} |
||||||
|
|
||||||
|
private CuratorFramework buildClient() { |
||||||
|
logger.info("zookeeper registry center init, server lists is: {}.", zookeeperConfig.getServerList()); |
||||||
|
|
||||||
|
CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder().ensembleProvider(new DefaultEnsembleProvider(checkNotNull(zookeeperConfig.getServerList(),"zookeeper quorum can't be null"))) |
||||||
|
.retryPolicy(new ExponentialBackoffRetry(zookeeperConfig.getBaseSleepTimeMs(), zookeeperConfig.getMaxRetries(), zookeeperConfig.getMaxSleepMs())); |
||||||
|
|
||||||
|
//these has default value
|
||||||
|
if (0 != zookeeperConfig.getSessionTimeoutMs()) { |
||||||
|
builder.sessionTimeoutMs(zookeeperConfig.getSessionTimeoutMs()); |
||||||
|
} |
||||||
|
if (0 != zookeeperConfig.getConnectionTimeoutMs()) { |
||||||
|
builder.connectionTimeoutMs(zookeeperConfig.getConnectionTimeoutMs()); |
||||||
|
} |
||||||
|
if (StringUtils.isNotBlank(zookeeperConfig.getDigest())) { |
||||||
|
builder.authorization("digest", zookeeperConfig.getDigest().getBytes(StandardCharsets.UTF_8)).aclProvider(new ACLProvider() { |
||||||
|
|
||||||
|
@Override |
||||||
|
public List<ACL> getDefaultAcl() { |
||||||
|
return ZooDefs.Ids.CREATOR_ALL_ACL; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public List<ACL> getAclForPath(final String path) { |
||||||
|
return ZooDefs.Ids.CREATOR_ALL_ACL; |
||||||
|
} |
||||||
|
}); |
||||||
|
} |
||||||
|
zkClient = builder.build(); |
||||||
|
zkClient.start(); |
||||||
|
try { |
||||||
|
zkClient.blockUntilConnected(); |
||||||
|
} catch (final Exception ex) { |
||||||
|
throw new RuntimeException(ex); |
||||||
|
} |
||||||
|
return zkClient; |
||||||
|
} |
||||||
|
|
||||||
|
public String get(final String key) { |
||||||
|
try { |
||||||
|
return new String(zkClient.getData().forPath(key), StandardCharsets.UTF_8); |
||||||
|
} catch (Exception ex) { |
||||||
|
logger.error("get key : {}", key, ex); |
||||||
|
} |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
public List<String> getChildrenKeys(final String key) { |
||||||
|
List<String> values; |
||||||
|
try { |
||||||
|
values = zkClient.getChildren().forPath(key); |
||||||
|
if (CollectionUtils.isEmpty(values)) { |
||||||
|
logger.warn("getChildrenKeys key : {} is empty", key); |
||||||
|
} |
||||||
|
return values; |
||||||
|
} catch (InterruptedException ex) { |
||||||
|
logger.error("getChildrenKeys key : {} InterruptedException", key); |
||||||
|
throw new IllegalStateException(ex); |
||||||
|
} catch (Exception ex) { |
||||||
|
logger.error("getChildrenKeys key : {}", key, ex); |
||||||
|
throw new RuntimeException(ex); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
public boolean isExisted(final String key) { |
||||||
|
try { |
||||||
|
return zkClient.checkExists().forPath(key) != null; |
||||||
|
} catch (Exception ex) { |
||||||
|
logger.error("isExisted key : {}", key, ex); |
||||||
|
} |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
public void persist(final String key, final String value) { |
||||||
|
try { |
||||||
|
if (!isExisted(key)) { |
||||||
|
zkClient.create().creatingParentsIfNeeded().withMode(CreateMode.PERSISTENT).forPath(key, value.getBytes(StandardCharsets.UTF_8)); |
||||||
|
} else { |
||||||
|
update(key, value); |
||||||
|
} |
||||||
|
} catch (Exception ex) { |
||||||
|
logger.error("persist key : {} , value : {}", key, value, ex); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
public void update(final String key, final String value) { |
||||||
|
try { |
||||||
|
zkClient.inTransaction().check().forPath(key).and().setData().forPath(key, value.getBytes(StandardCharsets.UTF_8)).and().commit(); |
||||||
|
} catch (Exception ex) { |
||||||
|
logger.error("update key : {} , value : {}", key, value, ex); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
public void persistEphemeral(final String key, final String value) { |
||||||
|
try { |
||||||
|
if (isExisted(key)) { |
||||||
|
try { |
||||||
|
zkClient.delete().deletingChildrenIfNeeded().forPath(key); |
||||||
|
} catch (KeeperException.NoNodeException ignore) { |
||||||
|
//NOP
|
||||||
|
} |
||||||
|
} |
||||||
|
zkClient.create().creatingParentsIfNeeded().withMode(CreateMode.EPHEMERAL).forPath(key, value.getBytes(StandardCharsets.UTF_8)); |
||||||
|
} catch (final Exception ex) { |
||||||
|
logger.error("persistEphemeral key : {} , value : {}", key, value, ex); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
public void persistEphemeral(String key, String value, boolean overwrite) { |
||||||
|
try { |
||||||
|
if (overwrite) { |
||||||
|
persistEphemeral(key, value); |
||||||
|
} else { |
||||||
|
if (!isExisted(key)) { |
||||||
|
zkClient.create().creatingParentsIfNeeded().withMode(CreateMode.EPHEMERAL).forPath(key, value.getBytes(StandardCharsets.UTF_8)); |
||||||
|
} |
||||||
|
} |
||||||
|
} catch (final Exception ex) { |
||||||
|
logger.error("persistEphemeral key : {} , value : {}, overwrite : {}", key, value, overwrite, ex); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
public void persistEphemeralSequential(final String key, String value) { |
||||||
|
try { |
||||||
|
zkClient.create().creatingParentsIfNeeded().withMode(CreateMode.EPHEMERAL_SEQUENTIAL).forPath(key, value.getBytes(StandardCharsets.UTF_8)); |
||||||
|
} catch (final Exception ex) { |
||||||
|
logger.error("persistEphemeralSequential key : {}", key, ex); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
public void remove(final String key) { |
||||||
|
try { |
||||||
|
if (isExisted(key)) { |
||||||
|
zkClient.delete().deletingChildrenIfNeeded().forPath(key); |
||||||
|
} |
||||||
|
} catch (KeeperException.NoNodeException ignore) { |
||||||
|
//NOP
|
||||||
|
} catch (final Exception ex) { |
||||||
|
logger.error("remove key : {}", key, ex); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
public CuratorFramework getZkClient() { |
||||||
|
return zkClient; |
||||||
|
} |
||||||
|
|
||||||
|
public ZookeeperConfig getZookeeperConfig() { |
||||||
|
return zookeeperConfig; |
||||||
|
} |
||||||
|
|
||||||
|
public void close() { |
||||||
|
CloseableUtils.closeQuietly(zkClient); |
||||||
|
} |
||||||
|
} |
||||||
Loading…
Reference in new issue