From 74712a44487ec67d4f1eb6b847b6d39c0f11e80c Mon Sep 17 00:00:00 2001 From: JinyLeeChina <42576980+JinyLeeChina@users.noreply.github.com> Date: Wed, 13 Jan 2021 14:23:19 +0800 Subject: [PATCH] [Feature][JsonSplit] snowFlakeUtils (#4432) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * snowFlakeUtils * snowFlakeUtils * snowFlakeUtils * fix instance * 'codestyle' * 'codestyle' * 'codestyle' * 'codestyle' * 'codestyle' Co-authored-by: 文蛤 --- dolphinscheduler-api/pom.xml | 5 + .../dolphinscheduler/common/Constants.java | 5 + .../common/utils/SnowFlakeUtils.java | 103 ++++++++++++++++++ .../src/main/resources/common.properties | 2 + .../common/utils/SnowFlakeUtilsTest.java | 33 ++++++ 5 files changed, 148 insertions(+) create mode 100644 dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SnowFlakeUtils.java create mode 100644 dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SnowFlakeUtilsTest.java diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index 6ee75837b7..3ff1ba4b2c 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -39,6 +39,11 @@ dolphinscheduler-dao + + org.apache.dolphinscheduler + dolphinscheduler-common + + org.springframework.boot diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java index 17fa753a37..3998f23bd6 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java @@ -1029,4 +1029,9 @@ public final class Constants { * pstree, get pud and sub pid */ public static final String PSTREE = "pstree"; + + /** + * snow flake, data center id, this id must be greater than 0 and less than 32 + */ + public static final String SNOW_FLAKE_DATA_CENTER_ID = "data.center.id"; } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SnowFlakeUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SnowFlakeUtils.java new file mode 100644 index 0000000000..4d31bbedb2 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SnowFlakeUtils.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.utils; + +import org.apache.dolphinscheduler.common.Constants; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Objects; + +public class SnowFlakeUtils { + // start timestamp + private static final long START_TIMESTAMP = 1609430400000L; //2021-01-01 + // Number of digits + private static final long SEQUENCE_BIT = 12; + private static final long MACHINE_BIT = 5; + private static final long DATA_CENTER_BIT = 5; + // Maximum value + private static final long MAX_DATA_CENTER_NUM = ~(-1L << DATA_CENTER_BIT); + private static final long MAX_SEQUENCE = ~(-1L << SEQUENCE_BIT); + // The displacement to the left + private static final long MACHINE_LEFT = SEQUENCE_BIT; + private static final long DATA_CENTER_LEFT = SEQUENCE_BIT + MACHINE_BIT; + private static final long TIMESTAMP_LEFT = DATA_CENTER_LEFT + DATA_CENTER_BIT; + private final int dataCenterId; + private final int machineId; + private long sequence = 0L; + private long lastTimestamp = -1L; + + private SnowFlakeUtils() throws SnowFlakeException { + this.dataCenterId = PropertyUtils.getInt(Constants.SNOW_FLAKE_DATA_CENTER_ID, 1); + if (dataCenterId > MAX_DATA_CENTER_NUM || dataCenterId < 0) { + throw new IllegalArgumentException(String.format("dataCenterId can't be greater than %d or less than 0", MAX_DATA_CENTER_NUM)); + } + try { + this.machineId = Math.abs(Objects.hash(InetAddress.getLocalHost().getHostName())) % 32; + } catch (UnknownHostException e) { + throw new SnowFlakeException(e.getMessage()); + } + } + + private static SnowFlakeUtils instance = null; + + public static synchronized SnowFlakeUtils getInstance() throws SnowFlakeException { + if (instance == null) { + instance = new SnowFlakeUtils(); + } + return instance; + } + + public synchronized long nextId() throws SnowFlakeException { + long currStmp = nowTimestamp(); + if (currStmp < lastTimestamp) { + throw new SnowFlakeException("Clock moved backwards. Refusing to generate id"); + } + if (currStmp == lastTimestamp) { + sequence = (sequence + 1) & MAX_SEQUENCE; + if (sequence == 0L) { + currStmp = getNextMill(); + } + } else { + sequence = 0L; + } + lastTimestamp = currStmp; + return (currStmp - START_TIMESTAMP) << TIMESTAMP_LEFT + | dataCenterId << DATA_CENTER_LEFT + | machineId << MACHINE_LEFT + | sequence; + } + + private long getNextMill() { + long mill = nowTimestamp(); + while (mill <= lastTimestamp) { + mill = nowTimestamp(); + } + return mill; + } + + private long nowTimestamp() { + return System.currentTimeMillis(); + } + + public static class SnowFlakeException extends Exception { + public SnowFlakeException(String message) { + super(message); + } + } +} diff --git a/dolphinscheduler-common/src/main/resources/common.properties b/dolphinscheduler-common/src/main/resources/common.properties index b8c21c853b..e11b45569f 100644 --- a/dolphinscheduler-common/src/main/resources/common.properties +++ b/dolphinscheduler-common/src/main/resources/common.properties @@ -75,3 +75,5 @@ datasource.encryption.salt=!@#$%^&* # Network IP gets priority, default inner outer #dolphin.scheduler.network.priority.strategy=default +# 0