ligang
6 years ago
108 changed files with 13345 additions and 0 deletions
@ -0,0 +1,380 @@ |
|||||||
|
<?xml version="1.0"?> |
||||||
|
<project |
||||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" |
||||||
|
xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> |
||||||
|
<modelVersion>4.0.0</modelVersion> |
||||||
|
<parent> |
||||||
|
<artifactId>escheduler</artifactId> |
||||||
|
<groupId>cn.analysys</groupId> |
||||||
|
<version>1.0.0</version> |
||||||
|
</parent> |
||||||
|
<artifactId>escheduler-common</artifactId> |
||||||
|
<name>escheduler-common</name> |
||||||
|
<url>http://maven.apache.org</url> |
||||||
|
<packaging>jar</packaging> |
||||||
|
<properties> |
||||||
|
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> |
||||||
|
</properties> |
||||||
|
<dependencies> |
||||||
|
<dependency> |
||||||
|
<groupId>com.alibaba</groupId> |
||||||
|
<artifactId>fastjson</artifactId> |
||||||
|
<scope>compile</scope> |
||||||
|
</dependency> |
||||||
|
|
||||||
|
<dependency> |
||||||
|
<groupId>org.apache.httpcomponents</groupId> |
||||||
|
<artifactId>httpclient</artifactId> |
||||||
|
</dependency> |
||||||
|
|
||||||
|
<dependency> |
||||||
|
<groupId>junit</groupId> |
||||||
|
<artifactId>junit</artifactId> |
||||||
|
<scope>test</scope> |
||||||
|
</dependency> |
||||||
|
|
||||||
|
<dependency> |
||||||
|
<groupId>commons-configuration</groupId> |
||||||
|
<artifactId>commons-configuration</artifactId> |
||||||
|
</dependency> |
||||||
|
<dependency> |
||||||
|
<groupId>com.fasterxml.jackson.core</groupId> |
||||||
|
<artifactId>jackson-annotations</artifactId> |
||||||
|
</dependency> |
||||||
|
<dependency> |
||||||
|
<groupId>com.fasterxml.jackson.core</groupId> |
||||||
|
<artifactId>jackson-databind</artifactId> |
||||||
|
</dependency> |
||||||
|
<dependency> |
||||||
|
<groupId>org.apache.curator</groupId> |
||||||
|
<artifactId>curator-client</artifactId> |
||||||
|
<version>2.12.0</version> |
||||||
|
<exclusions> |
||||||
|
<exclusion> |
||||||
|
<groupId>log4j</groupId> |
||||||
|
<artifactId>log4j</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>io.netty</groupId> |
||||||
|
<artifactId>netty</artifactId> |
||||||
|
</exclusion> |
||||||
|
</exclusions> |
||||||
|
</dependency> |
||||||
|
<dependency> |
||||||
|
<groupId>org.apache.commons</groupId> |
||||||
|
<artifactId>commons-collections4</artifactId> |
||||||
|
</dependency> |
||||||
|
|
||||||
|
<dependency> |
||||||
|
<groupId>org.apache.hadoop</groupId> |
||||||
|
<artifactId>hadoop-common</artifactId> |
||||||
|
<exclusions> |
||||||
|
<exclusion> |
||||||
|
<groupId>org.slf4j</groupId> |
||||||
|
<artifactId>slf4j-log4j12</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>jdk.tools</groupId> |
||||||
|
<artifactId>jdk.tools</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<artifactId>servlet-api</artifactId> |
||||||
|
<groupId>javax.servlet</groupId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>javax.servlet</groupId> |
||||||
|
<artifactId>servlet-api</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>log4j</groupId> |
||||||
|
<artifactId>log4j</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>org.apache.curator</groupId> |
||||||
|
<artifactId>curator-client</artifactId> |
||||||
|
</exclusion> |
||||||
|
|
||||||
|
<exclusion> |
||||||
|
<groupId>commons-configuration</groupId> |
||||||
|
<artifactId>commons-configuration</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>io.grpc</groupId> |
||||||
|
<artifactId>grpc-protobuf</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>io.netty</groupId> |
||||||
|
<artifactId>netty</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>org.codehaus.jackson</groupId> |
||||||
|
<artifactId>jackson-core-asl</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>org.codehaus.jackson</groupId> |
||||||
|
<artifactId>jackson-mapper-asl</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>com.google.protobuf</groupId> |
||||||
|
<artifactId>jackson-mapper-asl</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>com.google.code.gson</groupId> |
||||||
|
<artifactId>gson</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>org.apache.commons</groupId> |
||||||
|
<artifactId>commons-math3</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>xmlenc</groupId> |
||||||
|
<artifactId>xmlenc</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>commons-net</groupId> |
||||||
|
<artifactId>commons-net</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>org.apache.avro</groupId> |
||||||
|
<artifactId>avro</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>org.apache.zookeeper</groupId> |
||||||
|
<artifactId>zookeeper</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<artifactId>jsr305</artifactId> |
||||||
|
<groupId>com.google.code.findbugs</groupId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>javax.servlet.jsp</groupId> |
||||||
|
<artifactId>jsp-api</artifactId> |
||||||
|
</exclusion> |
||||||
|
|
||||||
|
</exclusions> |
||||||
|
</dependency> |
||||||
|
<dependency> |
||||||
|
<groupId>org.apache.hadoop</groupId> |
||||||
|
<artifactId>hadoop-client</artifactId> |
||||||
|
<exclusions> |
||||||
|
<exclusion> |
||||||
|
<groupId>org.slf4j</groupId> |
||||||
|
<artifactId>slf4j-log4j12</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<artifactId>servlet-api</artifactId> |
||||||
|
<groupId>javax.servlet</groupId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>org.codehaus.jackson</groupId> |
||||||
|
<artifactId>jackson-jaxrs</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>org.codehaus.jackson</groupId> |
||||||
|
<artifactId>jackson-xc</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>com.google.protobuf</groupId> |
||||||
|
<artifactId>protobuf-java</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>org.fusesource.leveldbjni</groupId> |
||||||
|
<artifactId>leveldbjni-all</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>org.apache.zookeeper</groupId> |
||||||
|
<artifactId>zookeeper</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>org.apache.hadoop</groupId> |
||||||
|
<artifactId>hadoop-mapreduce-client-shuffle</artifactId> |
||||||
|
</exclusion> |
||||||
|
</exclusions> |
||||||
|
</dependency> |
||||||
|
|
||||||
|
<dependency> |
||||||
|
<groupId>javax.servlet</groupId> |
||||||
|
<artifactId>javax.servlet-api</artifactId> |
||||||
|
</dependency> |
||||||
|
|
||||||
|
<dependency> |
||||||
|
<groupId>org.apache.hadoop</groupId> |
||||||
|
<artifactId>hadoop-hdfs</artifactId> |
||||||
|
<exclusions> |
||||||
|
<exclusion> |
||||||
|
<groupId>javax.servlet</groupId> |
||||||
|
<artifactId>servlet-api</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>io.netty</groupId> |
||||||
|
<artifactId>netty</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>com.google.protobuf</groupId> |
||||||
|
<artifactId>protobuf-java</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>xmlenc</groupId> |
||||||
|
<artifactId>xmlenc</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>io.netty</groupId> |
||||||
|
<artifactId>netty-all</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>org.fusesource.leveldbjni</groupId> |
||||||
|
<artifactId>leveldbjni-all</artifactId> |
||||||
|
</exclusion> |
||||||
|
</exclusions> |
||||||
|
</dependency> |
||||||
|
|
||||||
|
<dependency> |
||||||
|
<groupId>org.apache.commons</groupId> |
||||||
|
<artifactId>commons-lang3</artifactId> |
||||||
|
</dependency> |
||||||
|
|
||||||
|
<dependency> |
||||||
|
<groupId>org.postgresql</groupId> |
||||||
|
<artifactId>postgresql</artifactId> |
||||||
|
</dependency> |
||||||
|
|
||||||
|
<dependency> |
||||||
|
<groupId>org.apache.httpcomponents</groupId> |
||||||
|
<artifactId>httpclient</artifactId> |
||||||
|
</dependency> |
||||||
|
<dependency> |
||||||
|
<groupId>org.apache.hive</groupId> |
||||||
|
<artifactId>hive-jdbc</artifactId> |
||||||
|
<exclusions> |
||||||
|
<exclusion> |
||||||
|
<artifactId>slf4j-log4j12</artifactId> |
||||||
|
<groupId>org.slf4j</groupId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>org.eclipse.jetty.aggregate</groupId> |
||||||
|
<artifactId>jetty-all</artifactId> |
||||||
|
</exclusion> |
||||||
|
|
||||||
|
<exclusion> |
||||||
|
<groupId>org.apache.ant</groupId> |
||||||
|
<artifactId>ant</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>io.dropwizard.metrics</groupId> |
||||||
|
<artifactId>metrics-json</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>io.dropwizard.metrics</groupId> |
||||||
|
<artifactId>metrics-jvm</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>com.github.joshelser</groupId> |
||||||
|
<artifactId>dropwizard-metrics-hadoop-metrics2-reporter</artifactId> |
||||||
|
</exclusion> |
||||||
|
|
||||||
|
<exclusion> |
||||||
|
<groupId>io.netty</groupId> |
||||||
|
<artifactId>netty-all</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>com.google.code.gson</groupId> |
||||||
|
<artifactId>gson</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>com.google.code.findbugs</groupId> |
||||||
|
<artifactId>jsr305</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>io.dropwizard.metrics</groupId> |
||||||
|
<artifactId>metrics-core</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>javax.servlet</groupId> |
||||||
|
<artifactId>servlet-api</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>org.apache.avro</groupId> |
||||||
|
<artifactId>avro</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>org.apache.commons</groupId> |
||||||
|
<artifactId>commons-compress</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>org.apache.curator</groupId> |
||||||
|
<artifactId>curator-client</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>org.apache.hadoop</groupId> |
||||||
|
<artifactId>hadoop-auth</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>org.apache.hadoop</groupId> |
||||||
|
<artifactId>hadoop-mapreduce-client-core</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>org.apache.hadoop</groupId> |
||||||
|
<artifactId>hadoop-yarn-api</artifactId> |
||||||
|
</exclusion> |
||||||
|
|
||||||
|
<exclusion> |
||||||
|
<groupId>org.apache.zookeeper</groupId> |
||||||
|
<artifactId>zookeeper</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>org.codehaus.jackson</groupId> |
||||||
|
<artifactId>jackson-jaxrs</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>org.codehaus.jackson</groupId> |
||||||
|
<artifactId>jackson-xc</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>com.google.protobuf</groupId> |
||||||
|
<artifactId>protobuf-java</artifactId> |
||||||
|
</exclusion> |
||||||
|
|
||||||
|
<exclusion> |
||||||
|
<groupId>org.json</groupId> |
||||||
|
<artifactId>json</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<artifactId>log4j-slf4j-impl</artifactId> |
||||||
|
<groupId>org.apache.logging.log4j</groupId> |
||||||
|
</exclusion> |
||||||
|
</exclusions> |
||||||
|
</dependency> |
||||||
|
|
||||||
|
<dependency> |
||||||
|
<groupId>org.mybatis</groupId> |
||||||
|
<artifactId>mybatis</artifactId> |
||||||
|
</dependency> |
||||||
|
<dependency> |
||||||
|
<groupId>ch.qos.logback</groupId> |
||||||
|
<artifactId>logback-classic</artifactId> |
||||||
|
</dependency> |
||||||
|
<dependency> |
||||||
|
<groupId>ch.qos.logback</groupId> |
||||||
|
<artifactId>logback-core</artifactId> |
||||||
|
</dependency> |
||||||
|
<dependency> |
||||||
|
<groupId>com.github.oshi</groupId> |
||||||
|
<artifactId>oshi-core</artifactId> |
||||||
|
</dependency> |
||||||
|
</dependencies> |
||||||
|
|
||||||
|
|
||||||
|
<build> |
||||||
|
<plugins> |
||||||
|
<plugin> |
||||||
|
<groupId>org.apache.maven.plugins</groupId> |
||||||
|
<artifactId>maven-compiler-plugin</artifactId> |
||||||
|
<configuration> |
||||||
|
<source>${java.version}</source> |
||||||
|
<target>${java.version}</target> |
||||||
|
<encoding>${project.build.sourceEncoding}</encoding> |
||||||
|
</configuration> |
||||||
|
</plugin> |
||||||
|
</plugins> |
||||||
|
</build> |
||||||
|
</project> |
@ -0,0 +1,815 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common; |
||||||
|
|
||||||
|
import cn.escheduler.common.utils.OSUtils; |
||||||
|
|
||||||
|
import java.util.regex.Pattern; |
||||||
|
|
||||||
|
/** |
||||||
|
* Constants |
||||||
|
*/ |
||||||
|
public final class Constants { |
||||||
|
|
||||||
|
/** |
||||||
|
* zookeeper properties path |
||||||
|
*/ |
||||||
|
public static final String ZOOKEEPER_PROPERTIES_PATH = "zookeeper.properties"; |
||||||
|
|
||||||
|
/** |
||||||
|
* worker properties path |
||||||
|
*/ |
||||||
|
public static final String WORKER_PROPERTIES_PATH = "worker.properties"; |
||||||
|
|
||||||
|
/** |
||||||
|
* master properties path |
||||||
|
*/ |
||||||
|
public static final String MASTER_PROPERTIES_PATH = "master.properties"; |
||||||
|
|
||||||
|
/** |
||||||
|
* hadoop properties path |
||||||
|
*/ |
||||||
|
public static final String HADOOP_PROPERTIES_PATH = "/common/hadoop/hadoop.properties"; |
||||||
|
|
||||||
|
/** |
||||||
|
* common properties path |
||||||
|
*/ |
||||||
|
public static final String COMMON_PROPERTIES_PATH = "/common/common.properties"; |
||||||
|
|
||||||
|
/** |
||||||
|
* dao properties path |
||||||
|
*/ |
||||||
|
public static final String DAO_PROPERTIES_PATH = "/dao/data_source.properties"; |
||||||
|
|
||||||
|
/** |
||||||
|
* fs.defaultFS |
||||||
|
*/ |
||||||
|
public static final String FS_DEFAULTFS = "fs.defaultFS"; |
||||||
|
|
||||||
|
/** |
||||||
|
* yarn.resourcemanager.ha.rm.idsfs.defaultFS |
||||||
|
*/ |
||||||
|
public static final String YARN_RESOURCEMANAGER_HA_RM_IDS = "yarn.resourcemanager.ha.rm.ids"; |
||||||
|
|
||||||
|
/** |
||||||
|
* yarn.application.status.address |
||||||
|
*/ |
||||||
|
public static final String YARN_APPLICATION_STATUS_ADDRESS = "yarn.application.status.address"; |
||||||
|
|
||||||
|
/** |
||||||
|
* spring.redis.maxIdle |
||||||
|
*/ |
||||||
|
public static final String SPRING_REDIS_MAXIDLE = "spring.redis.maxIdle"; |
||||||
|
|
||||||
|
/** |
||||||
|
* spring.redis.maxTotal |
||||||
|
*/ |
||||||
|
public static final String SPRING_REDIS_MAXTOTAL = "spring.redis.maxTotal"; |
||||||
|
|
||||||
|
/** |
||||||
|
* spring.redis.host |
||||||
|
*/ |
||||||
|
public static final String SPRING_REDIS_HOST = "spring.redis.host"; |
||||||
|
|
||||||
|
/** |
||||||
|
* spring.redis.port |
||||||
|
*/ |
||||||
|
public static final String SPRING_REDIS_PORT = "spring.redis.port"; |
||||||
|
|
||||||
|
/** |
||||||
|
* hdfs configuration |
||||||
|
* data.store2hdfs.basepath |
||||||
|
*/ |
||||||
|
public static final String DATA_STORE_2_HDFS_BASEPATH = "data.store2hdfs.basepath"; |
||||||
|
|
||||||
|
/** |
||||||
|
* data.basedir.path |
||||||
|
*/ |
||||||
|
public static final String DATA_BASEDIR_PATH = "data.basedir.path"; |
||||||
|
|
||||||
|
/** |
||||||
|
* data.download.basedir.path |
||||||
|
*/ |
||||||
|
public static final String DATA_DOWNLOAD_BASEDIR_PATH = "data.download.basedir.path"; |
||||||
|
|
||||||
|
/** |
||||||
|
* process.exec.basepath |
||||||
|
*/ |
||||||
|
public static final String PROCESS_EXEC_BASEPATH = "process.exec.basepath"; |
||||||
|
|
||||||
|
/** |
||||||
|
* escheduler.env.path |
||||||
|
*/ |
||||||
|
public static final String ESCHEDULER_ENV_PATH = "escheduler.env.path"; |
||||||
|
|
||||||
|
/** |
||||||
|
* escheduler.env.py |
||||||
|
*/ |
||||||
|
public static final String ESCHEDULER_ENV_PY = "escheduler.env.py"; |
||||||
|
|
||||||
|
/** |
||||||
|
* resource.view.suffixs |
||||||
|
*/ |
||||||
|
public static final String RESOURCE_VIEW_SUFFIXS = "resource.view.suffixs"; |
||||||
|
|
||||||
|
/** |
||||||
|
* development.state |
||||||
|
*/ |
||||||
|
public static final String DEVELOPMENT_STATE = "development.state"; |
||||||
|
|
||||||
|
/** |
||||||
|
* hdfs.startup.state |
||||||
|
*/ |
||||||
|
public static final String HDFS_STARTUP_STATE = "hdfs.startup.state"; |
||||||
|
|
||||||
|
/** |
||||||
|
* zookeeper quorum |
||||||
|
*/ |
||||||
|
public static final String ZOOKEEPER_QUORUM = "zookeeper.quorum"; |
||||||
|
|
||||||
|
/** |
||||||
|
* MasterServer directory registered in zookeeper |
||||||
|
*/ |
||||||
|
public static final String ZOOKEEPER_ESCHEDULER_MASTERS = "zookeeper.escheduler.masters"; |
||||||
|
|
||||||
|
/** |
||||||
|
* WorkerServer directory registered in zookeeper |
||||||
|
*/ |
||||||
|
public static final String ZOOKEEPER_ESCHEDULER_WORKERS = "zookeeper.escheduler.workers"; |
||||||
|
|
||||||
|
/** |
||||||
|
* all servers directory registered in zookeeper |
||||||
|
*/ |
||||||
|
public static final String ZOOKEEPER_ESCHEDULER_DEAD_SERVERS = "zookeeper.escheduler.dead.servers"; |
||||||
|
|
||||||
|
/** |
||||||
|
* MasterServer lock directory registered in zookeeper |
||||||
|
*/ |
||||||
|
public static final String ZOOKEEPER_ESCHEDULER_LOCK_MASTERS = "zookeeper.escheduler.lock.masters"; |
||||||
|
|
||||||
|
/** |
||||||
|
* WorkerServer lock directory registered in zookeeper |
||||||
|
*/ |
||||||
|
public static final String ZOOKEEPER_ESCHEDULER_LOCK_WORKERS = "zookeeper.escheduler.lock.workers"; |
||||||
|
|
||||||
|
/** |
||||||
|
* MasterServer failover directory registered in zookeeper |
||||||
|
*/ |
||||||
|
public static final String ZOOKEEPER_ESCHEDULER_LOCK_FAILOVER_MASTERS = "zookeeper.escheduler.lock.failover.masters"; |
||||||
|
|
||||||
|
/** |
||||||
|
* WorkerServer failover directory registered in zookeeper |
||||||
|
*/ |
||||||
|
public static final String ZOOKEEPER_ESCHEDULER_LOCK_FAILOVER_WORKERS = "zookeeper.escheduler.lock.failover.workers"; |
||||||
|
|
||||||
|
/** |
||||||
|
* need send warn times when master server or worker server failover |
||||||
|
*/ |
||||||
|
public static final int ESCHEDULER_WARN_TIMES_FAILOVER = 3; |
||||||
|
|
||||||
|
/** |
||||||
|
* comma , |
||||||
|
*/ |
||||||
|
public static final String COMMA = ","; |
||||||
|
|
||||||
|
/** |
||||||
|
* COLON : |
||||||
|
*/ |
||||||
|
public static final String COLON = ":"; |
||||||
|
|
||||||
|
/** |
||||||
|
* SINGLE_SLASH / |
||||||
|
*/ |
||||||
|
public static final String SINGLE_SLASH = "/"; |
||||||
|
|
||||||
|
/** |
||||||
|
* DOUBLE_SLASH //
|
||||||
|
*/ |
||||||
|
public static final String DOUBLE_SLASH = "//"; |
||||||
|
|
||||||
|
/** |
||||||
|
* SEMICOLON ; |
||||||
|
*/ |
||||||
|
public static final String SEMICOLON = ";"; |
||||||
|
|
||||||
|
/** |
||||||
|
* ZOOKEEPER_SESSION_TIMEOUT |
||||||
|
*/ |
||||||
|
public static final String ZOOKEEPER_SESSION_TIMEOUT = "zookeeper.session.timeout"; |
||||||
|
|
||||||
|
public static final String ZOOKEEPER_CONNECTION_TIMEOUT = "zookeeper.connection.timeout"; |
||||||
|
|
||||||
|
public static final String ZOOKEEPER_RETRY_SLEEP = "zookeeper.retry.sleep"; |
||||||
|
|
||||||
|
public static final String ZOOKEEPER_RETRY_MAXTIME = "zookeeper.retry.maxtime"; |
||||||
|
|
||||||
|
|
||||||
|
public static final String MASTER_HEARTBEAT_INTERVAL = "master.heartbeat.interval"; |
||||||
|
|
||||||
|
public static final String MASTER_EXEC_THREADS = "master.exec.threads"; |
||||||
|
|
||||||
|
public static final String MASTER_EXEC_TASK_THREADS = "master.exec.task.number"; |
||||||
|
|
||||||
|
|
||||||
|
public static final String MASTER_COMMIT_RETRY_TIMES = "master.task.commit.retryTimes"; |
||||||
|
|
||||||
|
public static final String MASTER_COMMIT_RETRY_INTERVAL = "master.task.commit.interval"; |
||||||
|
|
||||||
|
|
||||||
|
public static final String WORKER_EXEC_THREADS = "worker.exec.threads"; |
||||||
|
|
||||||
|
public static final String WORKER_HEARTBEAT_INTERVAL = "worker.heartbeat.interval"; |
||||||
|
|
||||||
|
public static final String WORKER_FETCH_TASK_NUM = "worker.fetch.task.num"; |
||||||
|
|
||||||
|
public static final String WORKER_MAX_CPULOAD_AVG = "worker.max.cpuload.avg"; |
||||||
|
|
||||||
|
public static final String WORKER_RESERVED_MEMORY = "worker.reserved.memory"; |
||||||
|
|
||||||
|
public static final String MASTER_MAX_CPULOAD_AVG = "master.max.cpuload.avg"; |
||||||
|
|
||||||
|
public static final String MASTER_RESERVED_MEMORY = "master.reserved.memory"; |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* escheduler tasks queue |
||||||
|
*/ |
||||||
|
public static final String SCHEDULER_TASKS_QUEUE = "tasks_queue"; |
||||||
|
|
||||||
|
public static final String SCHEDULER_TASKS_KILL = "tasks_kill"; |
||||||
|
public static final String ZOOKEEPER_SCHEDULER_ROOT = "zookeeper.escheduler.root"; |
||||||
|
|
||||||
|
public static final String SCHEDULER_QUEUE_IMPL = "escheduler.queue.impl"; |
||||||
|
|
||||||
|
public static final String SCHEDULER_QUEUE_REDIS_IMPL = "redis"; |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* date format of yyyy-MM-dd HH:mm:ss |
||||||
|
*/ |
||||||
|
public static final String YYYY_MM_DD_HH_MM_SS = "yyyy-MM-dd HH:mm:ss"; |
||||||
|
|
||||||
|
/** |
||||||
|
* date format of yyyyMMddHHmmss |
||||||
|
*/ |
||||||
|
public static final String YYYYMMDDHHMMSS = "yyyyMMddHHmmss"; |
||||||
|
|
||||||
|
/** |
||||||
|
* http connect time out |
||||||
|
*/ |
||||||
|
public static final int HTTP_CONNECT_TIMEOUT = 60 * 1000; |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* http connect request time out |
||||||
|
*/ |
||||||
|
public static final int HTTP_CONNECTION_REQUEST_TIMEOUT = 60 * 1000; |
||||||
|
|
||||||
|
/** |
||||||
|
* httpclient soceket time out |
||||||
|
*/ |
||||||
|
public static final int SOCKET_TIMEOUT = 60 * 1000; |
||||||
|
|
||||||
|
/** |
||||||
|
* http header |
||||||
|
*/ |
||||||
|
public static final String HTTP_HEADER_UNKNOWN = "unKnown"; |
||||||
|
|
||||||
|
/** |
||||||
|
* http X-Forwarded-For |
||||||
|
*/ |
||||||
|
public static final String HTTP_X_FORWARDED_FOR = "X-Forwarded-For"; |
||||||
|
|
||||||
|
/** |
||||||
|
* http X-Real-IP |
||||||
|
*/ |
||||||
|
public static final String HTTP_X_REAL_IP = "X-Real-IP"; |
||||||
|
|
||||||
|
/** |
||||||
|
* UTF-8 |
||||||
|
*/ |
||||||
|
public static final String UTF_8 = "UTF-8"; |
||||||
|
|
||||||
|
/** |
||||||
|
* user name regex |
||||||
|
*/ |
||||||
|
public static final Pattern REGEX_USER_NAME = Pattern.compile("[a-zA-Z0-9]{3,20}"); |
||||||
|
|
||||||
|
/** |
||||||
|
* email regex |
||||||
|
*/ |
||||||
|
public static final Pattern REGEX_MAIL_NAME = Pattern.compile("^[a-zA-Z0-9_-]+@[a-zA-Z0-9_-]+(\\.[a-zA-Z0-9_-]+)+$"); |
||||||
|
|
||||||
|
/** |
||||||
|
* read permission |
||||||
|
*/ |
||||||
|
public static final int READ_PERMISSION = 2 * 1; |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* write permission |
||||||
|
*/ |
||||||
|
public static final int WRITE_PERMISSION = 2 * 2; |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* execute permission |
||||||
|
*/ |
||||||
|
public static final int EXECUTE_PERMISSION = 1; |
||||||
|
|
||||||
|
/** |
||||||
|
* default admin permission |
||||||
|
*/ |
||||||
|
public static final int DEFAULT_ADMIN_PERMISSION = 7; |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* all permissions |
||||||
|
*/ |
||||||
|
public static final int ALL_PERMISSIONS = READ_PERMISSION | WRITE_PERMISSION | EXECUTE_PERMISSION; |
||||||
|
|
||||||
|
/** |
||||||
|
* max task timeout |
||||||
|
*/ |
||||||
|
public static final int MAX_TASK_TIMEOUT = 24 * 3600; |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* heartbeat threads number |
||||||
|
*/ |
||||||
|
public static final int defaulWorkerHeartbeatThreadNum = 5; |
||||||
|
|
||||||
|
/** |
||||||
|
* heartbeat interval |
||||||
|
*/ |
||||||
|
public static final int defaultWorkerHeartbeatInterval = 60; |
||||||
|
|
||||||
|
/** |
||||||
|
* worker fetch task number |
||||||
|
*/ |
||||||
|
public static final int defaultWorkerFetchTaskNum = 1; |
||||||
|
|
||||||
|
/** |
||||||
|
* worker execute threads number |
||||||
|
*/ |
||||||
|
public static final int defaultWorkerExecThreadNum = 10; |
||||||
|
|
||||||
|
/** |
||||||
|
* master cpu load |
||||||
|
*/ |
||||||
|
public static final int defaultMasterCpuLoad = Runtime.getRuntime().availableProcessors() * 2; |
||||||
|
|
||||||
|
/** |
||||||
|
* master reserved memory |
||||||
|
*/ |
||||||
|
public static final double defaultMasterReservedMemory = OSUtils.totalMemorySize() / 10; |
||||||
|
|
||||||
|
/** |
||||||
|
* worker cpu load |
||||||
|
*/ |
||||||
|
public static final int defaultWorkerCpuLoad = Runtime.getRuntime().availableProcessors() * 2; |
||||||
|
|
||||||
|
/** |
||||||
|
* worker reserved memory |
||||||
|
*/ |
||||||
|
public static final double defaultWorkerReservedMemory = OSUtils.totalMemorySize() / 10; |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* master execute threads number |
||||||
|
*/ |
||||||
|
public static final int defaultMasterExecThreadNum = 100; |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* default master concurrent task execute num |
||||||
|
*/ |
||||||
|
public static final int defaultMasterTaskExecNum = 20; |
||||||
|
|
||||||
|
/** |
||||||
|
* default log cache rows num,output when reach the number |
||||||
|
*/ |
||||||
|
public static final int defaultLogRowsNum = 4 * 16; |
||||||
|
|
||||||
|
/** |
||||||
|
* log flush interval,output when reach the interval |
||||||
|
*/ |
||||||
|
public static final int defaultLogFlushInterval = 1000; |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* default master heartbeat thread number |
||||||
|
*/ |
||||||
|
public static final int defaulMasterHeartbeatThreadNum = 5; |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* default master heartbeat interval |
||||||
|
*/ |
||||||
|
public static final int defaultMasterHeartbeatInterval = 60; |
||||||
|
|
||||||
|
/** |
||||||
|
* default master commit retry times |
||||||
|
*/ |
||||||
|
public static final int defaultMasterCommitRetryTimes = 5; |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* default master commit retry interval |
||||||
|
*/ |
||||||
|
public static final int defaultMasterCommitRetryInterval = 100; |
||||||
|
|
||||||
|
/** |
||||||
|
* time unit secong to minutes |
||||||
|
*/ |
||||||
|
public static final int SEC_2_MINUTES_TIME_UNIT = 60; |
||||||
|
|
||||||
|
|
||||||
|
/*** |
||||||
|
* |
||||||
|
* rpc port |
||||||
|
*/ |
||||||
|
public static final int RPC_PORT = 50051; |
||||||
|
|
||||||
|
/** |
||||||
|
* forbid running task |
||||||
|
*/ |
||||||
|
public static final String FLOWNODE_RUN_FLAG_FORBIDDEN = "FORBIDDEN"; |
||||||
|
|
||||||
|
/** |
||||||
|
* task record configuration path |
||||||
|
*/ |
||||||
|
public static final String TASK_RECORD_PROPERTIES_PATH = "dao/data_source.properties"; |
||||||
|
|
||||||
|
public static final String TASK_RECORD_URL = "task.record.datasource.url"; |
||||||
|
|
||||||
|
public static final String TASK_RECORD_FLAG = "task.record.flag"; |
||||||
|
|
||||||
|
public static final String TASK_RECORD_USER = "task.record.datasource.username"; |
||||||
|
|
||||||
|
public static final String TASK_RECORD_PWD = "task.record.datasource.password"; |
||||||
|
|
||||||
|
public static final String STATUS = "status"; |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* command parameter keys |
||||||
|
*/ |
||||||
|
public static final String CMDPARAM_RECOVER_PROCESS_ID_STRING = "ProcessInstanceId"; |
||||||
|
|
||||||
|
public static final String CMDPARAM_RECOVERY_START_NODE_STRING = "StartNodeIdList"; |
||||||
|
|
||||||
|
public static final String CMDPARAM_RECOVERY_WAITTING_THREAD = "WaittingThreadInstanceId"; |
||||||
|
|
||||||
|
public static final String CMDPARAM_SUB_PROCESS = "processInstanceId"; |
||||||
|
|
||||||
|
public static final String CMDPARAM_EMPTY_SUB_PROCESS = "0"; |
||||||
|
|
||||||
|
public static final String CMDPARAM_SUB_PROCESS_PARENT_INSTANCE_ID = "parentProcessInstanceId"; |
||||||
|
|
||||||
|
public static final String CMDPARAM_SUB_PROCESS_DEFINE_ID = "processDefinitionId"; |
||||||
|
|
||||||
|
public static final String CMDPARAM_START_NODE_NAMES = "StartNodeNameList"; |
||||||
|
|
||||||
|
/** |
||||||
|
* complement data start date |
||||||
|
*/ |
||||||
|
public static final String CMDPARAM_COMPLEMENT_DATA_START_DATE = "complementStartDate"; |
||||||
|
|
||||||
|
/** |
||||||
|
* complement data end date |
||||||
|
*/ |
||||||
|
public static final String CMDPARAM_COMPLEMENT_DATA_END_DATE = "complementEndDate"; |
||||||
|
|
||||||
|
/** |
||||||
|
* hadoop configuration |
||||||
|
*/ |
||||||
|
public static final String HADOOP_RM_STATE_ACTIVE = "ACTIVE"; |
||||||
|
|
||||||
|
public static final String HADOOP_RM_STATE_STANDBY = "STANDBY"; |
||||||
|
|
||||||
|
public static final String HADOOP_RESOURCE_MANAGER_HTTPADDRESS_PORT = "resource.manager.httpaddress.port"; |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* data source config |
||||||
|
*/ |
||||||
|
|
||||||
|
public static final String SPRING_DATASOURCE_DRIVER_CLASS_NAME = "spring.datasource.driver-class-name"; |
||||||
|
|
||||||
|
public static final String SPRING_DATASOURCE_URL = "spring.datasource.url"; |
||||||
|
|
||||||
|
public static final String SPRING_DATASOURCE_USERNAME = "spring.datasource.username"; |
||||||
|
|
||||||
|
public static final String SPRING_DATASOURCE_PASSWORD = "spring.datasource.password"; |
||||||
|
|
||||||
|
public static final String SPRING_DATASOURCE_VALIDATION_QUERY_TIMEOUT = "spring.datasource.validationQueryTimeout"; |
||||||
|
|
||||||
|
public static final String SPRING_DATASOURCE_INITIAL_SIZE = "spring.datasource.initialSize"; |
||||||
|
|
||||||
|
public static final String SPRING_DATASOURCE_MIN_IDLE = "spring.datasource.minIdle"; |
||||||
|
|
||||||
|
public static final String SPRING_DATASOURCE_MAX_ACTIVE = "spring.datasource.maxActive"; |
||||||
|
|
||||||
|
public static final String SPRING_DATASOURCE_MAX_WAIT = "spring.datasource.maxWait"; |
||||||
|
|
||||||
|
public static final String SPRING_DATASOURCE_TIME_BETWEEN_EVICTION_RUNS_MILLIS = "spring.datasource.timeBetweenEvictionRunsMillis"; |
||||||
|
|
||||||
|
public static final String SPRING_DATASOURCE_TIME_BETWEEN_CONNECT_ERROR_MILLIS = "spring.datasource.timeBetweenConnectErrorMillis"; |
||||||
|
|
||||||
|
public static final String SPRING_DATASOURCE_MIN_EVICTABLE_IDLE_TIME_MILLIS = "spring.datasource.minEvictableIdleTimeMillis"; |
||||||
|
|
||||||
|
public static final String SPRING_DATASOURCE_VALIDATION_QUERY = "spring.datasource.validationQuery"; |
||||||
|
|
||||||
|
public static final String SPRING_DATASOURCE_TEST_WHILE_IDLE = "spring.datasource.testWhileIdle"; |
||||||
|
|
||||||
|
public static final String SPRING_DATASOURCE_TEST_ON_BORROW = "spring.datasource.testOnBorrow"; |
||||||
|
|
||||||
|
public static final String SPRING_DATASOURCE_TEST_ON_RETURN = "spring.datasource.testOnReturn"; |
||||||
|
|
||||||
|
public static final String SPRING_DATASOURCE_POOL_PREPARED_STATEMENTS = "spring.datasource.poolPreparedStatements"; |
||||||
|
|
||||||
|
public static final String SPRING_DATASOURCE_DEFAULT_AUTO_COMMIT = "spring.datasource.defaultAutoCommit"; |
||||||
|
|
||||||
|
public static final String SPRING_DATASOURCE_KEEP_ALIVE = "spring.datasource.keepAlive"; |
||||||
|
|
||||||
|
public static final String SPRING_DATASOURCE_MAX_POOL_PREPARED_STATEMENT_PER_CONNECTION_SIZE = "spring.datasource.maxPoolPreparedStatementPerConnectionSize"; |
||||||
|
|
||||||
|
public static final String DEVELOPMENT = "development"; |
||||||
|
|
||||||
|
public static final String QUARTZ_PROPERTIES_PATH = "quartz.properties"; |
||||||
|
|
||||||
|
/** |
||||||
|
* sleep time |
||||||
|
*/ |
||||||
|
public static final int SLEEP_TIME_MILLIS = 1000; |
||||||
|
|
||||||
|
/** |
||||||
|
* heartbeat for zk info length |
||||||
|
*/ |
||||||
|
public static final int HEARTBEAT_FOR_ZOOKEEPER_INFO_LENGTH = 6; |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* hadoop params constant |
||||||
|
*/ |
||||||
|
/** |
||||||
|
* jar |
||||||
|
*/ |
||||||
|
public static final String JAR = "jar"; |
||||||
|
|
||||||
|
/** |
||||||
|
* hadoop |
||||||
|
*/ |
||||||
|
public static final String HADOOP = "hadoop"; |
||||||
|
|
||||||
|
/** |
||||||
|
* -D parameter |
||||||
|
*/ |
||||||
|
public static final String D = "-D"; |
||||||
|
|
||||||
|
/** |
||||||
|
* -D mapreduce.job.queuename=ququename |
||||||
|
*/ |
||||||
|
public static final String MR_QUEUE = "mapreduce.job.queuename"; |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* jdbc class name |
||||||
|
*/ |
||||||
|
/** |
||||||
|
* mysql |
||||||
|
*/ |
||||||
|
public static final String JDBC_MYSQL_CLASS_NAME = "com.mysql.jdbc.Driver"; |
||||||
|
|
||||||
|
/** |
||||||
|
* postgresql |
||||||
|
*/ |
||||||
|
public static final String JDBC_POSTGRESQL_CLASS_NAME = "org.postgresql.Driver"; |
||||||
|
|
||||||
|
/** |
||||||
|
* postgresql |
||||||
|
*/ |
||||||
|
public static final String JDBC_HIVE_CLASS_NAME = "org.apache.hive.jdbc.HiveDriver"; |
||||||
|
|
||||||
|
/** |
||||||
|
* postgresql |
||||||
|
*/ |
||||||
|
public static final String JDBC_SPARK_CLASS_NAME = "org.apache.hive.jdbc.HiveDriver"; |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* spark params constant |
||||||
|
*/ |
||||||
|
public static final String MASTER = "--master"; |
||||||
|
|
||||||
|
public static final String DEPLOY_MODE = "--deploy-mode"; |
||||||
|
|
||||||
|
/** |
||||||
|
* --class CLASS_NAME |
||||||
|
*/ |
||||||
|
public static final String CLASS = "--class"; |
||||||
|
|
||||||
|
/** |
||||||
|
* --driver-cores NUM |
||||||
|
*/ |
||||||
|
public static final String DRIVER_CORES = "--driver-cores"; |
||||||
|
|
||||||
|
/** |
||||||
|
* --driver-memory MEM |
||||||
|
*/ |
||||||
|
public static final String DRIVER_MEMORY = "--driver-memory"; |
||||||
|
|
||||||
|
/** |
||||||
|
* --num-executors NUM |
||||||
|
*/ |
||||||
|
public static final String NUM_EXECUTORS = "--num-executors"; |
||||||
|
|
||||||
|
/** |
||||||
|
* --executor-cores NUM |
||||||
|
*/ |
||||||
|
public static final String EXECUTOR_CORES = "--executor-cores"; |
||||||
|
|
||||||
|
/** |
||||||
|
* --executor-memory MEM |
||||||
|
*/ |
||||||
|
public static final String EXECUTOR_MEMORY = "--executor-memory"; |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* --queue QUEUE |
||||||
|
*/ |
||||||
|
public static final String SPARK_QUEUE = "--queue"; |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* exit code success |
||||||
|
*/ |
||||||
|
public static final int EXIT_CODE_SUCCESS = 0; |
||||||
|
|
||||||
|
/** |
||||||
|
* exit code kill |
||||||
|
*/ |
||||||
|
public static final int EXIT_CODE_KILL = 137; |
||||||
|
|
||||||
|
/** |
||||||
|
* exit code failure |
||||||
|
*/ |
||||||
|
public static final int EXIT_CODE_FAILURE = -1; |
||||||
|
|
||||||
|
/** |
||||||
|
* date format of yyyyMMdd |
||||||
|
*/ |
||||||
|
public static final String PARAMETER_FORMAT_DATE = "yyyyMMdd"; |
||||||
|
|
||||||
|
/** |
||||||
|
* date format of yyyyMMddHHmmss |
||||||
|
*/ |
||||||
|
public static final String PARAMETER_FORMAT_TIME = "yyyyMMddHHmmss"; |
||||||
|
|
||||||
|
/** |
||||||
|
* system date(yyyyMMddHHmmss) |
||||||
|
*/ |
||||||
|
public static final String PARAMETER_DATETIME = "system.datetime"; |
||||||
|
|
||||||
|
/** |
||||||
|
* system date(yyyymmdd) today |
||||||
|
*/ |
||||||
|
public static final String PARAMETER_CURRENT_DATE = "system.biz.curdate"; |
||||||
|
|
||||||
|
/** |
||||||
|
* system date(yyyymmdd) yesterday |
||||||
|
*/ |
||||||
|
public static final String PARAMETER_BUSINESS_DATE = "system.biz.date"; |
||||||
|
|
||||||
|
/** |
||||||
|
* ACCEPTED |
||||||
|
*/ |
||||||
|
public static final String ACCEPTED = "ACCEPTED"; |
||||||
|
|
||||||
|
/** |
||||||
|
* SUCCEEDED |
||||||
|
*/ |
||||||
|
public static final String SUCCEEDED = "SUCCEEDED"; |
||||||
|
/** |
||||||
|
* NEW |
||||||
|
*/ |
||||||
|
public static final String NEW = "NEW"; |
||||||
|
/** |
||||||
|
* NEW_SAVING |
||||||
|
*/ |
||||||
|
public static final String NEW_SAVING = "NEW_SAVING"; |
||||||
|
/** |
||||||
|
* SUBMITTED |
||||||
|
*/ |
||||||
|
public static final String SUBMITTED = "SUBMITTED"; |
||||||
|
/** |
||||||
|
* FAILED |
||||||
|
*/ |
||||||
|
public static final String FAILED = "FAILED"; |
||||||
|
/** |
||||||
|
* KILLED |
||||||
|
*/ |
||||||
|
public static final String KILLED = "KILLED"; |
||||||
|
/** |
||||||
|
* RUNNING |
||||||
|
*/ |
||||||
|
public static final String RUNNING = "RUNNING"; |
||||||
|
/** |
||||||
|
* underline "_" |
||||||
|
*/ |
||||||
|
public static final String UNDERLINE = "_"; |
||||||
|
/** |
||||||
|
* quartz job prifix |
||||||
|
*/ |
||||||
|
public static final String QUARTZ_JOB_PRIFIX = "job"; |
||||||
|
/** |
||||||
|
* quartz job group prifix |
||||||
|
*/ |
||||||
|
public static final String QUARTZ_JOB_GROUP_PRIFIX = "jobgroup"; |
||||||
|
/** |
||||||
|
* projectId |
||||||
|
*/ |
||||||
|
public static final String PROJECT_ID = "projectId"; |
||||||
|
/** |
||||||
|
* processId |
||||||
|
*/ |
||||||
|
public static final String SCHEDULE_ID = "scheduleId"; |
||||||
|
/** |
||||||
|
* schedule |
||||||
|
*/ |
||||||
|
public static final String SCHEDULE = "schedule"; |
||||||
|
/** |
||||||
|
* application regex |
||||||
|
*/ |
||||||
|
public static final String APPLICATION_REGEX = "application_\\d+_\\d+"; |
||||||
|
public static final String PID = "pid"; |
||||||
|
/** |
||||||
|
* month_begin |
||||||
|
*/ |
||||||
|
public static final String MONTH_BEGIN = "month_begin"; |
||||||
|
/** |
||||||
|
* add_months |
||||||
|
*/ |
||||||
|
public static final String ADD_MONTHS = "add_months"; |
||||||
|
/** |
||||||
|
* month_end |
||||||
|
*/ |
||||||
|
public static final String MONTH_END = "month_end"; |
||||||
|
/** |
||||||
|
* week_begin |
||||||
|
*/ |
||||||
|
public static final String WEEK_BEGIN = "week_begin"; |
||||||
|
/** |
||||||
|
* week_end |
||||||
|
*/ |
||||||
|
public static final String WEEK_END = "week_end"; |
||||||
|
/** |
||||||
|
* timestamp |
||||||
|
*/ |
||||||
|
public static final String TIMESTAMP = "timestamp"; |
||||||
|
public static final char SUBTRACT_CHAR = '-'; |
||||||
|
public static final char ADD_CHAR = '+'; |
||||||
|
public static final char MULTIPLY_CHAR = '*'; |
||||||
|
public static final char DIVISION_CHAR = '/'; |
||||||
|
public static final char LEFT_BRACE_CHAR = '('; |
||||||
|
public static final char RIGHT_BRACE_CHAR = ')'; |
||||||
|
public static final String ADD_STRING = "+"; |
||||||
|
public static final String MULTIPLY_STRING = "*"; |
||||||
|
public static final String DIVISION_STRING = "/"; |
||||||
|
public static final String LEFT_BRACE_STRING = "("; |
||||||
|
public static final char P = 'P'; |
||||||
|
public static final char N = 'N'; |
||||||
|
public static final String SUBTRACT_STRING = "-"; |
||||||
|
public static final String GLOBAL_PARAMS = "globalParams"; |
||||||
|
public static final String LOCAL_PARAMS = "localParams"; |
||||||
|
public static final String PROCESS_INSTANCE_STATE = "processInstanceState"; |
||||||
|
public static final String TASK_LIST = "taskList"; |
||||||
|
public static final String RWXR_XR_X = "rwxr-xr-x"; |
||||||
|
|
||||||
|
/** |
||||||
|
* master/worker server use for zk |
||||||
|
*/ |
||||||
|
public static final String MASTER_PREFIX = "master"; |
||||||
|
public static final String WORKER_PREFIX = "worker"; |
||||||
|
public static final String DELETE_ZK_OP = "delete"; |
||||||
|
public static final String ADD_ZK_OP = "add"; |
||||||
|
public static final String ALIAS = "alias"; |
||||||
|
public static final String CONTENT = "content"; |
||||||
|
public static final String DEPENDENT_SPLIT = ":||"; |
||||||
|
public static final String DEPENDENT_ALL = "ALL"; |
||||||
|
} |
@ -0,0 +1,29 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common; |
||||||
|
|
||||||
|
/** |
||||||
|
* server stop interface. |
||||||
|
*/ |
||||||
|
public interface IStoppable { |
||||||
|
/** |
||||||
|
* Stop this service. |
||||||
|
* @param cause why stopping |
||||||
|
*/ |
||||||
|
public void stop(String cause); |
||||||
|
|
||||||
|
} |
@ -0,0 +1,27 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.enums; |
||||||
|
|
||||||
|
/** |
||||||
|
* alert status |
||||||
|
*/ |
||||||
|
public enum AlertStatus { |
||||||
|
/** |
||||||
|
* 0 waiting executed; 1 execute successfully,2 execute failed |
||||||
|
*/ |
||||||
|
WAIT_EXECUTION,EXECUTION_SUCCESS,EXECUTION_FAILURE |
||||||
|
} |
@ -0,0 +1,27 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.enums; |
||||||
|
|
||||||
|
/** |
||||||
|
* warning message notification method |
||||||
|
*/ |
||||||
|
public enum AlertType { |
||||||
|
/** |
||||||
|
* 0 email; 1 SMS |
||||||
|
*/ |
||||||
|
EMAIL,SMS |
||||||
|
} |
@ -0,0 +1,40 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.enums; |
||||||
|
|
||||||
|
/** |
||||||
|
* command types |
||||||
|
*/ |
||||||
|
public enum CommandType { |
||||||
|
|
||||||
|
/** |
||||||
|
* command types |
||||||
|
* 0 start a new process |
||||||
|
* 1 start a new process from current nodes |
||||||
|
* 2 recover tolerance fault work flow |
||||||
|
* 3 start process from paused task nodes |
||||||
|
* 4 start process from failure task nodes |
||||||
|
* 5 complement data |
||||||
|
* 6 start a new process from scheduler |
||||||
|
* 7 repeat running a work flow |
||||||
|
* 8 pause a process |
||||||
|
* 9 stop a process |
||||||
|
* 10 recover waiting thread |
||||||
|
*/ |
||||||
|
START_PROCESS, START_CURRENT_TASK_PROCESS, RECOVER_TOLERANCE_FAULT_PROCESS, RECOVER_SUSPENDED_PROCESS, |
||||||
|
START_FAILURE_TASK_PROCESS,COMPLEMENT_DATA,SCHEDULER, REPEAT_RUNNING,PAUSE,STOP,RECOVER_WAITTING_THREAD; |
||||||
|
} |
@ -0,0 +1,28 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.enums; |
||||||
|
|
||||||
|
/** |
||||||
|
* cycle enums |
||||||
|
*/ |
||||||
|
public enum CycleEnum { |
||||||
|
/** |
||||||
|
* 0 minute; 1 hour; 2 day; 3 week; 4 month; 5 year; |
||||||
|
*/ |
||||||
|
MINUTE, HOUR, DAY, WEEK, MONTH, YEAR |
||||||
|
|
||||||
|
} |
@ -0,0 +1,35 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.enums; |
||||||
|
|
||||||
|
/** |
||||||
|
* data types in user define parameter |
||||||
|
*/ |
||||||
|
public enum DataType { |
||||||
|
/** |
||||||
|
* 0 string |
||||||
|
* 1 integer |
||||||
|
* 2 long |
||||||
|
* 3 float |
||||||
|
* 4 double |
||||||
|
* 5 date, "YYYY-MM-DD" |
||||||
|
* 6 time, "HH:MM:SS" |
||||||
|
* 7 time stamp |
||||||
|
* 8 Boolean |
||||||
|
*/ |
||||||
|
VARCHAR,INTEGER,LONG,FLOAT,DOUBLE,DATE,TIME,TIMESTAMP,BOOLEAN |
||||||
|
} |
@ -0,0 +1,30 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.enums; |
||||||
|
|
||||||
|
/** |
||||||
|
* data base types |
||||||
|
*/ |
||||||
|
public enum DbType { |
||||||
|
/** |
||||||
|
* 0 mysql |
||||||
|
* 1 postgresql |
||||||
|
* 2 hive |
||||||
|
* 3 spark |
||||||
|
*/ |
||||||
|
MYSQL, POSTGRESQL, HIVE, SPARK |
||||||
|
} |
@ -0,0 +1,31 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.enums; |
||||||
|
|
||||||
|
/** |
||||||
|
* depend result |
||||||
|
*/ |
||||||
|
public enum DependResult { |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* 0 success |
||||||
|
* 1 waiting |
||||||
|
* 2 failed |
||||||
|
*/ |
||||||
|
SUCCESS, WAITING, FAILED |
||||||
|
} |
@ -0,0 +1,29 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.enums; |
||||||
|
|
||||||
|
/** |
||||||
|
* depend strategy |
||||||
|
*/ |
||||||
|
public enum DependStrategy { |
||||||
|
|
||||||
|
/** |
||||||
|
* 0 none,1 all success 2 all failed 3 one success 4 one failed |
||||||
|
*/ |
||||||
|
NONE, ALL_SUCCESS, ALL_FAILED, ONE_SUCCESS, ONE_FAILED |
||||||
|
|
||||||
|
} |
@ -0,0 +1,25 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.enums; |
||||||
|
|
||||||
|
/** |
||||||
|
* dependent relation: and or |
||||||
|
*/ |
||||||
|
public enum DependentRelation { |
||||||
|
|
||||||
|
AND,OR; |
||||||
|
} |
@ -0,0 +1,27 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.enums; |
||||||
|
|
||||||
|
/** |
||||||
|
* parameter of stored procedure |
||||||
|
*/ |
||||||
|
public enum Direct { |
||||||
|
/** |
||||||
|
* 0 in; 1 out; |
||||||
|
*/ |
||||||
|
IN,OUT |
||||||
|
} |
@ -0,0 +1,101 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.enums; |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* runing status for work flow and task nodes |
||||||
|
* |
||||||
|
*/ |
||||||
|
public enum ExecutionStatus { |
||||||
|
|
||||||
|
/** |
||||||
|
* status: |
||||||
|
* 0 submit success |
||||||
|
* 1 running |
||||||
|
* 2 ready pause |
||||||
|
* 3 pause |
||||||
|
* 4 ready stop |
||||||
|
* 5 stop |
||||||
|
* 6 failure |
||||||
|
* 7 success |
||||||
|
* 8 need fault tolerance |
||||||
|
* 9 kill |
||||||
|
* 10 waiting thread |
||||||
|
* 11 waiting depend node complete |
||||||
|
*/ |
||||||
|
SUBMITTED_SUCCESS,RUNNING_EXEUTION,READY_PAUSE,PAUSE,READY_STOP,STOP,FAILURE,SUCCESS, |
||||||
|
NEED_FAULT_TOLERANCE,KILL,WAITTING_THREAD,WAITTING_DEPEND; |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* status is success |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public boolean typeIsSuccess(){ |
||||||
|
return this == SUCCESS; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* status is failure |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public boolean typeIsFailure(){ |
||||||
|
return this == FAILURE || this == NEED_FAULT_TOLERANCE; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* status is finished |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public boolean typeIsFinished(){ |
||||||
|
|
||||||
|
return typeIsSuccess() || typeIsFailure() || typeIsCancel() || typeIsPause() |
||||||
|
|| typeIsWaittingThread(); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* status is waiting thread |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public boolean typeIsWaittingThread(){ |
||||||
|
return this == WAITTING_THREAD; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* status is pause |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public boolean typeIsPause(){ |
||||||
|
return this == PAUSE; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* status is running |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public boolean typeIsRunning(){ |
||||||
|
return this == RUNNING_EXEUTION || this == WAITTING_DEPEND; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* status is cancel |
||||||
|
*/ |
||||||
|
public boolean typeIsCancel(){ return this == KILL || this == STOP ;} |
||||||
|
|
||||||
|
|
||||||
|
} |
@ -0,0 +1,30 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.enums; |
||||||
|
|
||||||
|
/** |
||||||
|
* failure policy when some task node failed. |
||||||
|
*/ |
||||||
|
public enum FailureStrategy { |
||||||
|
|
||||||
|
/** |
||||||
|
* 0 ending process when some tasks failed. |
||||||
|
* 1 continue running when some tasks failed. |
||||||
|
**/ |
||||||
|
END, CONTINUE; |
||||||
|
|
||||||
|
} |
@ -0,0 +1,33 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.enums; |
||||||
|
|
||||||
|
/** |
||||||
|
* have_script |
||||||
|
* have_file |
||||||
|
* can_retry |
||||||
|
* have_arr_variables |
||||||
|
* have_map_variables |
||||||
|
* have_alert |
||||||
|
*/ |
||||||
|
public enum Flag { |
||||||
|
/** |
||||||
|
* 0 no |
||||||
|
* 1 yes |
||||||
|
*/ |
||||||
|
NO,YES |
||||||
|
} |
@ -0,0 +1,31 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.enums; |
||||||
|
|
||||||
|
/** |
||||||
|
* define process and task priority |
||||||
|
*/ |
||||||
|
public enum Priority { |
||||||
|
/** |
||||||
|
* 0 highest priority |
||||||
|
* 1 higher priority |
||||||
|
* 2 medium priority |
||||||
|
* 3 lower priority |
||||||
|
* 4 lowest priority |
||||||
|
*/ |
||||||
|
HIGHEST,HIGH,MEDIUM,LOW,LOWEST |
||||||
|
} |
@ -0,0 +1,30 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.enums; |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* support program types |
||||||
|
*/ |
||||||
|
public enum ProgramType { |
||||||
|
/** |
||||||
|
* 0 JAVA,1 SCALA,2 PYTHON |
||||||
|
*/ |
||||||
|
JAVA, |
||||||
|
SCALA, |
||||||
|
PYTHON |
||||||
|
} |
@ -0,0 +1,40 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.enums; |
||||||
|
|
||||||
|
/** |
||||||
|
* process define release state |
||||||
|
*/ |
||||||
|
public enum ReleaseState { |
||||||
|
|
||||||
|
/** |
||||||
|
* 0 offline |
||||||
|
* 1 on line |
||||||
|
*/ |
||||||
|
OFFLINE,ONLINE; |
||||||
|
|
||||||
|
|
||||||
|
public static ReleaseState getEnum(int value){ |
||||||
|
for (ReleaseState e:ReleaseState.values()) { |
||||||
|
if(e.ordinal() == value) { |
||||||
|
return e; |
||||||
|
} |
||||||
|
} |
||||||
|
//For values out of enum scope
|
||||||
|
return null; |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,27 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.enums; |
||||||
|
|
||||||
|
/** |
||||||
|
* resource type |
||||||
|
*/ |
||||||
|
public enum ResourceType { |
||||||
|
/** |
||||||
|
* 0 file, 1 udf |
||||||
|
*/ |
||||||
|
FILE,UDF |
||||||
|
} |
@ -0,0 +1,28 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.enums; |
||||||
|
|
||||||
|
/** |
||||||
|
* complement data run mode |
||||||
|
*/ |
||||||
|
public enum RunMode { |
||||||
|
/** |
||||||
|
* 0 serial run |
||||||
|
* 1 parallel run |
||||||
|
* */ |
||||||
|
RUN_MODE_SERIAL, RUN_MODE_PARALLEL |
||||||
|
} |
@ -0,0 +1,29 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.enums; |
||||||
|
|
||||||
|
/** |
||||||
|
* self depency strategy |
||||||
|
*/ |
||||||
|
public enum SelfDependStrategy { |
||||||
|
|
||||||
|
/** |
||||||
|
* 0 donot depend the last cycle; |
||||||
|
* 1 depend the last cycle |
||||||
|
**/ |
||||||
|
NO_DEP_PRE, DEP_PRE |
||||||
|
} |
@ -0,0 +1,34 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.enums; |
||||||
|
|
||||||
|
/** |
||||||
|
* show type for email |
||||||
|
*/ |
||||||
|
public enum ShowType { |
||||||
|
/** |
||||||
|
* 0 TABLE; |
||||||
|
* 1 TEXT; |
||||||
|
* 2 attachment; |
||||||
|
* 3 TABLE+attachment; |
||||||
|
*/ |
||||||
|
TABLE, |
||||||
|
TEXT, |
||||||
|
ATTACHMENT, |
||||||
|
TABLEATTACHMENT |
||||||
|
|
||||||
|
} |
@ -0,0 +1,30 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.enums; |
||||||
|
|
||||||
|
/** |
||||||
|
* task node depend type |
||||||
|
*/ |
||||||
|
public enum TaskDependType { |
||||||
|
/** |
||||||
|
* 0 run current tasks only |
||||||
|
* 1 run current tasks and previous tasks |
||||||
|
* 2 run current tasks and the other tasks that depend on current tasks; |
||||||
|
*/ |
||||||
|
TASK_ONLY, TASK_PRE, TASK_POST; |
||||||
|
|
||||||
|
} |
@ -0,0 +1,67 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.enums; |
||||||
|
|
||||||
|
/** |
||||||
|
* type of task state |
||||||
|
*/ |
||||||
|
public enum TaskStateType { |
||||||
|
/** |
||||||
|
* 0 waiting running |
||||||
|
* 1 running |
||||||
|
* 2 finish |
||||||
|
* 3 failed |
||||||
|
* 4 success |
||||||
|
*/ |
||||||
|
WAITTING, RUNNING, FINISH, FAILED, SUCCESS; |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* convert task state to execute status integer array ; |
||||||
|
* @param taskStateType |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static int[] convert2ExecutStatusIntArray(TaskStateType taskStateType){ |
||||||
|
|
||||||
|
switch (taskStateType){ |
||||||
|
case SUCCESS: |
||||||
|
return new int[]{ExecutionStatus.SUCCESS.ordinal()}; |
||||||
|
case FAILED: |
||||||
|
return new int[]{ |
||||||
|
ExecutionStatus.FAILURE.ordinal(), |
||||||
|
ExecutionStatus.NEED_FAULT_TOLERANCE.ordinal()}; |
||||||
|
case FINISH: |
||||||
|
return new int[]{ |
||||||
|
ExecutionStatus.PAUSE.ordinal(), |
||||||
|
ExecutionStatus.STOP.ordinal() |
||||||
|
}; |
||||||
|
case RUNNING: |
||||||
|
return new int[]{ExecutionStatus.SUBMITTED_SUCCESS.ordinal(), |
||||||
|
ExecutionStatus.RUNNING_EXEUTION.ordinal(), |
||||||
|
ExecutionStatus.READY_PAUSE.ordinal(), |
||||||
|
ExecutionStatus.READY_STOP.ordinal()}; |
||||||
|
case WAITTING: |
||||||
|
return new int[]{ |
||||||
|
ExecutionStatus.SUBMITTED_SUCCESS.ordinal() |
||||||
|
}; |
||||||
|
default: |
||||||
|
break; |
||||||
|
} |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
} |
@ -0,0 +1,29 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.enums; |
||||||
|
|
||||||
|
/** |
||||||
|
* task timeout strategy |
||||||
|
*/ |
||||||
|
public enum TaskTimeoutStrategy { |
||||||
|
/** |
||||||
|
* 0 warn |
||||||
|
* 1 failed |
||||||
|
* 2 warn+failed |
||||||
|
*/ |
||||||
|
WARN, FAILED, WARNFAILED |
||||||
|
} |
@ -0,0 +1,34 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.enums; |
||||||
|
|
||||||
|
/** |
||||||
|
* task node type |
||||||
|
*/ |
||||||
|
public enum TaskType { |
||||||
|
/** |
||||||
|
* 0 SHELL |
||||||
|
* 1 SQL |
||||||
|
* 2 SUB_PROCESS |
||||||
|
* 3 PROCEDURE |
||||||
|
* 4 MR |
||||||
|
* 5 SPARK |
||||||
|
* 6 PYTHON |
||||||
|
* 7 DEPENDENT |
||||||
|
*/ |
||||||
|
SHELL,SQL, SUB_PROCESS,PROCEDURE,MR,SPARK,PYTHON,DEPENDENT |
||||||
|
} |
@ -0,0 +1,27 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.enums; |
||||||
|
|
||||||
|
/** |
||||||
|
* UDF type |
||||||
|
*/ |
||||||
|
public enum UdfType { |
||||||
|
/** |
||||||
|
* 0 hive; 1 spark |
||||||
|
*/ |
||||||
|
HIVE, SPARK |
||||||
|
} |
@ -0,0 +1,28 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.enums; |
||||||
|
|
||||||
|
/** |
||||||
|
* user type |
||||||
|
*/ |
||||||
|
public enum UserType { |
||||||
|
/** |
||||||
|
* 0 admin user; 1 general user |
||||||
|
*/ |
||||||
|
ADMIN_USER, |
||||||
|
GENERAL_USER |
||||||
|
} |
@ -0,0 +1,31 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.enums; |
||||||
|
|
||||||
|
/** |
||||||
|
* types for whether to send warning when process ending; |
||||||
|
*/ |
||||||
|
public enum WarningType { |
||||||
|
/** |
||||||
|
* 0 do not send warning; |
||||||
|
* 1 send if process success; |
||||||
|
* 2 send if process failed; |
||||||
|
* 3 send if process ending; |
||||||
|
*/ |
||||||
|
NONE, SUCCESS, FAILURE, ALL; |
||||||
|
|
||||||
|
} |
@ -0,0 +1,519 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.graph; |
||||||
|
|
||||||
|
import cn.escheduler.common.utils.CollectionUtils; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
import java.util.*; |
||||||
|
import java.util.concurrent.locks.ReadWriteLock; |
||||||
|
import java.util.concurrent.locks.ReentrantReadWriteLock; |
||||||
|
|
||||||
|
/** |
||||||
|
* analysis of DAG |
||||||
|
* Node: node |
||||||
|
* NodeInfo:node description information |
||||||
|
* EdgeInfo: edge description information |
||||||
|
*/ |
||||||
|
public class DAG<Node, NodeInfo, EdgeInfo> { |
||||||
|
|
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(DAG.class); |
||||||
|
|
||||||
|
private final ReadWriteLock lock = new ReentrantReadWriteLock(); |
||||||
|
|
||||||
|
/** |
||||||
|
* node map, key is node, value is node information |
||||||
|
*/ |
||||||
|
private volatile Map<Node, NodeInfo> nodesMap; |
||||||
|
|
||||||
|
/** |
||||||
|
* edge map. key is node of origin;value is Map with key for destination node and value for edge |
||||||
|
*/ |
||||||
|
private volatile Map<Node, Map<Node, EdgeInfo>> edgesMap; |
||||||
|
|
||||||
|
/** |
||||||
|
* reversed edge set,key is node of destination, value is Map with key for origin node and value for edge |
||||||
|
*/ |
||||||
|
private volatile Map<Node, Map<Node, EdgeInfo>> reverseEdgesMap; |
||||||
|
|
||||||
|
|
||||||
|
public DAG() { |
||||||
|
nodesMap = new HashMap<>(); |
||||||
|
edgesMap = new HashMap<>(); |
||||||
|
reverseEdgesMap = new HashMap<>(); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* add node information |
||||||
|
* |
||||||
|
* @param node node |
||||||
|
* @param nodeInfo node information |
||||||
|
*/ |
||||||
|
public void addNode(Node node, NodeInfo nodeInfo) { |
||||||
|
lock.writeLock().lock(); |
||||||
|
|
||||||
|
try{ |
||||||
|
nodesMap.put(node, nodeInfo); |
||||||
|
}finally { |
||||||
|
lock.writeLock().unlock(); |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* add edge |
||||||
|
* @param fromNode node of origin |
||||||
|
* @param toNode node of destination |
||||||
|
* @return The result of adding an edge. returns false if the DAG result is a ring result |
||||||
|
*/ |
||||||
|
public boolean addEdge(Node fromNode, Node toNode) { |
||||||
|
return addEdge(fromNode, toNode, false); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* add edge |
||||||
|
* @param fromNode node of origin |
||||||
|
* @param toNode node of destination |
||||||
|
* @param createNode whether the node needs to be created if it does not exist |
||||||
|
* @return The result of adding an edge. returns false if the DAG result is a ring result |
||||||
|
*/ |
||||||
|
private boolean addEdge(Node fromNode, Node toNode, boolean createNode) { |
||||||
|
return addEdge(fromNode, toNode, null, createNode); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* add edge |
||||||
|
* |
||||||
|
* @param fromNode node of origin |
||||||
|
* @param toNode node of destination |
||||||
|
* @param edge edge description |
||||||
|
* @param createNode whether the node needs to be created if it does not exist |
||||||
|
* @return The result of adding an edge. returns false if the DAG result is a ring result |
||||||
|
*/ |
||||||
|
public boolean addEdge(Node fromNode, Node toNode, EdgeInfo edge, boolean createNode) { |
||||||
|
lock.writeLock().lock(); |
||||||
|
|
||||||
|
try{ |
||||||
|
|
||||||
|
// Whether an edge can be successfully added(fromNode -> toNode)
|
||||||
|
if (!isLegalAddEdge(fromNode, toNode, createNode)) { |
||||||
|
logger.error("serious error: add edge({} -> {}) is invalid, cause cycle!", fromNode, toNode); |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
addNodeIfAbsent(fromNode, null); |
||||||
|
addNodeIfAbsent(toNode, null); |
||||||
|
|
||||||
|
addEdge(fromNode, toNode, edge, edgesMap); |
||||||
|
addEdge(toNode, fromNode, edge, reverseEdgesMap); |
||||||
|
|
||||||
|
return true; |
||||||
|
}finally { |
||||||
|
lock.writeLock().unlock(); |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* whether this node is contained |
||||||
|
* |
||||||
|
* @param node node |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public boolean containsNode(Node node) { |
||||||
|
lock.readLock().lock(); |
||||||
|
|
||||||
|
try{ |
||||||
|
return nodesMap.containsKey(node); |
||||||
|
}finally { |
||||||
|
lock.readLock().unlock(); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* whether this edge is contained |
||||||
|
* |
||||||
|
* @param fromNode node of origin |
||||||
|
* @param toNode node of destination |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public boolean containsEdge(Node fromNode, Node toNode) { |
||||||
|
lock.readLock().lock(); |
||||||
|
try{ |
||||||
|
Map<Node, EdgeInfo> endEdges = edgesMap.get(fromNode); |
||||||
|
if (endEdges == null) { |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
return endEdges.containsKey(toNode); |
||||||
|
}finally { |
||||||
|
lock.readLock().unlock(); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* get node description |
||||||
|
* |
||||||
|
* @param node node |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public NodeInfo getNode(Node node) { |
||||||
|
lock.readLock().lock(); |
||||||
|
|
||||||
|
try{ |
||||||
|
return nodesMap.get(node); |
||||||
|
}finally { |
||||||
|
lock.readLock().unlock(); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* Get the number of nodes |
||||||
|
* |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public int getNodesCount() { |
||||||
|
lock.readLock().lock(); |
||||||
|
|
||||||
|
try{ |
||||||
|
return nodesMap.size(); |
||||||
|
}finally { |
||||||
|
lock.readLock().unlock(); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Get the number of edges |
||||||
|
* |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public int getEdgesCount() { |
||||||
|
lock.readLock().lock(); |
||||||
|
try{ |
||||||
|
int count = 0; |
||||||
|
|
||||||
|
for (Map.Entry<Node, Map<Node, EdgeInfo>> entry : edgesMap.entrySet()) { |
||||||
|
count += entry.getValue().size(); |
||||||
|
} |
||||||
|
|
||||||
|
return count; |
||||||
|
}finally { |
||||||
|
lock.readLock().unlock(); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* get the start node of DAG |
||||||
|
* |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public Collection<Node> getBeginNode() { |
||||||
|
lock.readLock().lock(); |
||||||
|
|
||||||
|
try{ |
||||||
|
return CollectionUtils.subtract(nodesMap.keySet(), reverseEdgesMap.keySet()); |
||||||
|
}finally { |
||||||
|
lock.readLock().unlock(); |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* get the end node of DAG |
||||||
|
* |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public Collection<Node> getEndNode() { |
||||||
|
|
||||||
|
lock.readLock().lock(); |
||||||
|
|
||||||
|
try{ |
||||||
|
return CollectionUtils.subtract(nodesMap.keySet(), edgesMap.keySet()); |
||||||
|
}finally { |
||||||
|
lock.readLock().unlock(); |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* Gets all previous nodes of the node |
||||||
|
* |
||||||
|
* @param node node id to be calculated |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public Set<Node> getPreviousNodes(Node node) { |
||||||
|
lock.readLock().lock(); |
||||||
|
|
||||||
|
try{ |
||||||
|
return getNeighborNodes(node, reverseEdgesMap); |
||||||
|
}finally { |
||||||
|
lock.readLock().unlock(); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* Get all subsequent nodes of the node |
||||||
|
* |
||||||
|
* @param node node id to be calculated |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public Set<Node> getSubsequentNodes(Node node) { |
||||||
|
lock.readLock().lock(); |
||||||
|
|
||||||
|
try{ |
||||||
|
return getNeighborNodes(node, edgesMap); |
||||||
|
}finally { |
||||||
|
lock.readLock().unlock(); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* Gets the degree of entry of the node |
||||||
|
* |
||||||
|
* @param node node id |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public int getIndegree(Node node) { |
||||||
|
lock.readLock().lock(); |
||||||
|
|
||||||
|
try{ |
||||||
|
return getPreviousNodes(node).size(); |
||||||
|
}finally { |
||||||
|
lock.readLock().unlock(); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* whether the graph has a ring |
||||||
|
* |
||||||
|
* @return true if has cycle, else return false. |
||||||
|
*/ |
||||||
|
public boolean hasCycle() { |
||||||
|
lock.readLock().lock(); |
||||||
|
try{ |
||||||
|
return !topologicalSortImpl().getKey(); |
||||||
|
}finally { |
||||||
|
lock.readLock().unlock(); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* Only DAG has a topological sort |
||||||
|
* @return topologically sorted results, returns false if the DAG result is a ring result |
||||||
|
* @throws Exception |
||||||
|
*/ |
||||||
|
public List<Node> topologicalSort() throws Exception { |
||||||
|
lock.readLock().lock(); |
||||||
|
|
||||||
|
try{ |
||||||
|
Map.Entry<Boolean, List<Node>> entry = topologicalSortImpl(); |
||||||
|
|
||||||
|
if (entry.getKey()) { |
||||||
|
return entry.getValue(); |
||||||
|
} |
||||||
|
|
||||||
|
throw new Exception("serious error: graph has cycle ! "); |
||||||
|
}finally { |
||||||
|
lock.readLock().unlock(); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* if tho node does not exist,add this node |
||||||
|
* |
||||||
|
* @param node node |
||||||
|
* @param nodeInfo node information |
||||||
|
*/ |
||||||
|
private void addNodeIfAbsent(Node node, NodeInfo nodeInfo) { |
||||||
|
if (!containsNode(node)) { |
||||||
|
addNode(node, nodeInfo); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* add edge |
||||||
|
* |
||||||
|
* @param fromNode node of origin |
||||||
|
* @param toNode node of destination |
||||||
|
* @param edge edge description |
||||||
|
* @param edges edge set |
||||||
|
*/ |
||||||
|
private void addEdge(Node fromNode, Node toNode, EdgeInfo edge, Map<Node, Map<Node, EdgeInfo>> edges) { |
||||||
|
edges.putIfAbsent(fromNode, new HashMap<>()); |
||||||
|
Map<Node, EdgeInfo> toNodeEdges = edges.get(fromNode); |
||||||
|
toNodeEdges.put(toNode, edge); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* Whether an edge can be successfully added(fromNode -> toNode) |
||||||
|
* need to determine whether the DAG has cycle |
||||||
|
* |
||||||
|
* @param fromNode node of origin |
||||||
|
* @param toNode node of destination |
||||||
|
* @param createNode whether to create a node |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
private boolean isLegalAddEdge(Node fromNode, Node toNode, boolean createNode) { |
||||||
|
if (fromNode.equals(toNode)) { |
||||||
|
logger.error("edge fromNode({}) can't equals toNode({})", fromNode, toNode); |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
if (!createNode) { |
||||||
|
if (!containsNode(fromNode) || !containsNode(toNode)){ |
||||||
|
logger.error("edge fromNode({}) or toNode({}) is not in vertices map", fromNode, toNode); |
||||||
|
return false; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// Whether an edge can be successfully added(fromNode -> toNode),need to determine whether the DAG has cycle!
|
||||||
|
int verticesCount = getNodesCount(); |
||||||
|
|
||||||
|
Queue<Node> queue = new LinkedList<>(); |
||||||
|
|
||||||
|
queue.add(toNode); |
||||||
|
|
||||||
|
// if DAG doesn't find fromNode, it's not has cycle!
|
||||||
|
while (!queue.isEmpty() && (--verticesCount > 0)) { |
||||||
|
Node key = queue.poll(); |
||||||
|
|
||||||
|
for (Node subsequentNode : getSubsequentNodes(key)) { |
||||||
|
if (subsequentNode.equals(fromNode)) { |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
queue.add(subsequentNode); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
return true; |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* Get all neighbor nodes of the node |
||||||
|
* |
||||||
|
* @param node Node id to be calculated |
||||||
|
* @param edges neighbor edge information |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
private Set<Node> getNeighborNodes(Node node, final Map<Node, Map<Node, EdgeInfo>> edges) { |
||||||
|
final Map<Node, EdgeInfo> neighborEdges = edges.get(node); |
||||||
|
|
||||||
|
if (neighborEdges == null) { |
||||||
|
return Collections.EMPTY_MAP.keySet(); |
||||||
|
} |
||||||
|
|
||||||
|
return neighborEdges.keySet(); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* Determine whether there are ring and topological sorting results |
||||||
|
* |
||||||
|
* Directed acyclic graph (DAG) has topological ordering |
||||||
|
* Breadth First Search: |
||||||
|
* 1、Traversal of all the vertices in the graph, the degree of entry is 0 vertex into the queue |
||||||
|
* 2、Poll a vertex in the queue to update its adjacency (minus 1) and queue the adjacency if it is 0 after minus 1 |
||||||
|
* 3、Do step 2 until the queue is empty |
||||||
|
* If you cannot traverse all the nodes, it means that the current graph is not a directed acyclic graph. |
||||||
|
* There is no topological sort. |
||||||
|
* |
||||||
|
* |
||||||
|
* @return key Returns the state |
||||||
|
* if success (acyclic) is true, failure (acyclic) is looped, |
||||||
|
* and value (possibly one of the topological sort results) |
||||||
|
*/ |
||||||
|
private Map.Entry<Boolean, List<Node>> topologicalSortImpl() { |
||||||
|
// node queue with degree of entry 0
|
||||||
|
Queue<Node> zeroIndegreeNodeQueue = new LinkedList<>(); |
||||||
|
// save result
|
||||||
|
List<Node> topoResultList = new ArrayList<>(); |
||||||
|
// save the node whose degree is not 0
|
||||||
|
Map<Node, Integer> notZeroIndegreeNodeMap = new HashMap<>(); |
||||||
|
|
||||||
|
// Scan all the vertices and push vertexs with an entry degree of 0 to queue
|
||||||
|
for (Map.Entry<Node, NodeInfo> vertices : nodesMap.entrySet()) { |
||||||
|
Node node = vertices.getKey(); |
||||||
|
int inDegree = getIndegree(node); |
||||||
|
|
||||||
|
if (inDegree == 0) { |
||||||
|
zeroIndegreeNodeQueue.add(node); |
||||||
|
topoResultList.add(node); |
||||||
|
} else { |
||||||
|
notZeroIndegreeNodeMap.put(node, inDegree); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* After scanning, there is no node with 0 degree of entry, |
||||||
|
* indicating that there is a ring, and return directly |
||||||
|
*/ |
||||||
|
if(zeroIndegreeNodeQueue.isEmpty()){ |
||||||
|
return new AbstractMap.SimpleEntry(false, topoResultList); |
||||||
|
} |
||||||
|
|
||||||
|
// The topology algorithm is used to delete nodes with 0 degree of entry and its associated edges
|
||||||
|
while (!zeroIndegreeNodeQueue.isEmpty()) { |
||||||
|
Node v = zeroIndegreeNodeQueue.poll(); |
||||||
|
// Get the neighbor node
|
||||||
|
Set<Node> subsequentNodes = getSubsequentNodes(v); |
||||||
|
|
||||||
|
for (Node subsequentNode : subsequentNodes) { |
||||||
|
|
||||||
|
Integer degree = notZeroIndegreeNodeMap.get(subsequentNode); |
||||||
|
|
||||||
|
if(--degree == 0){ |
||||||
|
topoResultList.add(subsequentNode); |
||||||
|
zeroIndegreeNodeQueue.add(subsequentNode); |
||||||
|
notZeroIndegreeNodeMap.remove(subsequentNode); |
||||||
|
}else{ |
||||||
|
notZeroIndegreeNodeMap.put(subsequentNode, degree); |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// if notZeroIndegreeNodeMap is empty,there is no ring!
|
||||||
|
AbstractMap.SimpleEntry resultMap = new AbstractMap.SimpleEntry(notZeroIndegreeNodeMap.size() == 0 , topoResultList); |
||||||
|
return resultMap; |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
|
|
@ -0,0 +1,100 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.job.db; |
||||||
|
|
||||||
|
/** |
||||||
|
* data source base class
|
||||||
|
*/ |
||||||
|
public abstract class BaseDataSource { |
||||||
|
/** |
||||||
|
* user name |
||||||
|
*/ |
||||||
|
private String user; |
||||||
|
|
||||||
|
/** |
||||||
|
* user password |
||||||
|
*/ |
||||||
|
private String password; |
||||||
|
|
||||||
|
/** |
||||||
|
* data source address |
||||||
|
*/ |
||||||
|
private String address; |
||||||
|
|
||||||
|
/** |
||||||
|
* database name |
||||||
|
*/ |
||||||
|
private String database; |
||||||
|
|
||||||
|
/** |
||||||
|
* other connection parameters for the data source |
||||||
|
*/ |
||||||
|
private String other; |
||||||
|
|
||||||
|
/** |
||||||
|
* test whether the data source can be connected successfully |
||||||
|
* @throws Exception |
||||||
|
*/ |
||||||
|
public abstract void isConnectable() throws Exception; |
||||||
|
|
||||||
|
/** |
||||||
|
* gets the JDBC url for the data source connection |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public abstract String getJdbcUrl(); |
||||||
|
|
||||||
|
public String getUser() { |
||||||
|
return user; |
||||||
|
} |
||||||
|
|
||||||
|
public void setUser(String user) { |
||||||
|
this.user = user; |
||||||
|
} |
||||||
|
|
||||||
|
public String getPassword() { |
||||||
|
return password; |
||||||
|
} |
||||||
|
|
||||||
|
public void setPassword(String password) { |
||||||
|
this.password = password; |
||||||
|
} |
||||||
|
|
||||||
|
public String getAddress() { |
||||||
|
return address; |
||||||
|
} |
||||||
|
|
||||||
|
public void setAddress(String address) { |
||||||
|
this.address = address; |
||||||
|
} |
||||||
|
|
||||||
|
public String getDatabase() { |
||||||
|
return database; |
||||||
|
} |
||||||
|
|
||||||
|
public void setDatabase(String database) { |
||||||
|
this.database = database; |
||||||
|
} |
||||||
|
|
||||||
|
public String getOther() { |
||||||
|
return other; |
||||||
|
} |
||||||
|
|
||||||
|
public void setOther(String other) { |
||||||
|
this.other = other; |
||||||
|
} |
||||||
|
|
||||||
|
} |
@ -0,0 +1,50 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.job.db; |
||||||
|
|
||||||
|
import cn.escheduler.common.enums.DbType; |
||||||
|
import cn.escheduler.common.utils.JSONUtils; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
/** |
||||||
|
* produce datasource in this custom defined datasource factory. |
||||||
|
*/ |
||||||
|
public class DataSourceFactory { |
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(DataSourceFactory.class); |
||||||
|
|
||||||
|
public static BaseDataSource getDatasource(DbType dbType, String parameter) { |
||||||
|
try { |
||||||
|
switch (dbType) { |
||||||
|
case MYSQL: |
||||||
|
return JSONUtils.parseObject(parameter, MySQLDataSource.class); |
||||||
|
case POSTGRESQL: |
||||||
|
return JSONUtils.parseObject(parameter, PostgreDataSource.class); |
||||||
|
case HIVE: |
||||||
|
return JSONUtils.parseObject(parameter, HiveDataSource.class); |
||||||
|
case SPARK: |
||||||
|
return JSONUtils.parseObject(parameter, SparkDataSource.class); |
||||||
|
default: |
||||||
|
return null; |
||||||
|
} |
||||||
|
} catch (Exception e) { |
||||||
|
logger.error("Get datasource object error", e); |
||||||
|
return null; |
||||||
|
} |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,77 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.job.db; |
||||||
|
|
||||||
|
import org.apache.commons.lang3.StringUtils; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
import java.sql.Connection; |
||||||
|
import java.sql.DriverManager; |
||||||
|
import java.sql.SQLException; |
||||||
|
|
||||||
|
/** |
||||||
|
* data source of hive |
||||||
|
*/ |
||||||
|
public class HiveDataSource extends BaseDataSource { |
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(HiveDataSource.class); |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* gets the JDBC url for the data source connection |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
@Override |
||||||
|
public String getJdbcUrl() { |
||||||
|
String jdbcUrl = getAddress(); |
||||||
|
if (jdbcUrl.lastIndexOf("/") != (jdbcUrl.length() - 1)) { |
||||||
|
jdbcUrl += "/"; |
||||||
|
} |
||||||
|
|
||||||
|
jdbcUrl += getDatabase(); |
||||||
|
|
||||||
|
if (StringUtils.isNotEmpty(getOther())) { |
||||||
|
jdbcUrl += ";" + getOther(); |
||||||
|
} |
||||||
|
|
||||||
|
return jdbcUrl; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* test whether the data source can be connected successfully |
||||||
|
* @throws Exception |
||||||
|
*/ |
||||||
|
@Override |
||||||
|
public void isConnectable() throws Exception { |
||||||
|
Connection con = null; |
||||||
|
try { |
||||||
|
Class.forName("org.apache.hive.jdbc.HiveDriver"); |
||||||
|
con = DriverManager.getConnection(getJdbcUrl(), getUser(), ""); |
||||||
|
} finally { |
||||||
|
if (con != null) { |
||||||
|
try { |
||||||
|
con.close(); |
||||||
|
} catch (SQLException e) { |
||||||
|
logger.error("Postgre datasource try conn close conn error", e); |
||||||
|
throw e; |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
|
} |
@ -0,0 +1,73 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.job.db; |
||||||
|
|
||||||
|
import org.apache.commons.lang3.StringUtils; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
import java.sql.Connection; |
||||||
|
import java.sql.DriverManager; |
||||||
|
import java.sql.SQLException; |
||||||
|
|
||||||
|
/** |
||||||
|
* data source of mySQL |
||||||
|
*/ |
||||||
|
public class MySQLDataSource extends BaseDataSource { |
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(MySQLDataSource.class); |
||||||
|
|
||||||
|
/** |
||||||
|
* gets the JDBC url for the data source connection |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
@Override |
||||||
|
public String getJdbcUrl() { |
||||||
|
String address = getAddress(); |
||||||
|
if (address.lastIndexOf("/") != (address.length() - 1)) { |
||||||
|
address += "/"; |
||||||
|
} |
||||||
|
String jdbcUrl = address + getDatabase(); |
||||||
|
if (StringUtils.isNotEmpty(getOther())) { |
||||||
|
jdbcUrl += "?" + getOther(); |
||||||
|
} |
||||||
|
return jdbcUrl; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* test whether the data source can be connected successfully |
||||||
|
* @throws Exception |
||||||
|
*/ |
||||||
|
@Override |
||||||
|
public void isConnectable() throws Exception { |
||||||
|
Connection con = null; |
||||||
|
try { |
||||||
|
Class.forName("com.mysql.jdbc.Driver"); |
||||||
|
con = DriverManager.getConnection(getJdbcUrl(), getUser(), getPassword()); |
||||||
|
} finally { |
||||||
|
if (con != null) { |
||||||
|
try { |
||||||
|
con.close(); |
||||||
|
} catch (SQLException e) { |
||||||
|
logger.error("Mysql datasource try conn close conn error", e); |
||||||
|
throw e; |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
} |
@ -0,0 +1,77 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.job.db; |
||||||
|
|
||||||
|
import org.apache.commons.lang3.StringUtils; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
import java.sql.Connection; |
||||||
|
import java.sql.DriverManager; |
||||||
|
import java.sql.SQLException; |
||||||
|
|
||||||
|
/** |
||||||
|
* data source of postgreSQL |
||||||
|
*/ |
||||||
|
public class PostgreDataSource extends BaseDataSource { |
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(PostgreDataSource.class); |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* gets the JDBC url for the data source connection |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
@Override |
||||||
|
public String getJdbcUrl() { |
||||||
|
String jdbcUrl = getAddress(); |
||||||
|
if (jdbcUrl.lastIndexOf("/") != (jdbcUrl.length() - 1)) { |
||||||
|
jdbcUrl += "/"; |
||||||
|
} |
||||||
|
|
||||||
|
jdbcUrl += getDatabase(); |
||||||
|
|
||||||
|
if (StringUtils.isNotEmpty(getOther())) { |
||||||
|
jdbcUrl += "?" + getOther(); |
||||||
|
} |
||||||
|
|
||||||
|
return jdbcUrl; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* test whether the data source can be connected successfully |
||||||
|
* @throws Exception |
||||||
|
*/ |
||||||
|
@Override |
||||||
|
public void isConnectable() throws Exception { |
||||||
|
Connection con = null; |
||||||
|
try { |
||||||
|
Class.forName("org.postgresql.Driver"); |
||||||
|
con = DriverManager.getConnection(getJdbcUrl(), getUser(), getPassword()); |
||||||
|
} finally { |
||||||
|
if (con != null) { |
||||||
|
try { |
||||||
|
con.close(); |
||||||
|
} catch (SQLException e) { |
||||||
|
logger.error("Postgre datasource try conn close conn error", e); |
||||||
|
throw e; |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
|
} |
@ -0,0 +1,77 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.job.db; |
||||||
|
|
||||||
|
import org.apache.commons.lang3.StringUtils; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
import java.sql.Connection; |
||||||
|
import java.sql.DriverManager; |
||||||
|
import java.sql.SQLException; |
||||||
|
|
||||||
|
/** |
||||||
|
* data source of spark |
||||||
|
*/ |
||||||
|
public class SparkDataSource extends BaseDataSource { |
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(SparkDataSource.class); |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* gets the JDBC url for the data source connection |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
@Override |
||||||
|
public String getJdbcUrl() { |
||||||
|
String jdbcUrl = getAddress(); |
||||||
|
if (jdbcUrl.lastIndexOf("/") != (jdbcUrl.length() - 1)) { |
||||||
|
jdbcUrl += "/"; |
||||||
|
} |
||||||
|
|
||||||
|
jdbcUrl += getDatabase(); |
||||||
|
|
||||||
|
if (StringUtils.isNotEmpty(getOther())) { |
||||||
|
jdbcUrl += ";" + getOther(); |
||||||
|
} |
||||||
|
|
||||||
|
return jdbcUrl; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* test whether the data source can be connected successfully |
||||||
|
* @throws Exception |
||||||
|
*/ |
||||||
|
@Override |
||||||
|
public void isConnectable() throws Exception { |
||||||
|
Connection con = null; |
||||||
|
try { |
||||||
|
Class.forName("org.apache.hive.jdbc.HiveDriver"); |
||||||
|
con = DriverManager.getConnection(getJdbcUrl(), getUser(), ""); |
||||||
|
} finally { |
||||||
|
if (con != null) { |
||||||
|
try { |
||||||
|
con.close(); |
||||||
|
} catch (SQLException e) { |
||||||
|
logger.error("Spark datasource try conn close conn error", e); |
||||||
|
throw e; |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
|
} |
@ -0,0 +1,62 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.model; |
||||||
|
|
||||||
|
import java.util.Date; |
||||||
|
|
||||||
|
/** |
||||||
|
* date interval class
|
||||||
|
*/ |
||||||
|
public class DateInterval { |
||||||
|
|
||||||
|
private Date startTime; |
||||||
|
|
||||||
|
private Date endTime; |
||||||
|
|
||||||
|
public DateInterval(Date beginTime, Date endTime){ |
||||||
|
this.startTime = beginTime; |
||||||
|
this.endTime = endTime; |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public boolean equals(Object obj) { |
||||||
|
try{ |
||||||
|
DateInterval dateInterval = (DateInterval) obj; |
||||||
|
return startTime.equals(dateInterval.getStartTime()) && |
||||||
|
endTime.equals(dateInterval.getEndTime()); |
||||||
|
}catch (Exception e){ |
||||||
|
return false; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
public Date getStartTime() { |
||||||
|
return startTime; |
||||||
|
} |
||||||
|
|
||||||
|
public void setStartTime(Date startTime) { |
||||||
|
this.startTime = startTime; |
||||||
|
} |
||||||
|
|
||||||
|
public Date getEndTime() { |
||||||
|
return endTime; |
||||||
|
} |
||||||
|
|
||||||
|
public void setEndTime(Date endTime) { |
||||||
|
this.endTime = endTime; |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,80 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.model; |
||||||
|
|
||||||
|
import cn.escheduler.common.enums.DependResult; |
||||||
|
|
||||||
|
/** |
||||||
|
* dependent item |
||||||
|
*/ |
||||||
|
public class DependentItem { |
||||||
|
|
||||||
|
private int definitionId; |
||||||
|
private String depTasks; |
||||||
|
private String cycle; |
||||||
|
private String dateValue; |
||||||
|
private DependResult dependResult; |
||||||
|
|
||||||
|
|
||||||
|
public String getKey(){ |
||||||
|
return String.format("%d-%s-%s-%s", |
||||||
|
getDefinitionId(), |
||||||
|
getDepTasks(), |
||||||
|
getCycle(), |
||||||
|
getDateValue()); |
||||||
|
} |
||||||
|
|
||||||
|
public int getDefinitionId() { |
||||||
|
return definitionId; |
||||||
|
} |
||||||
|
|
||||||
|
public void setDefinitionId(int definitionId) { |
||||||
|
this.definitionId = definitionId; |
||||||
|
} |
||||||
|
|
||||||
|
public String getDepTasks() { |
||||||
|
return depTasks; |
||||||
|
} |
||||||
|
|
||||||
|
public void setDepTasks(String depTasks) { |
||||||
|
this.depTasks = depTasks; |
||||||
|
} |
||||||
|
|
||||||
|
public String getCycle() { |
||||||
|
return cycle; |
||||||
|
} |
||||||
|
|
||||||
|
public void setCycle(String cycle) { |
||||||
|
this.cycle = cycle; |
||||||
|
} |
||||||
|
|
||||||
|
public String getDateValue() { |
||||||
|
return dateValue; |
||||||
|
} |
||||||
|
|
||||||
|
public void setDateValue(String dateValue) { |
||||||
|
this.dateValue = dateValue; |
||||||
|
} |
||||||
|
|
||||||
|
public DependResult getDependResult() { |
||||||
|
return dependResult; |
||||||
|
} |
||||||
|
|
||||||
|
public void setDependResult(DependResult dependResult) { |
||||||
|
this.dependResult = dependResult; |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,44 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.model; |
||||||
|
|
||||||
|
import cn.escheduler.common.enums.DependentRelation; |
||||||
|
|
||||||
|
import java.util.List; |
||||||
|
|
||||||
|
public class DependentTaskModel { |
||||||
|
|
||||||
|
|
||||||
|
private List<DependentItem> dependItemList; |
||||||
|
private DependentRelation relation; |
||||||
|
|
||||||
|
public List<DependentItem> getDependItemList() { |
||||||
|
return dependItemList; |
||||||
|
} |
||||||
|
|
||||||
|
public void setDependItemList(List<DependentItem> dependItemList) { |
||||||
|
this.dependItemList = dependItemList; |
||||||
|
} |
||||||
|
|
||||||
|
public DependentRelation getRelation() { |
||||||
|
return relation; |
||||||
|
} |
||||||
|
|
||||||
|
public void setRelation(DependentRelation relation) { |
||||||
|
this.relation = relation; |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,308 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.model; |
||||||
|
|
||||||
|
import cn.escheduler.common.Constants; |
||||||
|
import cn.escheduler.common.enums.Priority; |
||||||
|
import cn.escheduler.common.enums.TaskTimeoutStrategy; |
||||||
|
import cn.escheduler.common.task.TaskTimeoutParameter; |
||||||
|
import cn.escheduler.common.utils.CollectionUtils; |
||||||
|
import cn.escheduler.common.utils.JSONUtils; |
||||||
|
import com.alibaba.fastjson.JSONObject; |
||||||
|
import com.fasterxml.jackson.core.JsonProcessingException; |
||||||
|
import com.fasterxml.jackson.databind.annotation.JsonDeserialize; |
||||||
|
import com.fasterxml.jackson.databind.annotation.JsonSerialize; |
||||||
|
import org.apache.commons.lang3.StringUtils; |
||||||
|
|
||||||
|
import java.io.IOException; |
||||||
|
import java.util.List; |
||||||
|
import java.util.Objects; |
||||||
|
|
||||||
|
|
||||||
|
public class TaskNode { |
||||||
|
|
||||||
|
/** |
||||||
|
* task node id |
||||||
|
*/ |
||||||
|
private String id; |
||||||
|
|
||||||
|
/** |
||||||
|
* task node name |
||||||
|
*/ |
||||||
|
private String name; |
||||||
|
|
||||||
|
/** |
||||||
|
* task node description |
||||||
|
*/ |
||||||
|
private String desc; |
||||||
|
|
||||||
|
/** |
||||||
|
* task node type |
||||||
|
*/ |
||||||
|
private String type; |
||||||
|
|
||||||
|
/** |
||||||
|
* the run flag has two states, NORMAL or FORBIDDEN |
||||||
|
*/ |
||||||
|
private String runFlag; |
||||||
|
|
||||||
|
/** |
||||||
|
* the front field |
||||||
|
*/ |
||||||
|
private String loc; |
||||||
|
|
||||||
|
/** |
||||||
|
* maximum number of retries |
||||||
|
*/ |
||||||
|
private int maxRetryTimes; |
||||||
|
|
||||||
|
/** |
||||||
|
* Unit of retry interval: points |
||||||
|
*/ |
||||||
|
private int retryInterval; |
||||||
|
|
||||||
|
/** |
||||||
|
* params information |
||||||
|
*/ |
||||||
|
@JsonDeserialize(using = JSONUtils.JsonDataDeserializer.class) |
||||||
|
@JsonSerialize(using = JSONUtils.JsonDataSerializer.class) |
||||||
|
private String params; |
||||||
|
|
||||||
|
/** |
||||||
|
* inner dependency information |
||||||
|
*/ |
||||||
|
@JsonDeserialize(using = JSONUtils.JsonDataDeserializer.class) |
||||||
|
@JsonSerialize(using = JSONUtils.JsonDataSerializer.class) |
||||||
|
private String preTasks; |
||||||
|
|
||||||
|
/** |
||||||
|
* users store additional information |
||||||
|
*/ |
||||||
|
@JsonDeserialize(using = JSONUtils.JsonDataDeserializer.class) |
||||||
|
@JsonSerialize(using = JSONUtils.JsonDataSerializer.class) |
||||||
|
private String extras; |
||||||
|
|
||||||
|
/** |
||||||
|
* node dependency list |
||||||
|
*/ |
||||||
|
private List<String> depList; |
||||||
|
|
||||||
|
/** |
||||||
|
* outer dependency information |
||||||
|
*/ |
||||||
|
@JsonDeserialize(using = JSONUtils.JsonDataDeserializer.class) |
||||||
|
@JsonSerialize(using = JSONUtils.JsonDataSerializer.class) |
||||||
|
private String dependence; |
||||||
|
|
||||||
|
/** |
||||||
|
* task instance priority |
||||||
|
*/ |
||||||
|
private Priority taskInstancePriority; |
||||||
|
|
||||||
|
/** |
||||||
|
* task time out |
||||||
|
*/ |
||||||
|
@JsonDeserialize(using = JSONUtils.JsonDataDeserializer.class) |
||||||
|
@JsonSerialize(using = JSONUtils.JsonDataSerializer.class) |
||||||
|
private String timeout; |
||||||
|
|
||||||
|
public String getId() { |
||||||
|
return id; |
||||||
|
} |
||||||
|
|
||||||
|
public void setId(String id) { |
||||||
|
this.id = id; |
||||||
|
} |
||||||
|
|
||||||
|
public String getName() { |
||||||
|
return name; |
||||||
|
} |
||||||
|
|
||||||
|
public void setName(String name) { |
||||||
|
this.name = name; |
||||||
|
} |
||||||
|
|
||||||
|
public String getDesc() { |
||||||
|
return desc; |
||||||
|
} |
||||||
|
|
||||||
|
public void setDesc(String desc) { |
||||||
|
this.desc = desc; |
||||||
|
} |
||||||
|
|
||||||
|
public String getType() { |
||||||
|
return type; |
||||||
|
} |
||||||
|
|
||||||
|
public void setType(String type) { |
||||||
|
this.type = type; |
||||||
|
} |
||||||
|
|
||||||
|
public String getParams() { |
||||||
|
return params; |
||||||
|
} |
||||||
|
|
||||||
|
public void setParams(String params) { |
||||||
|
this.params = params; |
||||||
|
} |
||||||
|
|
||||||
|
public String getPreTasks() { |
||||||
|
return preTasks; |
||||||
|
} |
||||||
|
|
||||||
|
public void setPreTasks(String preTasks) throws IOException { |
||||||
|
this.preTasks = preTasks; |
||||||
|
this.depList = JSONUtils.toList(preTasks, String.class); |
||||||
|
} |
||||||
|
|
||||||
|
public String getExtras() { |
||||||
|
return extras; |
||||||
|
} |
||||||
|
|
||||||
|
public void setExtras(String extras) { |
||||||
|
this.extras = extras; |
||||||
|
} |
||||||
|
|
||||||
|
public List<String> getDepList() { |
||||||
|
return depList; |
||||||
|
} |
||||||
|
|
||||||
|
public void setDepList(List<String> depList) throws JsonProcessingException { |
||||||
|
this.depList = depList; |
||||||
|
this.preTasks = JSONUtils.toJson(depList); |
||||||
|
} |
||||||
|
|
||||||
|
public String getLoc() { |
||||||
|
return loc; |
||||||
|
} |
||||||
|
|
||||||
|
public void setLoc(String loc) { |
||||||
|
this.loc = loc; |
||||||
|
} |
||||||
|
|
||||||
|
public String getRunFlag(){ |
||||||
|
return runFlag; |
||||||
|
} |
||||||
|
|
||||||
|
public void setRunFlag(String runFlag) { |
||||||
|
this.runFlag = runFlag; |
||||||
|
} |
||||||
|
|
||||||
|
public Boolean isForbidden(){ |
||||||
|
return (StringUtils.isNotEmpty(this.runFlag) && |
||||||
|
this.runFlag.equals(Constants.FLOWNODE_RUN_FLAG_FORBIDDEN)); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public boolean equals(Object o) { |
||||||
|
if (this == o) { |
||||||
|
return true; |
||||||
|
} |
||||||
|
if (o == null || getClass() != o.getClass()) { |
||||||
|
return false; |
||||||
|
} |
||||||
|
TaskNode taskNode = (TaskNode) o; |
||||||
|
return Objects.equals(name, taskNode.name) && |
||||||
|
Objects.equals(desc, taskNode.desc) && |
||||||
|
Objects.equals(type, taskNode.type) && |
||||||
|
Objects.equals(params, taskNode.params) && |
||||||
|
Objects.equals(preTasks, taskNode.preTasks) && |
||||||
|
Objects.equals(extras, taskNode.extras) && |
||||||
|
Objects.equals(runFlag, taskNode.runFlag) && |
||||||
|
Objects.equals(dependence, taskNode.dependence) && |
||||||
|
CollectionUtils.equalLists(depList, taskNode.depList); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public int hashCode() { |
||||||
|
return Objects.hash(name, desc, type, params, preTasks, extras, depList, runFlag); |
||||||
|
} |
||||||
|
|
||||||
|
public String getDependence() { |
||||||
|
return dependence; |
||||||
|
} |
||||||
|
|
||||||
|
public void setDependence(String dependence) { |
||||||
|
this.dependence = dependence; |
||||||
|
} |
||||||
|
|
||||||
|
public int getMaxRetryTimes() { |
||||||
|
return maxRetryTimes; |
||||||
|
} |
||||||
|
|
||||||
|
public void setMaxRetryTimes(int maxRetryTimes) { |
||||||
|
this.maxRetryTimes = maxRetryTimes; |
||||||
|
} |
||||||
|
|
||||||
|
public int getRetryInterval() { |
||||||
|
return retryInterval; |
||||||
|
} |
||||||
|
|
||||||
|
public void setRetryInterval(int retryInterval) { |
||||||
|
this.retryInterval = retryInterval; |
||||||
|
} |
||||||
|
|
||||||
|
public Priority getTaskInstancePriority() { |
||||||
|
return taskInstancePriority; |
||||||
|
} |
||||||
|
|
||||||
|
public void setTaskInstancePriority(Priority taskInstancePriority) { |
||||||
|
this.taskInstancePriority = taskInstancePriority; |
||||||
|
} |
||||||
|
|
||||||
|
public String getTimeout() { |
||||||
|
return timeout; |
||||||
|
} |
||||||
|
|
||||||
|
public void setTimeout(String timeout) { |
||||||
|
this.timeout = timeout; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get task time out parameter |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public TaskTimeoutParameter getTaskTimeoutParameter() { |
||||||
|
if(StringUtils.isNotEmpty(this.getTimeout())){ |
||||||
|
String formatStr = String.format("%s,%s", TaskTimeoutStrategy.WARN.name(), TaskTimeoutStrategy.FAILED.name()); |
||||||
|
String timeout = this.getTimeout().replace(formatStr,TaskTimeoutStrategy.WARNFAILED.name()); |
||||||
|
return JSONObject.parseObject(timeout,TaskTimeoutParameter.class); |
||||||
|
} |
||||||
|
return new TaskTimeoutParameter(false); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public String toString() { |
||||||
|
return "TaskNode{" + |
||||||
|
"id='" + id + '\'' + |
||||||
|
", name='" + name + '\'' + |
||||||
|
", desc='" + desc + '\'' + |
||||||
|
", type='" + type + '\'' + |
||||||
|
", runFlag='" + runFlag + '\'' + |
||||||
|
", loc='" + loc + '\'' + |
||||||
|
", maxRetryTimes=" + maxRetryTimes + |
||||||
|
", retryInterval=" + retryInterval + |
||||||
|
", params='" + params + '\'' + |
||||||
|
", preTasks='" + preTasks + '\'' + |
||||||
|
", extras='" + extras + '\'' + |
||||||
|
", depList=" + depList + |
||||||
|
", dependence='" + dependence + '\'' + |
||||||
|
", taskInstancePriority=" + taskInstancePriority + |
||||||
|
", timeout='" + timeout + '\'' + |
||||||
|
'}'; |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,67 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.model; |
||||||
|
|
||||||
|
public class TaskNodeRelation { |
||||||
|
|
||||||
|
/** |
||||||
|
* task start node name |
||||||
|
*/ |
||||||
|
private String startNode; |
||||||
|
|
||||||
|
/** |
||||||
|
* task end node name |
||||||
|
*/ |
||||||
|
private String endNode; |
||||||
|
|
||||||
|
public TaskNodeRelation() { |
||||||
|
} |
||||||
|
|
||||||
|
public TaskNodeRelation(String startNode, String endNode) { |
||||||
|
this.startNode = startNode; |
||||||
|
this.endNode = endNode; |
||||||
|
} |
||||||
|
|
||||||
|
public String getStartNode() { |
||||||
|
return startNode; |
||||||
|
} |
||||||
|
|
||||||
|
public void setStartNode(String startNode) { |
||||||
|
this.startNode = startNode; |
||||||
|
} |
||||||
|
|
||||||
|
public String getEndNode() { |
||||||
|
return endNode; |
||||||
|
} |
||||||
|
|
||||||
|
public void setEndNode(String endNode) { |
||||||
|
this.endNode = endNode; |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
public boolean equals(TaskNodeRelation e){ |
||||||
|
return (e.getStartNode() == this.startNode && e.getEndNode() == this.endNode); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public String toString() { |
||||||
|
return "TaskNodeRelation{" + |
||||||
|
"startNode='" + startNode + '\'' + |
||||||
|
", endNode='" + endNode + '\'' + |
||||||
|
'}'; |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,85 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.process; |
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
import cn.escheduler.common.model.TaskNode; |
||||||
|
import cn.escheduler.common.model.TaskNodeRelation; |
||||||
|
|
||||||
|
import java.util.List; |
||||||
|
|
||||||
|
public class ProcessDag { |
||||||
|
|
||||||
|
/** |
||||||
|
* DAG edge list |
||||||
|
**/ |
||||||
|
private List<TaskNodeRelation> edges; |
||||||
|
|
||||||
|
/** |
||||||
|
* DAG node list |
||||||
|
*/ |
||||||
|
private List<TaskNode> nodes; |
||||||
|
|
||||||
|
/** |
||||||
|
* getter method |
||||||
|
* |
||||||
|
* @return the edges |
||||||
|
* @see ProcessDag#edges |
||||||
|
*/ |
||||||
|
public List<TaskNodeRelation> getEdges() { |
||||||
|
return edges; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* setter method |
||||||
|
* |
||||||
|
* @param edges the edges to set |
||||||
|
* @see ProcessDag#edges |
||||||
|
*/ |
||||||
|
public void setEdges(List<TaskNodeRelation> edges) { |
||||||
|
this.edges = edges; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* getter method |
||||||
|
* |
||||||
|
* @return the nodes |
||||||
|
* @see ProcessDag#nodes |
||||||
|
*/ |
||||||
|
public List<TaskNode> getNodes() { |
||||||
|
return nodes; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* setter method |
||||||
|
* |
||||||
|
* @param nodes the nodes to set |
||||||
|
* @see ProcessDag#nodes |
||||||
|
*/ |
||||||
|
public void setNodes(List<TaskNode> nodes) { |
||||||
|
this.nodes = nodes; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public String toString() { |
||||||
|
return "ProcessDag{" + |
||||||
|
"edges=" + edges + |
||||||
|
", nodes=" + nodes + |
||||||
|
'}'; |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,143 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.process; |
||||||
|
|
||||||
|
|
||||||
|
import cn.escheduler.common.enums.DataType; |
||||||
|
import cn.escheduler.common.enums.Direct; |
||||||
|
|
||||||
|
import java.util.Objects; |
||||||
|
|
||||||
|
public class Property { |
||||||
|
/** |
||||||
|
* key |
||||||
|
*/ |
||||||
|
private String prop; |
||||||
|
|
||||||
|
/** |
||||||
|
* input/output |
||||||
|
*/ |
||||||
|
private Direct direct; |
||||||
|
|
||||||
|
/** |
||||||
|
* data type |
||||||
|
*/ |
||||||
|
private DataType type; |
||||||
|
|
||||||
|
/** |
||||||
|
* value |
||||||
|
*/ |
||||||
|
private String value; |
||||||
|
|
||||||
|
public Property() { |
||||||
|
} |
||||||
|
|
||||||
|
public Property(String prop,Direct direct,DataType type,String value) { |
||||||
|
this.prop = prop; |
||||||
|
this.direct = direct; |
||||||
|
this.type = type; |
||||||
|
this.value = value; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* getter method |
||||||
|
* |
||||||
|
* @return the prop |
||||||
|
* @see Property#prop |
||||||
|
*/ |
||||||
|
public String getProp() { |
||||||
|
return prop; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* setter method |
||||||
|
* |
||||||
|
* @param prop the prop to set |
||||||
|
* @see Property#prop |
||||||
|
*/ |
||||||
|
public void setProp(String prop) { |
||||||
|
this.prop = prop; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* getter method |
||||||
|
* |
||||||
|
* @return the value |
||||||
|
* @see Property#value |
||||||
|
*/ |
||||||
|
public String getValue() { |
||||||
|
return value; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* setter method |
||||||
|
* |
||||||
|
* @param value the value to set |
||||||
|
* @see Property#value |
||||||
|
*/ |
||||||
|
public void setValue(String value) { |
||||||
|
this.value = value; |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
public Direct getDirect() { |
||||||
|
return direct; |
||||||
|
} |
||||||
|
|
||||||
|
public void setDirect(Direct direct) { |
||||||
|
this.direct = direct; |
||||||
|
} |
||||||
|
|
||||||
|
public DataType getType() { |
||||||
|
return type; |
||||||
|
} |
||||||
|
|
||||||
|
public void setType(DataType type) { |
||||||
|
this.type = type; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public boolean equals(Object o) { |
||||||
|
if (this == o) { |
||||||
|
return true; |
||||||
|
} |
||||||
|
if (o == null || getClass() != o.getClass()) { |
||||||
|
return false; |
||||||
|
} |
||||||
|
Property property = (Property) o; |
||||||
|
return Objects.equals(prop, property.prop) && |
||||||
|
Objects.equals(value, property.value); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
@Override |
||||||
|
public int hashCode() { |
||||||
|
return Objects.hash(prop, value); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public String toString() { |
||||||
|
return "Property{" + |
||||||
|
"prop='" + prop + '\'' + |
||||||
|
", direct=" + direct + |
||||||
|
", type=" + type + |
||||||
|
", value='" + value + '\'' + |
||||||
|
'}'; |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
} |
@ -0,0 +1,37 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.process; |
||||||
|
|
||||||
|
/** |
||||||
|
* resource info |
||||||
|
*/ |
||||||
|
public class ResourceInfo { |
||||||
|
/** |
||||||
|
* res the name of the resource that was uploaded |
||||||
|
*/ |
||||||
|
private String res; |
||||||
|
|
||||||
|
public String getRes() { |
||||||
|
return res; |
||||||
|
} |
||||||
|
|
||||||
|
public void setRes(String res) { |
||||||
|
this.res = res; |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
} |
@ -0,0 +1,91 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.queue; |
||||||
|
|
||||||
|
import java.util.List; |
||||||
|
import java.util.Set; |
||||||
|
|
||||||
|
public interface ITaskQueue { |
||||||
|
|
||||||
|
/** |
||||||
|
* take out all the elements |
||||||
|
* |
||||||
|
* this method has deprecated |
||||||
|
* use checkTaskExists instead |
||||||
|
* |
||||||
|
* @param key |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
@Deprecated |
||||||
|
List<String> getAllTasks(String key); |
||||||
|
|
||||||
|
/** |
||||||
|
* check task exists in the task queue or not |
||||||
|
* |
||||||
|
* @param key queue name |
||||||
|
* @param task ${priority}_${processInstanceId}_${taskId} |
||||||
|
* @return true if exists in the queue |
||||||
|
*/ |
||||||
|
boolean checkTaskExists(String key, String task); |
||||||
|
|
||||||
|
/** |
||||||
|
* add an element to the queue |
||||||
|
* |
||||||
|
* @param key queue name |
||||||
|
* @param value |
||||||
|
*/ |
||||||
|
void add(String key, String value); |
||||||
|
|
||||||
|
/** |
||||||
|
* an element pops out of the queue |
||||||
|
* |
||||||
|
* @param key queue name |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
String poll(String key); |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* add an element to the set |
||||||
|
* |
||||||
|
* @param key |
||||||
|
* @param value |
||||||
|
*/ |
||||||
|
void sadd(String key, String value); |
||||||
|
|
||||||
|
/** |
||||||
|
* delete the value corresponding to the key in the set |
||||||
|
* |
||||||
|
* @param key |
||||||
|
* @param value |
||||||
|
*/ |
||||||
|
void srem(String key, String value); |
||||||
|
|
||||||
|
/** |
||||||
|
* gets all the elements of the set based on the key |
||||||
|
* |
||||||
|
* @param key |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
Set<String> smembers(String key); |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* clear the task queue for use by junit tests only |
||||||
|
*/ |
||||||
|
void delete(); |
||||||
|
} |
@ -0,0 +1,61 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.queue; |
||||||
|
|
||||||
|
import cn.escheduler.common.utils.CommonUtils; |
||||||
|
import org.apache.commons.lang.StringUtils; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
/** |
||||||
|
* task queue factory |
||||||
|
*/ |
||||||
|
public class TaskQueueFactory { |
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(TaskQueueFactory.class); |
||||||
|
|
||||||
|
|
||||||
|
private TaskQueueFactory(){ |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* get instance (singleton) |
||||||
|
* |
||||||
|
* @return instance |
||||||
|
*/ |
||||||
|
public static ITaskQueue getTaskQueueInstance() { |
||||||
|
String queueImplValue = CommonUtils.getQueueImplValue(); |
||||||
|
if (StringUtils.isNotBlank(queueImplValue)) { |
||||||
|
// queueImplValue = StringUtils.trim(queueImplValue);
|
||||||
|
|
||||||
|
// if (SCHEDULER_QUEUE_REDIS_IMPL.equals(queueImplValue)) {
|
||||||
|
// logger.info("task queue impl use reids ");
|
||||||
|
// return TaskQueueRedisImpl.getInstance();
|
||||||
|
// } else {
|
||||||
|
logger.info("task queue impl use zookeeper "); |
||||||
|
return TaskQueueZkImpl.getInstance(); |
||||||
|
// }
|
||||||
|
}else{ |
||||||
|
logger.error("property escheduler.queue.impl can't be blank "); |
||||||
|
System.exit(-1); |
||||||
|
} |
||||||
|
|
||||||
|
return null; |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,365 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.queue; |
||||||
|
|
||||||
|
|
||||||
|
import cn.escheduler.common.Constants; |
||||||
|
import cn.escheduler.common.utils.Bytes; |
||||||
|
import cn.escheduler.common.zk.AbstractZKClient; |
||||||
|
import org.apache.curator.framework.CuratorFramework; |
||||||
|
import org.apache.zookeeper.CreateMode; |
||||||
|
import org.apache.zookeeper.data.Stat; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
import java.util.ArrayList; |
||||||
|
import java.util.HashSet; |
||||||
|
import java.util.List; |
||||||
|
import java.util.Set; |
||||||
|
|
||||||
|
/** |
||||||
|
* A singleton of a task queue implemented with zookeeper |
||||||
|
* tasks queue implemention |
||||||
|
*/ |
||||||
|
public class TaskQueueZkImpl extends AbstractZKClient implements ITaskQueue { |
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(TaskQueueZkImpl.class); |
||||||
|
|
||||||
|
private static TaskQueueZkImpl instance; |
||||||
|
|
||||||
|
private TaskQueueZkImpl(){ |
||||||
|
init(); |
||||||
|
} |
||||||
|
|
||||||
|
public static TaskQueueZkImpl getInstance(){ |
||||||
|
if (null == instance) { |
||||||
|
synchronized (TaskQueueZkImpl.class) { |
||||||
|
if(null == instance) { |
||||||
|
instance = new TaskQueueZkImpl(); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
return instance; |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* get all tasks from tasks queue |
||||||
|
* @param key task queue name |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
@Deprecated |
||||||
|
@Override |
||||||
|
public List<String> getAllTasks(String key) { |
||||||
|
try { |
||||||
|
List<String> list = getZkClient().getChildren().forPath(getTasksPath(key)); |
||||||
|
|
||||||
|
return list; |
||||||
|
} catch (Exception e) { |
||||||
|
logger.error("get all tasks from tasks queue exception",e); |
||||||
|
} |
||||||
|
|
||||||
|
return new ArrayList<String>(); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* check task exists in the task queue or not |
||||||
|
* |
||||||
|
* @param key queue name |
||||||
|
* @param task ${priority}_${processInstanceId}_${taskId} |
||||||
|
* @return true if exists in the queue |
||||||
|
*/ |
||||||
|
@Override |
||||||
|
public boolean checkTaskExists(String key, String task) { |
||||||
|
String taskPath = getTasksPath(key) + Constants.SINGLE_SLASH + task; |
||||||
|
|
||||||
|
try { |
||||||
|
Stat stat = zkClient.checkExists().forPath(taskPath); |
||||||
|
|
||||||
|
if(null == stat){ |
||||||
|
logger.info("check task:{} not exist in task queue",task); |
||||||
|
return false; |
||||||
|
}else{ |
||||||
|
logger.info("check task {} exists in task queue ",task); |
||||||
|
return true; |
||||||
|
} |
||||||
|
|
||||||
|
} catch (Exception e) { |
||||||
|
logger.info(String.format("task {} check exists in task queue exception ", task), e); |
||||||
|
} |
||||||
|
|
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* add task to tasks queue |
||||||
|
* |
||||||
|
* @param key task queue name |
||||||
|
* @param value ${priority}_${processInstanceId}_${taskId} |
||||||
|
*/ |
||||||
|
@Override |
||||||
|
public void add(String key, String value) { |
||||||
|
try { |
||||||
|
String taskIdPath = getTasksPath(key) + Constants.SINGLE_SLASH + value; |
||||||
|
String result = getZkClient().create().withMode(CreateMode.PERSISTENT).forPath(taskIdPath, Bytes.toBytes(value)); |
||||||
|
|
||||||
|
// String path = conf.getString(Constants.ZOOKEEPER_SCHEDULER_ROOT) + Constants.SINGLE_SLASH + Constants.SCHEDULER_TASKS_QUEUE + "_add" + Constants.SINGLE_SLASH + value;
|
||||||
|
// getZkClient().create().creatingParentContainersIfNeeded().withMode(CreateMode.PERSISTENT).forPath(path,
|
||||||
|
// Bytes.toBytes(value));
|
||||||
|
logger.info("add task : {} to tasks queue , result success",result); |
||||||
|
} catch (Exception e) { |
||||||
|
logger.error("add task to tasks queue exception",e); |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* An element pops out of the queue <p> |
||||||
|
* note: |
||||||
|
* ${processInstancePriority}_${processInstanceId}_${taskInstancePriority}_${taskId} |
||||||
|
* The tasks with the highest priority are selected by comparing the priorities of the above four levels from high to low. |
||||||
|
* |
||||||
|
* 流程实例优先级_流程实例id_任务优先级_任务id high <- low |
||||||
|
* @param key task queue name |
||||||
|
* @return the task id to be executed |
||||||
|
*/ |
||||||
|
@Override |
||||||
|
public String poll(String key) { |
||||||
|
try{ |
||||||
|
CuratorFramework zk = getZkClient(); |
||||||
|
String tasksQueuePath = getTasksPath(key) + Constants.SINGLE_SLASH; |
||||||
|
List<String> list = zk.getChildren().forPath(getTasksPath(key)); |
||||||
|
|
||||||
|
if(list != null && list.size() > 0){ |
||||||
|
|
||||||
|
int size = list.size(); |
||||||
|
|
||||||
|
String formatTargetTask = null; |
||||||
|
String targetTaskKey = null; |
||||||
|
for (int i = 0; i < size; i++) { |
||||||
|
String taskDetail = list.get(i); |
||||||
|
String[] taskDetailArrs = taskDetail.split(Constants.UNDERLINE); |
||||||
|
|
||||||
|
if(taskDetailArrs.length == 4){ |
||||||
|
//format ${processInstancePriority}_${processInstanceId}_${taskInstancePriority}_${taskId}
|
||||||
|
String formatTask = String.format("%s_%010d_%s_%010d", taskDetailArrs[0], Long.parseLong(taskDetailArrs[1]), taskDetailArrs[2], Long.parseLong(taskDetailArrs[3])); |
||||||
|
if(i > 0){ |
||||||
|
int result = formatTask.compareTo(formatTargetTask); |
||||||
|
if(result < 0){ |
||||||
|
formatTargetTask = formatTask; |
||||||
|
targetTaskKey = taskDetail; |
||||||
|
} |
||||||
|
}else{ |
||||||
|
formatTargetTask = formatTask; |
||||||
|
targetTaskKey = taskDetail; |
||||||
|
} |
||||||
|
}else{ |
||||||
|
logger.error("task queue poll error, task detail :{} , please check!", taskDetail); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
if(formatTargetTask != null){ |
||||||
|
String taskIdPath = tasksQueuePath + targetTaskKey; |
||||||
|
|
||||||
|
logger.info("consume task {}", taskIdPath); |
||||||
|
|
||||||
|
String[] vals = targetTaskKey.split(Constants.UNDERLINE); |
||||||
|
|
||||||
|
try{ |
||||||
|
zk.delete().forPath(taskIdPath); |
||||||
|
|
||||||
|
// String path = conf.getString(Constants.ZOOKEEPER_SCHEDULER_ROOT) + Constants.SINGLE_SLASH + Constants.SCHEDULER_TASKS_QUEUE + "_remove" + Constants.SINGLE_SLASH + targetTaskKey;
|
||||||
|
// getZkClient().create().creatingParentContainersIfNeeded().withMode(CreateMode.PERSISTENT).forPath(path,
|
||||||
|
// Bytes.toBytes(targetTaskKey));
|
||||||
|
}catch(Exception e){ |
||||||
|
logger.error(String.format("delete task:%s from zookeeper fail, task detail: %s exception" ,targetTaskKey, vals[vals.length - 1]) ,e); |
||||||
|
} |
||||||
|
logger.info("consume task: {},there still have {} tasks need to be executed", targetTaskKey, size - 1); |
||||||
|
|
||||||
|
return vals[vals.length - 1]; |
||||||
|
}else{ |
||||||
|
logger.error("should not go here, task queue poll error, please check!"); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
} catch (Exception e) { |
||||||
|
logger.error("add task to tasks queue exception",e); |
||||||
|
} |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* In order to be compatible with redis implementation |
||||||
|
* |
||||||
|
* To be compatible with the redis implementation, add an element to the set |
||||||
|
* @param key The key is the kill/cancel queue path name |
||||||
|
* @param value host-taskId The name of the zookeeper node |
||||||
|
*/ |
||||||
|
@Override |
||||||
|
public void sadd(String key,String value) { |
||||||
|
try { |
||||||
|
|
||||||
|
if(value != null && value.trim().length() > 0){ |
||||||
|
String path = getTasksPath(key) + Constants.SINGLE_SLASH; |
||||||
|
CuratorFramework zk = getZkClient(); |
||||||
|
Stat stat = zk.checkExists().forPath(path + value); |
||||||
|
|
||||||
|
if(null == stat){ |
||||||
|
String result = zk.create().withMode(CreateMode.PERSISTENT).forPath(path + value,Bytes.toBytes(value)); |
||||||
|
logger.info("add task:{} to tasks set result:{} ",value,result); |
||||||
|
}else{ |
||||||
|
logger.info("task {} exists in tasks set ",value); |
||||||
|
} |
||||||
|
|
||||||
|
}else{ |
||||||
|
logger.warn("add host-taskId:{} to tasks set is empty ",value); |
||||||
|
} |
||||||
|
|
||||||
|
} catch (Exception e) { |
||||||
|
logger.error("add task to tasks set exception",e); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* delete the value corresponding to the key in the set |
||||||
|
* @param key The key is the kill/cancel queue path name |
||||||
|
* @param value host-taskId-taskType The name of the zookeeper node |
||||||
|
*/ |
||||||
|
@Override |
||||||
|
public void srem(String key, String value) { |
||||||
|
try{ |
||||||
|
String path = getTasksPath(key) + Constants.SINGLE_SLASH; |
||||||
|
CuratorFramework zk = getZkClient(); |
||||||
|
Stat stat = zk.checkExists().forPath(path + value); |
||||||
|
|
||||||
|
if(null != stat){ |
||||||
|
zk.delete().forPath(path + value); |
||||||
|
logger.info("delete task:{} from tasks set ",value); |
||||||
|
}else{ |
||||||
|
logger.info("delete task:{} from tasks set fail, there is no this task",value); |
||||||
|
} |
||||||
|
|
||||||
|
}catch(Exception e){ |
||||||
|
logger.error(String.format("delete task:" + value + " exception"),e); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* Gets all the elements of the set based on the key |
||||||
|
* @param key The key is the kill/cancel queue path name |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
@Override |
||||||
|
public Set<String> smembers(String key) { |
||||||
|
|
||||||
|
Set<String> tasksSet = new HashSet<>(); |
||||||
|
|
||||||
|
try { |
||||||
|
List<String> list = getZkClient().getChildren().forPath(getTasksPath(key)); |
||||||
|
|
||||||
|
for (String task : list) { |
||||||
|
tasksSet.add(task); |
||||||
|
} |
||||||
|
|
||||||
|
return tasksSet; |
||||||
|
} catch (Exception e) { |
||||||
|
logger.error("get all tasks from tasks queue exception",e); |
||||||
|
} |
||||||
|
|
||||||
|
return tasksSet; |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* Init the task queue of zookeeper node |
||||||
|
*/ |
||||||
|
private void init(){ |
||||||
|
try { |
||||||
|
String tasksQueuePath = getTasksPath(Constants.SCHEDULER_TASKS_QUEUE); |
||||||
|
String tasksCancelPath = getTasksPath(Constants.SCHEDULER_TASKS_KILL); |
||||||
|
|
||||||
|
for(String taskQueuePath : new String[]{tasksQueuePath,tasksCancelPath}){ |
||||||
|
if(zkClient.checkExists().forPath(taskQueuePath) == null){ |
||||||
|
// create a persistent parent node
|
||||||
|
zkClient.create().creatingParentContainersIfNeeded() |
||||||
|
.withMode(CreateMode.PERSISTENT).forPath(taskQueuePath); |
||||||
|
logger.info("create tasks queue parent node success : {} ",taskQueuePath); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
} catch (Exception e) { |
||||||
|
logger.error("create zk node failure",e); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* Clear the task queue of zookeeper node |
||||||
|
*/ |
||||||
|
@Override |
||||||
|
public void delete(){ |
||||||
|
try { |
||||||
|
String tasksQueuePath = getTasksPath(Constants.SCHEDULER_TASKS_QUEUE); |
||||||
|
String tasksCancelPath = getTasksPath(Constants.SCHEDULER_TASKS_KILL); |
||||||
|
|
||||||
|
for(String taskQueuePath : new String[]{tasksQueuePath,tasksCancelPath}){ |
||||||
|
if(zkClient.checkExists().forPath(taskQueuePath) != null){ |
||||||
|
|
||||||
|
List<String> list = zkClient.getChildren().forPath(taskQueuePath); |
||||||
|
|
||||||
|
for (String task : list) { |
||||||
|
zkClient.delete().forPath(taskQueuePath + Constants.SINGLE_SLASH + task); |
||||||
|
logger.info("delete task from tasks queue : {}/{} ",taskQueuePath,task); |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
} catch (Exception e) { |
||||||
|
logger.error("delete all tasks in tasks queue failure",e); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* get zookeeper client of CuratorFramework |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public CuratorFramework getZkClient() { |
||||||
|
return zkClient; |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* Get the task queue path |
||||||
|
* @param key task queue name |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public String getTasksPath(String key){ |
||||||
|
return conf.getString(Constants.ZOOKEEPER_SCHEDULER_ROOT) + Constants.SINGLE_SLASH + key; |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
} |
@ -0,0 +1,341 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.shell; |
||||||
|
|
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
import java.io.BufferedReader; |
||||||
|
import java.io.File; |
||||||
|
import java.io.IOException; |
||||||
|
import java.io.InputStreamReader; |
||||||
|
import java.util.Map; |
||||||
|
import java.util.Set; |
||||||
|
import java.util.Timer; |
||||||
|
import java.util.TimerTask; |
||||||
|
import java.util.concurrent.ConcurrentHashMap; |
||||||
|
import java.util.concurrent.atomic.AtomicBoolean; |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* A base class for running a Unix command. |
||||||
|
* |
||||||
|
* <code>AbstractShell</code> can be used to run unix commands like <code>du</code> or |
||||||
|
* <code>df</code>. It also offers facilities to gate commands by |
||||||
|
* time-intervals. |
||||||
|
*/ |
||||||
|
public abstract class AbstractShell { |
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(AbstractShell.class); |
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* Time after which the executing script would be timedout |
||||||
|
*/ |
||||||
|
protected long timeOutInterval = 0L; |
||||||
|
/** |
||||||
|
* If or not script timed out |
||||||
|
*/ |
||||||
|
private AtomicBoolean timedOut; |
||||||
|
|
||||||
|
/** |
||||||
|
* refresh interval in msec |
||||||
|
*/ |
||||||
|
private long interval; |
||||||
|
|
||||||
|
/** |
||||||
|
* last time the command was performed |
||||||
|
*/ |
||||||
|
private long lastTime; |
||||||
|
|
||||||
|
/** |
||||||
|
* env for the command execution |
||||||
|
*/ |
||||||
|
private Map<String, String> environment; |
||||||
|
private File dir; |
||||||
|
|
||||||
|
/** |
||||||
|
* sub process used to execute the command |
||||||
|
*/ |
||||||
|
private Process process; |
||||||
|
private int exitCode; |
||||||
|
|
||||||
|
/** |
||||||
|
* If or not script finished executing |
||||||
|
*/ |
||||||
|
private volatile AtomicBoolean completed; |
||||||
|
|
||||||
|
public AbstractShell() { |
||||||
|
this(0L); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* @param interval the minimum duration to wait before re-executing the |
||||||
|
* command. |
||||||
|
*/ |
||||||
|
public AbstractShell(long interval ) { |
||||||
|
this.interval = interval; |
||||||
|
this.lastTime = (interval<0) ? 0 : -interval; |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* set the environment for the command |
||||||
|
* @param env Mapping of environment variables |
||||||
|
*/ |
||||||
|
protected void setEnvironment(Map<String, String> env) { |
||||||
|
this.environment = env; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* set the working directory |
||||||
|
* @param dir The directory where the command would be executed |
||||||
|
*/ |
||||||
|
protected void setWorkingDirectory(File dir) { |
||||||
|
this.dir = dir; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* check to see if a command needs to be executed and execute if needed |
||||||
|
*/ |
||||||
|
protected void run() throws IOException { |
||||||
|
if (lastTime + interval > System.currentTimeMillis()) { |
||||||
|
return; |
||||||
|
} |
||||||
|
// reset for next run
|
||||||
|
exitCode = 0; |
||||||
|
runCommand(); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* Run a command actual work |
||||||
|
*/ |
||||||
|
private void runCommand() throws IOException { |
||||||
|
ProcessBuilder builder = new ProcessBuilder(getExecString()); |
||||||
|
Timer timeOutTimer = null; |
||||||
|
ShellTimeoutTimerTask timeoutTimerTask = null; |
||||||
|
timedOut = new AtomicBoolean(false); |
||||||
|
completed = new AtomicBoolean(false); |
||||||
|
|
||||||
|
if (environment != null) { |
||||||
|
builder.environment().putAll(this.environment); |
||||||
|
} |
||||||
|
if (dir != null) { |
||||||
|
builder.directory(this.dir); |
||||||
|
} |
||||||
|
|
||||||
|
process = builder.start(); |
||||||
|
ProcessContainer.putProcess(process); |
||||||
|
|
||||||
|
if (timeOutInterval > 0) { |
||||||
|
timeOutTimer = new Timer(); |
||||||
|
timeoutTimerTask = new ShellTimeoutTimerTask( |
||||||
|
this); |
||||||
|
//One time scheduling.
|
||||||
|
timeOutTimer.schedule(timeoutTimerTask, timeOutInterval); |
||||||
|
} |
||||||
|
final BufferedReader errReader = |
||||||
|
new BufferedReader(new InputStreamReader(process |
||||||
|
.getErrorStream())); |
||||||
|
BufferedReader inReader = |
||||||
|
new BufferedReader(new InputStreamReader(process |
||||||
|
.getInputStream())); |
||||||
|
final StringBuffer errMsg = new StringBuffer(); |
||||||
|
|
||||||
|
// read error and input streams as this would free up the buffers
|
||||||
|
// free the error stream buffer
|
||||||
|
Thread errThread = new Thread() { |
||||||
|
@Override |
||||||
|
public void run() { |
||||||
|
try { |
||||||
|
String line = errReader.readLine(); |
||||||
|
while((line != null) && !isInterrupted()) { |
||||||
|
errMsg.append(line); |
||||||
|
errMsg.append(System.getProperty("line.separator")); |
||||||
|
line = errReader.readLine(); |
||||||
|
} |
||||||
|
} catch(IOException ioe) { |
||||||
|
logger.warn("Error reading the error stream", ioe); |
||||||
|
} |
||||||
|
} |
||||||
|
}; |
||||||
|
try { |
||||||
|
errThread.start(); |
||||||
|
} catch (IllegalStateException ise) { } |
||||||
|
try { |
||||||
|
// parse the output
|
||||||
|
parseExecResult(inReader); |
||||||
|
exitCode = process.waitFor(); |
||||||
|
try { |
||||||
|
// make sure that the error thread exits
|
||||||
|
errThread.join(); |
||||||
|
} catch (InterruptedException ie) { |
||||||
|
logger.warn("Interrupted while reading the error stream", ie); |
||||||
|
} |
||||||
|
completed.set(true); |
||||||
|
//the timeout thread handling
|
||||||
|
//taken care in finally block
|
||||||
|
if (exitCode != 0) { |
||||||
|
throw new ExitCodeException(exitCode, errMsg.toString()); |
||||||
|
} |
||||||
|
} catch (InterruptedException ie) { |
||||||
|
throw new IOException(ie.toString()); |
||||||
|
} finally { |
||||||
|
if ((timeOutTimer!=null) && !timedOut.get()) { |
||||||
|
timeOutTimer.cancel(); |
||||||
|
} |
||||||
|
// close the input stream
|
||||||
|
try { |
||||||
|
inReader.close(); |
||||||
|
} catch (IOException ioe) { |
||||||
|
logger.warn("Error while closing the input stream", ioe); |
||||||
|
} |
||||||
|
if (!completed.get()) { |
||||||
|
errThread.interrupt(); |
||||||
|
} |
||||||
|
try { |
||||||
|
errReader.close(); |
||||||
|
} catch (IOException ioe) { |
||||||
|
logger.warn("Error while closing the error stream", ioe); |
||||||
|
} |
||||||
|
ProcessContainer.removeProcess(process); |
||||||
|
process.destroy(); |
||||||
|
lastTime = System.currentTimeMillis(); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* return an array containing the command name & its parameters |
||||||
|
* */ |
||||||
|
protected abstract String[] getExecString(); |
||||||
|
|
||||||
|
/** |
||||||
|
* Parse the execution result |
||||||
|
* */ |
||||||
|
protected abstract void parseExecResult(BufferedReader lines) |
||||||
|
throws IOException; |
||||||
|
|
||||||
|
/** |
||||||
|
* get the current sub-process executing the given command |
||||||
|
* @return process executing the command |
||||||
|
*/ |
||||||
|
public Process getProcess() { |
||||||
|
return process; |
||||||
|
} |
||||||
|
|
||||||
|
/** get the exit code |
||||||
|
* @return the exit code of the process |
||||||
|
*/ |
||||||
|
public int getExitCode() { |
||||||
|
return exitCode; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Set if the command has timed out. |
||||||
|
* |
||||||
|
*/ |
||||||
|
private void setTimedOut() { |
||||||
|
this.timedOut.set(true); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* Timer which is used to timeout scripts spawned off by shell. |
||||||
|
*/ |
||||||
|
private static class ShellTimeoutTimerTask extends TimerTask { |
||||||
|
|
||||||
|
private AbstractShell shell; |
||||||
|
|
||||||
|
public ShellTimeoutTimerTask(AbstractShell shell) { |
||||||
|
this.shell = shell; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public void run() { |
||||||
|
Process p = shell.getProcess(); |
||||||
|
try { |
||||||
|
p.exitValue(); |
||||||
|
} catch (Exception e) { |
||||||
|
//Process has not terminated.
|
||||||
|
//So check if it has completed
|
||||||
|
//if not just destroy it.
|
||||||
|
if (p != null && !shell.completed.get()) { |
||||||
|
shell.setTimedOut(); |
||||||
|
p.destroy(); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* This is an IOException with exit code added. |
||||||
|
*/ |
||||||
|
public static class ExitCodeException extends IOException { |
||||||
|
int exitCode; |
||||||
|
|
||||||
|
public ExitCodeException(int exitCode, String message) { |
||||||
|
super(message); |
||||||
|
this.exitCode = exitCode; |
||||||
|
} |
||||||
|
|
||||||
|
public int getExitCode() { |
||||||
|
return exitCode; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* process manage container |
||||||
|
* |
||||||
|
*/ |
||||||
|
public static class ProcessContainer extends ConcurrentHashMap<Integer, Process>{ |
||||||
|
private static final ProcessContainer container = new ProcessContainer(); |
||||||
|
private ProcessContainer(){ |
||||||
|
super(); |
||||||
|
} |
||||||
|
public static final ProcessContainer getInstance(){ |
||||||
|
return container; |
||||||
|
} |
||||||
|
|
||||||
|
public static void putProcess(Process process){ |
||||||
|
getInstance().put(process.hashCode(), process); |
||||||
|
} |
||||||
|
public static int processSize(){ |
||||||
|
return getInstance().size(); |
||||||
|
} |
||||||
|
|
||||||
|
public static void removeProcess(Process process){ |
||||||
|
getInstance().remove(process.hashCode()); |
||||||
|
} |
||||||
|
|
||||||
|
public static void destroyAllProcess(){ |
||||||
|
Set<Entry<Integer, Process>> set = getInstance().entrySet(); |
||||||
|
for (Entry<Integer, Process> entry : set) { |
||||||
|
try{ |
||||||
|
entry.getValue().destroy(); |
||||||
|
} catch (Exception e) { |
||||||
|
e.printStackTrace(); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
logger.info("close " + set.size() + " executing process tasks"); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,175 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.shell; |
||||||
|
|
||||||
|
import java.io.BufferedReader; |
||||||
|
import java.io.File; |
||||||
|
import java.io.IOException; |
||||||
|
import java.util.Map; |
||||||
|
|
||||||
|
/** |
||||||
|
* shell command executor. |
||||||
|
* |
||||||
|
* <code>ShellExecutor</code> should be used in cases where the output |
||||||
|
* of the command needs no explicit parsing and where the command, working |
||||||
|
* directory and the environment remains unchanged. The output of the command |
||||||
|
* is stored as-is and is expected to be small. |
||||||
|
*/ |
||||||
|
public class ShellExecutor extends AbstractShell { |
||||||
|
|
||||||
|
private String[] command; |
||||||
|
private StringBuffer output; |
||||||
|
|
||||||
|
|
||||||
|
public ShellExecutor(String... execString) { |
||||||
|
this(execString, null); |
||||||
|
} |
||||||
|
|
||||||
|
public ShellExecutor(String[] execString, File dir) { |
||||||
|
this(execString, dir, null); |
||||||
|
} |
||||||
|
|
||||||
|
public ShellExecutor(String[] execString, File dir, |
||||||
|
Map<String, String> env) { |
||||||
|
this(execString, dir, env , 0L); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Create a new instance of the ShellExecutor to execute a command. |
||||||
|
* |
||||||
|
* @param execString The command to execute with arguments |
||||||
|
* @param dir If not-null, specifies the directory which should be set |
||||||
|
* as the current working directory for the command. |
||||||
|
* If null, the current working directory is not modified. |
||||||
|
* @param env If not-null, environment of the command will include the |
||||||
|
* key-value pairs specified in the map. If null, the current |
||||||
|
* environment is not modified. |
||||||
|
* @param timeout Specifies the time in milliseconds, after which the |
||||||
|
* command will be killed and the status marked as timedout. |
||||||
|
* If 0, the command will not be timed out. |
||||||
|
*/ |
||||||
|
public ShellExecutor(String[] execString, File dir, |
||||||
|
Map<String, String> env, long timeout) { |
||||||
|
command = execString.clone(); |
||||||
|
if (dir != null) { |
||||||
|
setWorkingDirectory(dir); |
||||||
|
} |
||||||
|
if (env != null) { |
||||||
|
setEnvironment(env); |
||||||
|
} |
||||||
|
timeOutInterval = timeout; |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* Static method to execute a shell command. |
||||||
|
* Covers most of the simple cases without requiring the user to implement |
||||||
|
* the <code>AbstractShell</code> interface. |
||||||
|
* @param cmd shell command to execute. |
||||||
|
* @return the output of the executed command. |
||||||
|
*/ |
||||||
|
public static String execCommand(String... cmd) throws IOException { |
||||||
|
return execCommand(null, cmd, 0L); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Static method to execute a shell command. |
||||||
|
* Covers most of the simple cases without requiring the user to implement |
||||||
|
* the <code>AbstractShell</code> interface. |
||||||
|
* @param env the map of environment key=value |
||||||
|
* @param cmd shell command to execute. |
||||||
|
* @param timeout time in milliseconds after which script should be marked timeout |
||||||
|
* @return the output of the executed command.o |
||||||
|
*/ |
||||||
|
|
||||||
|
public static String execCommand(Map<String, String> env, String[] cmd, |
||||||
|
long timeout) throws IOException { |
||||||
|
ShellExecutor exec = new ShellExecutor(cmd, null, env, |
||||||
|
timeout); |
||||||
|
exec.execute(); |
||||||
|
return exec.getOutput(); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Static method to execute a shell command. |
||||||
|
* Covers most of the simple cases without requiring the user to implement |
||||||
|
* the <code>AbstractShell</code> interface. |
||||||
|
* @param env the map of environment key=value |
||||||
|
* @param cmd shell command to execute. |
||||||
|
* @return the output of the executed command. |
||||||
|
*/ |
||||||
|
public static String execCommand(Map<String,String> env, String ... cmd) |
||||||
|
throws IOException { |
||||||
|
return execCommand(env, cmd, 0L); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Execute the shell command |
||||||
|
* |
||||||
|
*/ |
||||||
|
public void execute() throws IOException { |
||||||
|
this.run(); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
protected String[] getExecString() { |
||||||
|
return command; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
protected void parseExecResult(BufferedReader lines) throws IOException { |
||||||
|
output = new StringBuffer(); |
||||||
|
char[] buf = new char[1024]; |
||||||
|
int nRead; |
||||||
|
String line = ""; |
||||||
|
while ( (nRead = lines.read(buf, 0, buf.length)) > 0 ) { |
||||||
|
line = new String(buf,0,nRead); |
||||||
|
} |
||||||
|
output.append(line); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* |
||||||
|
* Get the output of the shell command |
||||||
|
*/ |
||||||
|
public String getOutput() { |
||||||
|
return (output == null) ? "" : output.toString(); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* Returns the commands of this instance. |
||||||
|
* Arguments with spaces in are presented with quotes round; other |
||||||
|
* arguments are presented raw |
||||||
|
* |
||||||
|
* @return a string representation of the object |
||||||
|
*/ |
||||||
|
@Override |
||||||
|
public String toString() { |
||||||
|
StringBuilder builder = new StringBuilder(); |
||||||
|
String[] args = getExecString(); |
||||||
|
for (String s : args) { |
||||||
|
if (s.indexOf(' ') >= 0) { |
||||||
|
builder.append('"').append(s).append('"'); |
||||||
|
} else { |
||||||
|
builder.append(s); |
||||||
|
} |
||||||
|
builder.append(' '); |
||||||
|
} |
||||||
|
return builder.toString(); |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,69 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.task; |
||||||
|
|
||||||
|
import cn.escheduler.common.process.Property; |
||||||
|
|
||||||
|
import java.util.LinkedHashMap; |
||||||
|
import java.util.List; |
||||||
|
import java.util.Map; |
||||||
|
|
||||||
|
/** |
||||||
|
* job params related class
|
||||||
|
*/ |
||||||
|
public abstract class AbstractParameters implements IParameters { |
||||||
|
|
||||||
|
@Override |
||||||
|
public abstract boolean checkParameters(); |
||||||
|
|
||||||
|
@Override |
||||||
|
public abstract List<String> getResourceFilesList(); |
||||||
|
|
||||||
|
/** |
||||||
|
* local parameters |
||||||
|
*/ |
||||||
|
public List<Property> localParams; |
||||||
|
|
||||||
|
/** |
||||||
|
* get local parameters list |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public List<Property> getLocalParams() { |
||||||
|
return localParams; |
||||||
|
} |
||||||
|
|
||||||
|
public void setLocalParams(List<Property> localParams) { |
||||||
|
this.localParams = localParams; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get local parameters map |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public Map<String,Property> getLocalParametersMap() { |
||||||
|
if (localParams != null) { |
||||||
|
Map<String,Property> localParametersMaps = new LinkedHashMap<>(); |
||||||
|
|
||||||
|
for (Property property : localParams) { |
||||||
|
localParametersMaps.put(property.getProp(),property); |
||||||
|
} |
||||||
|
return localParametersMaps; |
||||||
|
} |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
} |
@ -0,0 +1,38 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.task; |
||||||
|
|
||||||
|
import java.util.List; |
||||||
|
|
||||||
|
/** |
||||||
|
* job params interface
|
||||||
|
*/ |
||||||
|
public interface IParameters { |
||||||
|
/** |
||||||
|
* check parameters is valid |
||||||
|
* |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
boolean checkParameters(); |
||||||
|
|
||||||
|
/** |
||||||
|
* get project resource files list |
||||||
|
* |
||||||
|
* @return resource files list |
||||||
|
*/ |
||||||
|
List<String> getResourceFilesList(); |
||||||
|
} |
@ -0,0 +1,81 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.task; |
||||||
|
|
||||||
|
import cn.escheduler.common.enums.TaskTimeoutStrategy; |
||||||
|
|
||||||
|
/** |
||||||
|
* task timeout parameter |
||||||
|
*/ |
||||||
|
public class TaskTimeoutParameter { |
||||||
|
|
||||||
|
private boolean enable; |
||||||
|
/** |
||||||
|
* task timeout strategy |
||||||
|
*/ |
||||||
|
private TaskTimeoutStrategy strategy; |
||||||
|
/** |
||||||
|
* task timeout interval |
||||||
|
*/ |
||||||
|
private int interval; |
||||||
|
|
||||||
|
public boolean getEnable() { |
||||||
|
return enable; |
||||||
|
} |
||||||
|
|
||||||
|
public void setEnable(boolean enable) { |
||||||
|
this.enable = enable; |
||||||
|
} |
||||||
|
|
||||||
|
public TaskTimeoutStrategy getStrategy() { |
||||||
|
return strategy; |
||||||
|
} |
||||||
|
|
||||||
|
public void setStrategy(TaskTimeoutStrategy strategy) { |
||||||
|
this.strategy = strategy; |
||||||
|
} |
||||||
|
|
||||||
|
public int getInterval() { |
||||||
|
return interval; |
||||||
|
} |
||||||
|
|
||||||
|
public void setInterval(int interval) { |
||||||
|
this.interval = interval; |
||||||
|
} |
||||||
|
|
||||||
|
public TaskTimeoutParameter() { |
||||||
|
} |
||||||
|
|
||||||
|
public TaskTimeoutParameter(boolean enable) { |
||||||
|
this.enable = enable; |
||||||
|
} |
||||||
|
|
||||||
|
public TaskTimeoutParameter(boolean enable, TaskTimeoutStrategy strategy, int interval) { |
||||||
|
this.enable = enable; |
||||||
|
this.strategy = strategy; |
||||||
|
this.interval = interval; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public String toString() { |
||||||
|
return "TaskTimeoutParameter{" + |
||||||
|
"enable=" + enable + |
||||||
|
", strategy=" + strategy + |
||||||
|
", interval=" + interval + |
||||||
|
'}'; |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,58 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.task.dependent; |
||||||
|
|
||||||
|
import cn.escheduler.common.enums.DependentRelation; |
||||||
|
import cn.escheduler.common.model.DependentTaskModel; |
||||||
|
import cn.escheduler.common.task.AbstractParameters; |
||||||
|
|
||||||
|
import java.util.ArrayList; |
||||||
|
import java.util.List; |
||||||
|
|
||||||
|
public class DependentParameters extends AbstractParameters { |
||||||
|
|
||||||
|
private List<DependentTaskModel> dependTaskList; |
||||||
|
private DependentRelation relation; |
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
@Override |
||||||
|
public boolean checkParameters() { |
||||||
|
return true; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public List<String> getResourceFilesList() { |
||||||
|
return new ArrayList<>(); |
||||||
|
} |
||||||
|
|
||||||
|
public List<DependentTaskModel> getDependTaskList() { |
||||||
|
return dependTaskList; |
||||||
|
} |
||||||
|
|
||||||
|
public void setDependTaskList(List<DependentTaskModel> dependTaskList) { |
||||||
|
this.dependTaskList = dependTaskList; |
||||||
|
} |
||||||
|
|
||||||
|
public DependentRelation getRelation() { |
||||||
|
return relation; |
||||||
|
} |
||||||
|
|
||||||
|
public void setRelation(DependentRelation relation) { |
||||||
|
this.relation = relation; |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,145 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.task.mr; |
||||||
|
|
||||||
|
import cn.escheduler.common.enums.ProgramType; |
||||||
|
import cn.escheduler.common.process.ResourceInfo; |
||||||
|
import cn.escheduler.common.task.AbstractParameters; |
||||||
|
|
||||||
|
import java.util.List; |
||||||
|
import java.util.stream.Collectors; |
||||||
|
|
||||||
|
public class MapreduceParameters extends AbstractParameters { |
||||||
|
|
||||||
|
/** |
||||||
|
* major jar |
||||||
|
*/ |
||||||
|
private ResourceInfo mainJar; |
||||||
|
|
||||||
|
/** |
||||||
|
* major class
|
||||||
|
*/ |
||||||
|
private String mainClass; |
||||||
|
|
||||||
|
/** |
||||||
|
* arguments |
||||||
|
*/ |
||||||
|
private String mainArgs; |
||||||
|
|
||||||
|
/** |
||||||
|
* other arguments |
||||||
|
*/ |
||||||
|
private String others; |
||||||
|
|
||||||
|
/** |
||||||
|
* queue |
||||||
|
*/ |
||||||
|
private String queue; |
||||||
|
|
||||||
|
/** |
||||||
|
* resource list |
||||||
|
*/ |
||||||
|
private List<ResourceInfo> resourceList; |
||||||
|
|
||||||
|
/** |
||||||
|
* program type |
||||||
|
* 0 JAVA,1 SCALA,2 PYTHON |
||||||
|
*/ |
||||||
|
private ProgramType programType; |
||||||
|
|
||||||
|
|
||||||
|
public String getMainClass() { |
||||||
|
return mainClass; |
||||||
|
} |
||||||
|
|
||||||
|
public void setMainClass(String mainClass) { |
||||||
|
this.mainClass = mainClass; |
||||||
|
} |
||||||
|
|
||||||
|
public String getMainArgs() { |
||||||
|
return mainArgs; |
||||||
|
} |
||||||
|
|
||||||
|
public void setMainArgs(String mainArgs) { |
||||||
|
this.mainArgs = mainArgs; |
||||||
|
} |
||||||
|
|
||||||
|
public String getOthers() { |
||||||
|
return others; |
||||||
|
} |
||||||
|
|
||||||
|
public void setOthers(String others) { |
||||||
|
this.others = others; |
||||||
|
} |
||||||
|
|
||||||
|
public String getQueue() { |
||||||
|
return queue; |
||||||
|
} |
||||||
|
|
||||||
|
public void setQueue(String queue) { |
||||||
|
this.queue = queue; |
||||||
|
} |
||||||
|
|
||||||
|
public List<ResourceInfo> getResourceList() { |
||||||
|
return this.resourceList; |
||||||
|
} |
||||||
|
|
||||||
|
public void setResourceList(List<ResourceInfo> resourceList) { |
||||||
|
this.resourceList = resourceList; |
||||||
|
} |
||||||
|
|
||||||
|
public void setMainJar(ResourceInfo mainJar) { |
||||||
|
this.mainJar = mainJar; |
||||||
|
} |
||||||
|
|
||||||
|
public ResourceInfo getMainJar() { |
||||||
|
return mainJar; |
||||||
|
} |
||||||
|
|
||||||
|
public ProgramType getProgramType() { |
||||||
|
return programType; |
||||||
|
} |
||||||
|
|
||||||
|
public void setProgramType(ProgramType programType) { |
||||||
|
this.programType = programType; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public boolean checkParameters() { |
||||||
|
return this.mainJar != null && this.programType != null; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public List<String> getResourceFilesList() { |
||||||
|
if (resourceList != null) { |
||||||
|
this.resourceList.add(mainJar); |
||||||
|
return resourceList.stream() |
||||||
|
.map(p -> p.getRes()).collect(Collectors.toList()); |
||||||
|
} |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public String toString() { |
||||||
|
return "mainJar= " + mainJar |
||||||
|
+ "mainClass=" + mainClass |
||||||
|
+ "mainArgs=" + mainArgs |
||||||
|
+ "queue=" + queue |
||||||
|
+ "other mainArgs=" + others |
||||||
|
; |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,89 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.task.procedure; |
||||||
|
|
||||||
|
import cn.escheduler.common.task.AbstractParameters; |
||||||
|
import org.apache.commons.lang.StringUtils; |
||||||
|
|
||||||
|
import java.util.ArrayList; |
||||||
|
import java.util.List; |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* procedure parameter |
||||||
|
*/ |
||||||
|
public class ProcedureParameters extends AbstractParameters { |
||||||
|
|
||||||
|
/** |
||||||
|
* data source type,eg MYSQL, POSTGRES, HIVE ... |
||||||
|
*/ |
||||||
|
private String type; |
||||||
|
|
||||||
|
/** |
||||||
|
* data source id |
||||||
|
*/ |
||||||
|
private int datasource; |
||||||
|
|
||||||
|
/** |
||||||
|
* procedure name |
||||||
|
*/ |
||||||
|
private String method; |
||||||
|
|
||||||
|
|
||||||
|
public String getType() { |
||||||
|
return type; |
||||||
|
} |
||||||
|
|
||||||
|
public void setType(String type) { |
||||||
|
this.type = type; |
||||||
|
} |
||||||
|
|
||||||
|
public int getDatasource() { |
||||||
|
return datasource; |
||||||
|
} |
||||||
|
|
||||||
|
public void setDatasource(int datasource) { |
||||||
|
this.datasource = datasource; |
||||||
|
} |
||||||
|
|
||||||
|
public String getMethod() { |
||||||
|
return method; |
||||||
|
} |
||||||
|
|
||||||
|
public void setMethod(String method) { |
||||||
|
this.method = method; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public boolean checkParameters() { |
||||||
|
return datasource != 0 && StringUtils.isNotEmpty(type) && StringUtils.isNotEmpty(method); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public List<String> getResourceFilesList() { |
||||||
|
return new ArrayList<>(); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public String toString() { |
||||||
|
return "ProcessdureParam{" + |
||||||
|
"type='" + type + '\'' + |
||||||
|
", datasource=" + datasource + |
||||||
|
", method='" + method + '\'' + |
||||||
|
'}'; |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,67 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.task.python; |
||||||
|
|
||||||
|
|
||||||
|
import cn.escheduler.common.process.ResourceInfo; |
||||||
|
import cn.escheduler.common.task.AbstractParameters; |
||||||
|
|
||||||
|
import java.util.List; |
||||||
|
import java.util.stream.Collectors; |
||||||
|
|
||||||
|
public class PythonParameters extends AbstractParameters { |
||||||
|
/** |
||||||
|
* origin python script |
||||||
|
*/ |
||||||
|
private String rawScript; |
||||||
|
|
||||||
|
/** |
||||||
|
* resource list |
||||||
|
*/ |
||||||
|
private List<ResourceInfo> resourceList; |
||||||
|
|
||||||
|
public String getRawScript() { |
||||||
|
return rawScript; |
||||||
|
} |
||||||
|
|
||||||
|
public void setRawScript(String rawScript) { |
||||||
|
this.rawScript = rawScript; |
||||||
|
} |
||||||
|
|
||||||
|
public List<ResourceInfo> getResourceList() { |
||||||
|
return resourceList; |
||||||
|
} |
||||||
|
|
||||||
|
public void setResourceList(List<ResourceInfo> resourceList) { |
||||||
|
this.resourceList = resourceList; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public boolean checkParameters() { |
||||||
|
return rawScript != null && !rawScript.isEmpty(); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public List<String> getResourceFilesList() { |
||||||
|
if (resourceList != null) { |
||||||
|
return resourceList.stream() |
||||||
|
.map(p -> p.getRes()).collect(Collectors.toList()); |
||||||
|
} |
||||||
|
|
||||||
|
return null; |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,70 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.task.shell; |
||||||
|
|
||||||
|
|
||||||
|
import cn.escheduler.common.process.ResourceInfo; |
||||||
|
import cn.escheduler.common.task.AbstractParameters; |
||||||
|
|
||||||
|
import java.util.List; |
||||||
|
import java.util.stream.Collectors; |
||||||
|
|
||||||
|
/** |
||||||
|
* shell parameters |
||||||
|
*/ |
||||||
|
public class ShellParameters extends AbstractParameters { |
||||||
|
/** |
||||||
|
* shell script |
||||||
|
*/ |
||||||
|
private String rawScript; |
||||||
|
|
||||||
|
/** |
||||||
|
* resource list |
||||||
|
*/ |
||||||
|
private List<ResourceInfo> resourceList; |
||||||
|
|
||||||
|
public String getRawScript() { |
||||||
|
return rawScript; |
||||||
|
} |
||||||
|
|
||||||
|
public void setRawScript(String rawScript) { |
||||||
|
this.rawScript = rawScript; |
||||||
|
} |
||||||
|
|
||||||
|
public List<ResourceInfo> getResourceList() { |
||||||
|
return resourceList; |
||||||
|
} |
||||||
|
|
||||||
|
public void setResourceList(List<ResourceInfo> resourceList) { |
||||||
|
this.resourceList = resourceList; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public boolean checkParameters() { |
||||||
|
return rawScript != null && !rawScript.isEmpty(); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public List<String> getResourceFilesList() { |
||||||
|
if (resourceList != null) { |
||||||
|
return resourceList.stream() |
||||||
|
.map(p -> p.getRes()).collect(Collectors.toList()); |
||||||
|
} |
||||||
|
|
||||||
|
return null; |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,220 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.task.spark; |
||||||
|
|
||||||
|
import cn.escheduler.common.enums.ProgramType; |
||||||
|
import cn.escheduler.common.process.ResourceInfo; |
||||||
|
import cn.escheduler.common.task.AbstractParameters; |
||||||
|
|
||||||
|
import java.util.List; |
||||||
|
import java.util.stream.Collectors; |
||||||
|
|
||||||
|
/** |
||||||
|
* spark parameters |
||||||
|
*/ |
||||||
|
public class SparkParameters extends AbstractParameters { |
||||||
|
|
||||||
|
/** |
||||||
|
* major jar |
||||||
|
*/ |
||||||
|
private ResourceInfo mainJar; |
||||||
|
|
||||||
|
/** |
||||||
|
* major class
|
||||||
|
*/ |
||||||
|
private String mainClass; |
||||||
|
|
||||||
|
/** |
||||||
|
* deploy mode |
||||||
|
*/ |
||||||
|
private String deployMode; |
||||||
|
|
||||||
|
/** |
||||||
|
* arguments |
||||||
|
*/ |
||||||
|
private String mainArgs; |
||||||
|
|
||||||
|
/** |
||||||
|
* driver-cores Number of cores used by the driver, only in cluster mode |
||||||
|
*/ |
||||||
|
private int driverCores; |
||||||
|
|
||||||
|
/** |
||||||
|
* driver-memory Memory for driver |
||||||
|
*/ |
||||||
|
|
||||||
|
private String driverMemory; |
||||||
|
|
||||||
|
/** |
||||||
|
* num-executors Number of executors to launch |
||||||
|
*/ |
||||||
|
private int numExecutors; |
||||||
|
|
||||||
|
/** |
||||||
|
* executor-cores Number of cores per executor |
||||||
|
*/ |
||||||
|
private int executorCores; |
||||||
|
|
||||||
|
/** |
||||||
|
* Memory per executor |
||||||
|
*/ |
||||||
|
private String executorMemory; |
||||||
|
|
||||||
|
/** |
||||||
|
* resource list |
||||||
|
*/ |
||||||
|
private List<ResourceInfo> resourceList; |
||||||
|
|
||||||
|
/** |
||||||
|
* The YARN queue to submit to |
||||||
|
*/ |
||||||
|
private String queue; |
||||||
|
|
||||||
|
/** |
||||||
|
* other arguments |
||||||
|
*/ |
||||||
|
private String others; |
||||||
|
|
||||||
|
/** |
||||||
|
* program type |
||||||
|
* 0 JAVA,1 SCALA,2 PYTHON |
||||||
|
*/ |
||||||
|
private ProgramType programType; |
||||||
|
|
||||||
|
public ResourceInfo getMainJar() { |
||||||
|
return mainJar; |
||||||
|
} |
||||||
|
|
||||||
|
public void setMainJar(ResourceInfo mainJar) { |
||||||
|
this.mainJar = mainJar; |
||||||
|
} |
||||||
|
|
||||||
|
public String getMainClass() { |
||||||
|
return mainClass; |
||||||
|
} |
||||||
|
|
||||||
|
public void setMainClass(String mainClass) { |
||||||
|
this.mainClass = mainClass; |
||||||
|
} |
||||||
|
|
||||||
|
public String getDeployMode() { |
||||||
|
return deployMode; |
||||||
|
} |
||||||
|
|
||||||
|
public void setDeployMode(String deployMode) { |
||||||
|
this.deployMode = deployMode; |
||||||
|
} |
||||||
|
|
||||||
|
public String getMainArgs() { |
||||||
|
return mainArgs; |
||||||
|
} |
||||||
|
|
||||||
|
public void setMainArgs(String mainArgs) { |
||||||
|
this.mainArgs = mainArgs; |
||||||
|
} |
||||||
|
|
||||||
|
public int getDriverCores() { |
||||||
|
return driverCores; |
||||||
|
} |
||||||
|
|
||||||
|
public void setDriverCores(int driverCores) { |
||||||
|
this.driverCores = driverCores; |
||||||
|
} |
||||||
|
|
||||||
|
public String getDriverMemory() { |
||||||
|
return driverMemory; |
||||||
|
} |
||||||
|
|
||||||
|
public void setDriverMemory(String driverMemory) { |
||||||
|
this.driverMemory = driverMemory; |
||||||
|
} |
||||||
|
|
||||||
|
public int getNumExecutors() { |
||||||
|
return numExecutors; |
||||||
|
} |
||||||
|
|
||||||
|
public void setNumExecutors(int numExecutors) { |
||||||
|
this.numExecutors = numExecutors; |
||||||
|
} |
||||||
|
|
||||||
|
public int getExecutorCores() { |
||||||
|
return executorCores; |
||||||
|
} |
||||||
|
|
||||||
|
public void setExecutorCores(int executorCores) { |
||||||
|
this.executorCores = executorCores; |
||||||
|
} |
||||||
|
|
||||||
|
public String getExecutorMemory() { |
||||||
|
return executorMemory; |
||||||
|
} |
||||||
|
|
||||||
|
public void setExecutorMemory(String executorMemory) { |
||||||
|
this.executorMemory = executorMemory; |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
public String getQueue() { |
||||||
|
return queue; |
||||||
|
} |
||||||
|
|
||||||
|
public void setQueue(String queue) { |
||||||
|
this.queue = queue; |
||||||
|
} |
||||||
|
|
||||||
|
public List<ResourceInfo> getResourceList() { |
||||||
|
return resourceList; |
||||||
|
} |
||||||
|
|
||||||
|
public void setResourceList(List<ResourceInfo> resourceList) { |
||||||
|
this.resourceList = resourceList; |
||||||
|
} |
||||||
|
|
||||||
|
public String getOthers() { |
||||||
|
return others; |
||||||
|
} |
||||||
|
|
||||||
|
public void setOthers(String others) { |
||||||
|
this.others = others; |
||||||
|
} |
||||||
|
|
||||||
|
public ProgramType getProgramType() { |
||||||
|
return programType; |
||||||
|
} |
||||||
|
|
||||||
|
public void setProgramType(ProgramType programType) { |
||||||
|
this.programType = programType; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public boolean checkParameters() { |
||||||
|
return mainJar != null && programType != null; |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
@Override |
||||||
|
public List<String> getResourceFilesList() { |
||||||
|
if(resourceList !=null ) { |
||||||
|
this.resourceList.add(mainJar); |
||||||
|
return resourceList.stream() |
||||||
|
.map(p -> p.getRes()).collect(Collectors.toList()); |
||||||
|
} |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
} |
@ -0,0 +1,147 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.task.sql; |
||||||
|
|
||||||
|
import cn.escheduler.common.task.AbstractParameters; |
||||||
|
import org.apache.commons.lang.StringUtils; |
||||||
|
|
||||||
|
import java.util.ArrayList; |
||||||
|
import java.util.List; |
||||||
|
|
||||||
|
/** |
||||||
|
* Sql/Hql parameter |
||||||
|
*/ |
||||||
|
public class SqlParameters extends AbstractParameters { |
||||||
|
/** |
||||||
|
* data source type,eg MYSQL, POSTGRES, HIVE ... |
||||||
|
*/ |
||||||
|
private String type; |
||||||
|
|
||||||
|
/** |
||||||
|
* datasource id |
||||||
|
*/ |
||||||
|
private int datasource; |
||||||
|
|
||||||
|
/** |
||||||
|
* sql |
||||||
|
*/ |
||||||
|
private String sql; |
||||||
|
|
||||||
|
/** |
||||||
|
* sql type |
||||||
|
* 0 query |
||||||
|
* 1 NON_QUERY |
||||||
|
*/ |
||||||
|
private int sqlType; |
||||||
|
|
||||||
|
/** |
||||||
|
* udf list |
||||||
|
*/ |
||||||
|
private String udfs; |
||||||
|
/** |
||||||
|
* show type |
||||||
|
* 0 TABLE |
||||||
|
* 1 TEXT |
||||||
|
* 2 attachment |
||||||
|
* 3 TABLE+attachment |
||||||
|
*/ |
||||||
|
private String showType; |
||||||
|
/** |
||||||
|
* SQL connection parameters |
||||||
|
*/ |
||||||
|
private String connParams; |
||||||
|
|
||||||
|
public String getType() { |
||||||
|
return type; |
||||||
|
} |
||||||
|
|
||||||
|
public void setType(String type) { |
||||||
|
this.type = type; |
||||||
|
} |
||||||
|
|
||||||
|
public int getDatasource() { |
||||||
|
return datasource; |
||||||
|
} |
||||||
|
|
||||||
|
public void setDatasource(int datasource) { |
||||||
|
this.datasource = datasource; |
||||||
|
} |
||||||
|
|
||||||
|
public String getSql() { |
||||||
|
return sql; |
||||||
|
} |
||||||
|
|
||||||
|
public void setSql(String sql) { |
||||||
|
this.sql = sql; |
||||||
|
} |
||||||
|
|
||||||
|
public String getUdfs() { |
||||||
|
return udfs; |
||||||
|
} |
||||||
|
|
||||||
|
public void setUdfs(String udfs) { |
||||||
|
this.udfs = udfs; |
||||||
|
} |
||||||
|
|
||||||
|
public int getSqlType() { |
||||||
|
return sqlType; |
||||||
|
} |
||||||
|
|
||||||
|
public void setSqlType(int sqlType) { |
||||||
|
this.sqlType = sqlType; |
||||||
|
} |
||||||
|
|
||||||
|
public String getShowType() { |
||||||
|
return showType; |
||||||
|
} |
||||||
|
|
||||||
|
public void setShowType(String showType) { |
||||||
|
this.showType = showType; |
||||||
|
} |
||||||
|
|
||||||
|
public String getConnParams() { |
||||||
|
return connParams; |
||||||
|
} |
||||||
|
|
||||||
|
public void setConnParams(String connParams) { |
||||||
|
this.connParams = connParams; |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
@Override |
||||||
|
public boolean checkParameters() { |
||||||
|
return datasource != 0 && StringUtils.isNotEmpty(type) && StringUtils.isNotEmpty(sql); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public List<String> getResourceFilesList() { |
||||||
|
return new ArrayList<>(); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public String toString() { |
||||||
|
return "SqlParameters{" + |
||||||
|
"type='" + type + '\'' + |
||||||
|
", datasource=" + datasource + |
||||||
|
", sql='" + sql + '\'' + |
||||||
|
", sqlType=" + sqlType + |
||||||
|
", udfs='" + udfs + '\'' + |
||||||
|
", showType='" + showType + '\'' + |
||||||
|
", connParams='" + connParams + '\'' + |
||||||
|
'}'; |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,27 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.task.sql; |
||||||
|
|
||||||
|
|
||||||
|
public enum SqlType { |
||||||
|
/** |
||||||
|
* sql type |
||||||
|
* 0 query |
||||||
|
* 1 NON_QUERY |
||||||
|
*/ |
||||||
|
QUERY, NON_QUERY |
||||||
|
} |
@ -0,0 +1,48 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.task.subprocess; |
||||||
|
import cn.escheduler.common.task.AbstractParameters; |
||||||
|
|
||||||
|
import java.util.ArrayList; |
||||||
|
import java.util.List; |
||||||
|
|
||||||
|
|
||||||
|
public class SubProcessParameters extends AbstractParameters { |
||||||
|
|
||||||
|
/** |
||||||
|
* process definition id |
||||||
|
*/ |
||||||
|
private Integer processDefinitionId; |
||||||
|
|
||||||
|
public void setProcessDefinitionId(Integer processDefinitionId){ |
||||||
|
this.processDefinitionId = processDefinitionId; |
||||||
|
} |
||||||
|
|
||||||
|
public Integer getProcessDefinitionId(){ |
||||||
|
return this.processDefinitionId; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public boolean checkParameters() { |
||||||
|
return this.processDefinitionId != 0; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public List<String> getResourceFilesList() { |
||||||
|
return new ArrayList<>(); |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,39 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.thread; |
||||||
|
|
||||||
|
import java.util.concurrent.atomic.AtomicBoolean; |
||||||
|
|
||||||
|
/** |
||||||
|
* if the process closes, a signal is placed as true, and all threads get this flag to stop working |
||||||
|
*/ |
||||||
|
public class Stopper { |
||||||
|
|
||||||
|
private static volatile AtomicBoolean signal = new AtomicBoolean(false); |
||||||
|
|
||||||
|
public static final boolean isStoped(){ |
||||||
|
return signal.get(); |
||||||
|
} |
||||||
|
|
||||||
|
public static final boolean isRunning(){ |
||||||
|
return !signal.get(); |
||||||
|
} |
||||||
|
|
||||||
|
public static final void stop(){ |
||||||
|
signal.getAndSet(true); |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,310 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.thread; |
||||||
|
|
||||||
|
|
||||||
|
import com.google.common.collect.Lists; |
||||||
|
import com.google.common.collect.Maps; |
||||||
|
import com.google.common.util.concurrent.ThreadFactoryBuilder; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
import java.io.PrintWriter; |
||||||
|
import java.lang.management.ThreadInfo; |
||||||
|
import java.util.List; |
||||||
|
import java.util.Map; |
||||||
|
import java.util.concurrent.*; |
||||||
|
import java.util.concurrent.atomic.AtomicLong; |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* |
||||||
|
* thread pool's single instance |
||||||
|
* |
||||||
|
*/ |
||||||
|
public class ThreadPoolExecutors { |
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(ThreadPoolExecutors.class); |
||||||
|
private static Executor executor; |
||||||
|
private static ThreadPoolExecutors threadPoolExecutors; |
||||||
|
|
||||||
|
private ThreadPoolExecutors(){} |
||||||
|
|
||||||
|
|
||||||
|
public static ThreadPoolExecutors getInstance(){ |
||||||
|
return getInstance("thread_pool",0); |
||||||
|
} |
||||||
|
|
||||||
|
public static ThreadPoolExecutors getInstance(String name, int maxThreads){ |
||||||
|
|
||||||
|
if (null == threadPoolExecutors) { |
||||||
|
|
||||||
|
synchronized (ThreadPoolExecutors.class) { |
||||||
|
|
||||||
|
if(null == threadPoolExecutors) { |
||||||
|
threadPoolExecutors = new ThreadPoolExecutors(); |
||||||
|
} |
||||||
|
if(null == executor) { |
||||||
|
executor = new Executor(null == name? "thread_pool" : name, maxThreads == 0? Runtime.getRuntime().availableProcessors() * 3 : maxThreads); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
return threadPoolExecutors; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Executes the given task sometime in the future. The task may execute in a new thread or in an existing pooled thread. |
||||||
|
* If the task cannot be submitted for execution, either because this executor has been shutdown or because its capacity has been reached, |
||||||
|
* the task is handled by the current RejectedExecutionHandler. |
||||||
|
* @param event |
||||||
|
*/ |
||||||
|
public void execute(final Runnable event) { |
||||||
|
Executor executor = getExecutor(); |
||||||
|
if (executor == null) { |
||||||
|
logger.error("Cannot execute [" + event + "] because the executor is missing."); |
||||||
|
} else { |
||||||
|
executor.execute(event); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
public Future<?> submit(Runnable event) { |
||||||
|
Executor executor = getExecutor(); |
||||||
|
if (executor == null) { |
||||||
|
logger.error("Cannot submit [" + event + "] because the executor is missing."); |
||||||
|
} else { |
||||||
|
return executor.submit(event); |
||||||
|
} |
||||||
|
|
||||||
|
return null; |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
public Future<?> submit(Callable<?> task) { |
||||||
|
Executor executor = getExecutor(); |
||||||
|
if (executor == null) { |
||||||
|
logger.error("Cannot submit [" + task + "] because the executor is missing."); |
||||||
|
} else { |
||||||
|
return executor.submit(task); |
||||||
|
} |
||||||
|
|
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
public void printStatus() { |
||||||
|
Executor executor = getExecutor(); |
||||||
|
executor.getStatus().dumpInfo(); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
private Executor getExecutor() { |
||||||
|
return executor; |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
public void shutdown() { |
||||||
|
if (executor != null) { |
||||||
|
List<Runnable> wasRunning = executor.threadPoolExecutor |
||||||
|
.shutdownNow(); |
||||||
|
if (!wasRunning.isEmpty()) { |
||||||
|
logger.info(executor + " had " + wasRunning + " on shutdown"); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* Executor instance. |
||||||
|
*/ |
||||||
|
private static class Executor { |
||||||
|
/** |
||||||
|
* how long to retain excess threads |
||||||
|
*/ |
||||||
|
final long keepAliveTimeInMillis = 1000; |
||||||
|
/** |
||||||
|
* the thread pool executor that services the requests |
||||||
|
*/ |
||||||
|
final TrackingThreadPoolExecutor threadPoolExecutor; |
||||||
|
/** |
||||||
|
* work queue to use - unbounded queue |
||||||
|
*/ |
||||||
|
final BlockingQueue<Runnable> q = new LinkedBlockingQueue<Runnable>(); |
||||||
|
private final String name; |
||||||
|
private static final AtomicLong seqids = new AtomicLong(0); |
||||||
|
private final long id; |
||||||
|
|
||||||
|
protected Executor(String name, int maxThreads) { |
||||||
|
this.id = seqids.incrementAndGet(); |
||||||
|
this.name = name; |
||||||
|
//create the thread pool executor
|
||||||
|
this.threadPoolExecutor = new TrackingThreadPoolExecutor( |
||||||
|
maxThreads, maxThreads, keepAliveTimeInMillis, |
||||||
|
TimeUnit.MILLISECONDS, q); |
||||||
|
// name the threads for this threadpool
|
||||||
|
ThreadFactoryBuilder tfb = new ThreadFactoryBuilder(); |
||||||
|
tfb.setNameFormat(this.name + "-%d"); |
||||||
|
this.threadPoolExecutor.setThreadFactory(tfb.build()); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Submit the event to the queue for handling. |
||||||
|
* |
||||||
|
* @param event |
||||||
|
*/ |
||||||
|
void execute(final Runnable event) { |
||||||
|
this.threadPoolExecutor.execute(event); |
||||||
|
} |
||||||
|
|
||||||
|
Future<?> submit(Runnable event) { |
||||||
|
return this.threadPoolExecutor.submit(event); |
||||||
|
} |
||||||
|
|
||||||
|
Future<?> submit(Callable<?> event) { |
||||||
|
return this.threadPoolExecutor.submit(event); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
@Override |
||||||
|
public String toString() { |
||||||
|
return getClass().getSimpleName() + "-" + id + "-" + name; |
||||||
|
} |
||||||
|
|
||||||
|
public ExecutorStatus getStatus() { |
||||||
|
List<Runnable> queuedEvents = Lists.newArrayList(); |
||||||
|
for (Runnable r : q) { |
||||||
|
queuedEvents.add(r); |
||||||
|
} |
||||||
|
|
||||||
|
List<RunningEventStatus> running = Lists.newArrayList(); |
||||||
|
for (Map.Entry<Thread, Runnable> e : threadPoolExecutor |
||||||
|
.getRunningTasks().entrySet()) { |
||||||
|
Runnable r = e.getValue(); |
||||||
|
running.add(new RunningEventStatus(e.getKey(), r)); |
||||||
|
} |
||||||
|
|
||||||
|
return new ExecutorStatus(this, queuedEvents, running); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* A subclass of ThreadPoolExecutor that keeps track of the Runnables that |
||||||
|
* are executing at any given point in time. |
||||||
|
*/ |
||||||
|
static class TrackingThreadPoolExecutor extends ThreadPoolExecutor { |
||||||
|
private ConcurrentMap<Thread, Runnable> running = Maps |
||||||
|
.newConcurrentMap(); |
||||||
|
|
||||||
|
public TrackingThreadPoolExecutor(int corePoolSize, |
||||||
|
int maximumPoolSize, long keepAliveTime, TimeUnit unit, |
||||||
|
BlockingQueue<Runnable> workQueue) { |
||||||
|
super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
protected void afterExecute(Runnable r, Throwable t) { |
||||||
|
super.afterExecute(r, t); |
||||||
|
running.remove(Thread.currentThread()); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
protected void beforeExecute(Thread t, Runnable r) { |
||||||
|
Runnable oldPut = running.put(t, r); |
||||||
|
assert oldPut == null : "inconsistency for thread " + t; |
||||||
|
super.beforeExecute(t, r); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* @return a map of the threads currently running tasks inside this |
||||||
|
* executor. Each key is an active thread, and the value is the |
||||||
|
* task that is currently running. Note that this is not a |
||||||
|
* stable snapshot of the map. |
||||||
|
*/ |
||||||
|
public ConcurrentMap<Thread, Runnable> getRunningTasks() { |
||||||
|
return running; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* A snapshot of the status of a particular executor. This includes the |
||||||
|
* contents of the executor's pending queue, as well as the threads and |
||||||
|
* events currently being processed. |
||||||
|
* |
||||||
|
* This is a consistent snapshot that is immutable once constructed. |
||||||
|
*/ |
||||||
|
public static class ExecutorStatus { |
||||||
|
final Executor executor; |
||||||
|
final List<Runnable> queuedEvents; |
||||||
|
final List<RunningEventStatus> running; |
||||||
|
|
||||||
|
ExecutorStatus(Executor executor, List<Runnable> queuedEvents, |
||||||
|
List<RunningEventStatus> running) { |
||||||
|
this.executor = executor; |
||||||
|
this.queuedEvents = queuedEvents; |
||||||
|
this.running = running; |
||||||
|
} |
||||||
|
|
||||||
|
public void dumpInfo() { |
||||||
|
|
||||||
|
PrintWriter out = new PrintWriter(System.out); |
||||||
|
|
||||||
|
out.write("Status for executor: " + executor + "\n"); |
||||||
|
out.write("=======================================\n"); |
||||||
|
out.write(queuedEvents.size() + " events queued, " |
||||||
|
+ running.size() + " running\n"); |
||||||
|
if (!queuedEvents.isEmpty()) { |
||||||
|
out.write("Queued:\n"); |
||||||
|
for (Runnable e : queuedEvents) { |
||||||
|
out.write(" " + e + "\n"); |
||||||
|
} |
||||||
|
out.write("\n"); |
||||||
|
} |
||||||
|
if (!running.isEmpty()) { |
||||||
|
out.write("Running:\n"); |
||||||
|
for (RunningEventStatus stat : running) { |
||||||
|
out.write(" Running on thread '" |
||||||
|
+ stat.threadInfo.getThreadName() + "': " |
||||||
|
+ stat.event + "\n"); |
||||||
|
out.write(ThreadUtils.formatThreadInfo( |
||||||
|
stat.threadInfo, " ")); |
||||||
|
out.write("\n"); |
||||||
|
} |
||||||
|
} |
||||||
|
out.flush(); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* The status of a particular event that is in the middle of being handled |
||||||
|
* by an executor. |
||||||
|
*/ |
||||||
|
public static class RunningEventStatus { |
||||||
|
final ThreadInfo threadInfo; |
||||||
|
final Runnable event; |
||||||
|
|
||||||
|
public RunningEventStatus(Thread t, Runnable event) { |
||||||
|
this.threadInfo = ThreadUtils.getThreadInfo(t); |
||||||
|
this.event = event; |
||||||
|
} |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,202 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.thread; |
||||||
|
|
||||||
|
import com.google.common.util.concurrent.ThreadFactoryBuilder; |
||||||
|
|
||||||
|
import java.lang.management.ManagementFactory; |
||||||
|
import java.lang.management.ThreadInfo; |
||||||
|
import java.lang.management.ThreadMXBean; |
||||||
|
import java.util.concurrent.*; |
||||||
|
|
||||||
|
/** |
||||||
|
* thread utils |
||||||
|
*/ |
||||||
|
public class ThreadUtils { |
||||||
|
|
||||||
|
|
||||||
|
private static final ThreadMXBean threadBean = ManagementFactory.getThreadMXBean(); |
||||||
|
private static final int STACK_DEPTH = 20; |
||||||
|
|
||||||
|
/** |
||||||
|
Wrapper over newCachedThreadPool. Thread names are formatted as prefix-ID, where ID is a |
||||||
|
* unique, sequentially assigned integer. |
||||||
|
* @param prefix |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static ThreadPoolExecutor newDaemonCachedThreadPool(String prefix){ |
||||||
|
ThreadFactory threadFactory = namedThreadFactory(prefix); |
||||||
|
return ((ThreadPoolExecutor) Executors.newCachedThreadPool(threadFactory)); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Create a thread factory that names threads with a prefix and also sets the threads to daemon. |
||||||
|
* @param prefix |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
private static ThreadFactory namedThreadFactory(String prefix) { |
||||||
|
return new ThreadFactoryBuilder().setDaemon(true).setNameFormat(prefix + "-%d").build(); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* Create a cached thread pool whose max number of threads is `maxThreadNumber`. Thread names |
||||||
|
* are formatted as prefix-ID, where ID is a unique, sequentially assigned integer. |
||||||
|
* @param prefix |
||||||
|
* @param maxThreadNumber |
||||||
|
* @param keepAliveSeconds |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static ThreadPoolExecutor newDaemonCachedThreadPool(String prefix , |
||||||
|
int maxThreadNumber, |
||||||
|
int keepAliveSeconds){ |
||||||
|
ThreadFactory threadFactory = namedThreadFactory(prefix); |
||||||
|
ThreadPoolExecutor threadPool = new ThreadPoolExecutor( |
||||||
|
// corePoolSize: the max number of threads to create before queuing the tasks
|
||||||
|
maxThreadNumber, |
||||||
|
// maximumPoolSize: because we use LinkedBlockingDeque, this one is not used
|
||||||
|
maxThreadNumber, |
||||||
|
keepAliveSeconds, |
||||||
|
TimeUnit.SECONDS, |
||||||
|
new LinkedBlockingQueue<Runnable>(), |
||||||
|
threadFactory); |
||||||
|
threadPool.allowCoreThreadTimeOut(true); |
||||||
|
return threadPool; |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* Wrapper over newFixedThreadPool. Thread names are formatted as prefix-ID, where ID is a |
||||||
|
* unique, sequentially assigned integer. |
||||||
|
* @param nThreads |
||||||
|
* @param prefix |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static ThreadPoolExecutor newDaemonFixedThreadPool(int nThreads , String prefix){ |
||||||
|
ThreadFactory threadFactory = namedThreadFactory(prefix); |
||||||
|
return ((ThreadPoolExecutor) Executors.newFixedThreadPool(nThreads, threadFactory)); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Wrapper over newSingleThreadExecutor. |
||||||
|
* @param threadName |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static ExecutorService newDaemonSingleThreadExecutor(String threadName){ |
||||||
|
ThreadFactory threadFactory = new ThreadFactoryBuilder() |
||||||
|
.setDaemon(true) |
||||||
|
.setNameFormat(threadName) |
||||||
|
.build(); |
||||||
|
return Executors.newSingleThreadExecutor(threadFactory); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Wrapper over newDaemonFixedThreadExecutor. |
||||||
|
* @param threadName |
||||||
|
* @param threadsNum |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static ExecutorService newDaemonFixedThreadExecutor(String threadName,int threadsNum){ |
||||||
|
ThreadFactory threadFactory = new ThreadFactoryBuilder() |
||||||
|
.setDaemon(true) |
||||||
|
.setNameFormat(threadName) |
||||||
|
.build(); |
||||||
|
return Executors.newFixedThreadPool(threadsNum,threadFactory); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Wrapper over ScheduledThreadPoolExecutor |
||||||
|
* @param corePoolSize |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static ScheduledExecutorService newDaemonThreadScheduledExecutor(String threadName,int corePoolSize) { |
||||||
|
ThreadFactory threadFactory = new ThreadFactoryBuilder() |
||||||
|
.setDaemon(true) |
||||||
|
.setNameFormat(threadName) |
||||||
|
.build(); |
||||||
|
ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(corePoolSize, threadFactory); |
||||||
|
// By default, a cancelled task is not automatically removed from the work queue until its delay
|
||||||
|
// elapses. We have to enable it manually.
|
||||||
|
executor.setRemoveOnCancelPolicy(true); |
||||||
|
return executor; |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
public static ThreadInfo getThreadInfo(Thread t) { |
||||||
|
long tid = t.getId(); |
||||||
|
return threadBean.getThreadInfo(tid, STACK_DEPTH); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* Format the given ThreadInfo object as a String. |
||||||
|
* @param indent a prefix for each line, used for nested indentation |
||||||
|
*/ |
||||||
|
public static String formatThreadInfo(ThreadInfo threadInfo, String indent) { |
||||||
|
StringBuilder sb = new StringBuilder(); |
||||||
|
appendThreadInfo(sb, threadInfo, indent); |
||||||
|
return sb.toString(); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* Print all of the thread's information and stack traces. |
||||||
|
* |
||||||
|
* @param sb |
||||||
|
* @param info |
||||||
|
* @param indent |
||||||
|
*/ |
||||||
|
public static void appendThreadInfo(StringBuilder sb, |
||||||
|
ThreadInfo info, |
||||||
|
String indent) { |
||||||
|
boolean contention = threadBean.isThreadContentionMonitoringEnabled(); |
||||||
|
|
||||||
|
if (info == null) { |
||||||
|
sb.append(indent).append("Inactive (perhaps exited while monitoring was done)\n"); |
||||||
|
return; |
||||||
|
} |
||||||
|
String taskName = getTaskName(info.getThreadId(), info.getThreadName()); |
||||||
|
sb.append(indent).append("Thread ").append(taskName).append(":\n"); |
||||||
|
|
||||||
|
Thread.State state = info.getThreadState(); |
||||||
|
sb.append(indent).append(" State: ").append(state).append("\n"); |
||||||
|
sb.append(indent).append(" Blocked count: ").append(info.getBlockedCount()).append("\n"); |
||||||
|
sb.append(indent).append(" Waited count: ").append(info.getWaitedCount()).append("\n"); |
||||||
|
if (contention) { |
||||||
|
sb.append(indent).append(" Blocked time: " + info.getBlockedTime()).append("\n"); |
||||||
|
sb.append(indent).append(" Waited time: " + info.getWaitedTime()).append("\n"); |
||||||
|
} |
||||||
|
if (state == Thread.State.WAITING) { |
||||||
|
sb.append(indent).append(" Waiting on ").append(info.getLockName()).append("\n"); |
||||||
|
} else if (state == Thread.State.BLOCKED) { |
||||||
|
sb.append(indent).append(" Blocked on ").append(info.getLockName()).append("\n"); |
||||||
|
sb.append(indent).append(" Blocked by ").append( |
||||||
|
getTaskName(info.getLockOwnerId(), info.getLockOwnerName())).append("\n"); |
||||||
|
} |
||||||
|
sb.append(indent).append(" Stack:").append("\n"); |
||||||
|
for (StackTraceElement frame: info.getStackTrace()) { |
||||||
|
sb.append(indent).append(" ").append(frame.toString()).append("\n"); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
private static String getTaskName(long id, String name) { |
||||||
|
if (name == null) { |
||||||
|
return Long.toString(id); |
||||||
|
} |
||||||
|
return id + " (" + name + ")"; |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,697 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.utils; |
||||||
|
|
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
import java.math.BigDecimal; |
||||||
|
import java.math.BigInteger; |
||||||
|
import java.nio.ByteBuffer; |
||||||
|
import java.nio.charset.StandardCharsets; |
||||||
|
|
||||||
|
/** |
||||||
|
* Utility class that handles Bytes |
||||||
|
*/ |
||||||
|
public class Bytes { |
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(Bytes.class); |
||||||
|
public static final String UTF8_ENCODING = "UTF-8"; |
||||||
|
//An empty instance.
|
||||||
|
public static final byte [] EMPTY_BYTE_ARRAY = new byte [0]; |
||||||
|
|
||||||
|
/** |
||||||
|
* Size of int in bytes |
||||||
|
*/ |
||||||
|
public static final int SIZEOF_INT = Integer.SIZE / Byte.SIZE; |
||||||
|
|
||||||
|
/** |
||||||
|
* Size of long in bytes |
||||||
|
*/ |
||||||
|
public static final int SIZEOF_LONG = Long.SIZE / Byte.SIZE; |
||||||
|
|
||||||
|
/** |
||||||
|
* Size of short in bytes |
||||||
|
*/ |
||||||
|
public static final int SIZEOF_SHORT = Short.SIZE / Byte.SIZE; |
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* Put bytes at the specified byte array position. |
||||||
|
* @param tgtBytes the byte array |
||||||
|
* @param tgtOffset position in the array |
||||||
|
* @param srcBytes array to write out |
||||||
|
* @param srcOffset source offset |
||||||
|
* @param srcLength source length |
||||||
|
* @return incremented offset |
||||||
|
*/ |
||||||
|
public static int putBytes(byte[] tgtBytes, int tgtOffset, byte[] srcBytes, |
||||||
|
int srcOffset, int srcLength) { |
||||||
|
System.arraycopy(srcBytes, srcOffset, tgtBytes, tgtOffset, srcLength); |
||||||
|
return tgtOffset + srcLength; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Write a single byte out to the specified byte array position. |
||||||
|
* @param bytes the byte array |
||||||
|
* @param offset position in the array |
||||||
|
* @param b byte to write out |
||||||
|
* @return incremented offset |
||||||
|
*/ |
||||||
|
public static int putByte(byte[] bytes, int offset, byte b) { |
||||||
|
bytes[offset] = b; |
||||||
|
return offset + 1; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Returns a new byte array, copied from the passed ByteBuffer. |
||||||
|
* @param bb A ByteBuffer |
||||||
|
* @return the byte array |
||||||
|
*/ |
||||||
|
public static byte[] toBytes(ByteBuffer bb) { |
||||||
|
int length = bb.limit(); |
||||||
|
byte [] result = new byte[length]; |
||||||
|
System.arraycopy(bb.array(), bb.arrayOffset(), result, 0, length); |
||||||
|
return result; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* @param b Presumed UTF-8 encoded byte array. |
||||||
|
* @return String made from <code>b</code> |
||||||
|
*/ |
||||||
|
public static String toString(final byte [] b) { |
||||||
|
if (b == null) { |
||||||
|
return null; |
||||||
|
} |
||||||
|
return toString(b, 0, b.length); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Joins two byte arrays together using a separator. |
||||||
|
* @param b1 The first byte array. |
||||||
|
* @param sep The separator to use. |
||||||
|
* @param b2 The second byte array. |
||||||
|
*/ |
||||||
|
public static String toString(final byte [] b1, |
||||||
|
String sep, |
||||||
|
final byte [] b2) { |
||||||
|
return toString(b1, 0, b1.length) + sep + toString(b2, 0, b2.length); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* This method will convert utf8 encoded bytes into a string. If |
||||||
|
* an UnsupportedEncodingException occurs, this method will eat it |
||||||
|
* and return null instead. |
||||||
|
* |
||||||
|
* @param b Presumed UTF-8 encoded byte array. |
||||||
|
* @param off offset into array |
||||||
|
* @param len length of utf-8 sequence |
||||||
|
* @return String made from <code>b</code> or null |
||||||
|
*/ |
||||||
|
public static String toString(final byte [] b, int off, int len) { |
||||||
|
if (b == null) { |
||||||
|
return null; |
||||||
|
} |
||||||
|
if (len == 0) { |
||||||
|
return ""; |
||||||
|
} |
||||||
|
return new String(b, off, len, StandardCharsets.UTF_8); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* Converts a string to a UTF-8 byte array. |
||||||
|
* @param s string |
||||||
|
* @return the byte array |
||||||
|
*/ |
||||||
|
public static byte[] toBytes(String s) { |
||||||
|
return s.getBytes(StandardCharsets.UTF_8); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Convert a boolean to a byte array. True becomes -1 |
||||||
|
* and false becomes 0. |
||||||
|
* |
||||||
|
* @param b value |
||||||
|
* @return <code>b</code> encoded in a byte array. |
||||||
|
*/ |
||||||
|
public static byte [] toBytes(final boolean b) { |
||||||
|
return new byte[] { b ? (byte) -1 : (byte) 0 }; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Reverses {@link #toBytes(boolean)} |
||||||
|
* @param b array |
||||||
|
* @return True or false. |
||||||
|
*/ |
||||||
|
public static boolean toBoolean(final byte [] b) { |
||||||
|
if (b.length != 1) { |
||||||
|
throw new IllegalArgumentException("Array has wrong size: " + b.length); |
||||||
|
} |
||||||
|
return b[0] != (byte) 0; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Convert a long value to a byte array using big-endian. |
||||||
|
* |
||||||
|
* @param val value to convert |
||||||
|
* @return the byte array |
||||||
|
*/ |
||||||
|
public static byte[] toBytes(long val) { |
||||||
|
byte [] b = new byte[8]; |
||||||
|
for (int i = 7; i > 0; i--) { |
||||||
|
b[i] = (byte) val; |
||||||
|
val >>>= 8; |
||||||
|
} |
||||||
|
b[0] = (byte) val; |
||||||
|
return b; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Converts a byte array to a long value. Reverses |
||||||
|
* {@link #toBytes(long)} |
||||||
|
* @param bytes array |
||||||
|
* @return the long value |
||||||
|
*/ |
||||||
|
public static long toLong(byte[] bytes) { |
||||||
|
return toLong(bytes, 0, SIZEOF_LONG); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Converts a byte array to a long value. Assumes there will be |
||||||
|
* {@link #SIZEOF_LONG} bytes available. |
||||||
|
* |
||||||
|
* @param bytes bytes |
||||||
|
* @param offset offset |
||||||
|
* @return the long value |
||||||
|
*/ |
||||||
|
public static long toLong(byte[] bytes, int offset) { |
||||||
|
return toLong(bytes, offset, SIZEOF_LONG); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Converts a byte array to a long value. |
||||||
|
* |
||||||
|
* @param bytes array of bytes |
||||||
|
* @param offset offset into array |
||||||
|
* @param length length of data (must be {@link #SIZEOF_LONG}) |
||||||
|
* @return the long value |
||||||
|
* @throws IllegalArgumentException if length is not {@link #SIZEOF_LONG} or |
||||||
|
* if there's not enough room in the array at the offset indicated. |
||||||
|
*/ |
||||||
|
public static long toLong(byte[] bytes, int offset, final int length) { |
||||||
|
if (length != SIZEOF_LONG || offset + length > bytes.length) { |
||||||
|
throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_LONG); |
||||||
|
} |
||||||
|
long l = 0; |
||||||
|
for(int i = offset; i < offset + length; i++) { |
||||||
|
l <<= 8; |
||||||
|
l ^= bytes[i] & 0xFF; |
||||||
|
} |
||||||
|
return l; |
||||||
|
} |
||||||
|
|
||||||
|
private static IllegalArgumentException |
||||||
|
explainWrongLengthOrOffset(final byte[] bytes, |
||||||
|
final int offset, |
||||||
|
final int length, |
||||||
|
final int expectedLength) { |
||||||
|
String reason; |
||||||
|
if (length != expectedLength) { |
||||||
|
reason = "Wrong length: " + length + ", expected " + expectedLength; |
||||||
|
} else { |
||||||
|
reason = "offset (" + offset + ") + length (" + length + ") exceed the" |
||||||
|
+ " capacity of the array: " + bytes.length; |
||||||
|
} |
||||||
|
return new IllegalArgumentException(reason); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Put a long value out to the specified byte array position. |
||||||
|
* @param bytes the byte array |
||||||
|
* @param offset position in the array |
||||||
|
* @param val long to write out |
||||||
|
* @return incremented offset |
||||||
|
* @throws IllegalArgumentException if the byte array given doesn't have |
||||||
|
* enough room at the offset specified. |
||||||
|
*/ |
||||||
|
public static int putLong(byte[] bytes, int offset, long val) { |
||||||
|
if (bytes.length - offset < SIZEOF_LONG) { |
||||||
|
throw new IllegalArgumentException("Not enough room to put a long at" |
||||||
|
+ " offset " + offset + " in a " + bytes.length + " byte array"); |
||||||
|
} |
||||||
|
for(int i = offset + 7; i > offset; i--) { |
||||||
|
bytes[i] = (byte) val; |
||||||
|
val >>>= 8; |
||||||
|
} |
||||||
|
bytes[offset] = (byte) val; |
||||||
|
return offset + SIZEOF_LONG; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Presumes float encoded as IEEE 754 floating-point "single format" |
||||||
|
* @param bytes byte array |
||||||
|
* @return Float made from passed byte array. |
||||||
|
*/ |
||||||
|
public static float toFloat(byte [] bytes) { |
||||||
|
return toFloat(bytes, 0); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Presumes float encoded as IEEE 754 floating-point "single format" |
||||||
|
* @param bytes array to convert |
||||||
|
* @param offset offset into array |
||||||
|
* @return Float made from passed byte array. |
||||||
|
*/ |
||||||
|
public static float toFloat(byte [] bytes, int offset) { |
||||||
|
return Float.intBitsToFloat(toInt(bytes, offset, SIZEOF_INT)); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* @param bytes byte array |
||||||
|
* @param offset offset to write to |
||||||
|
* @param f float value |
||||||
|
* @return New offset in <code>bytes</code> |
||||||
|
*/ |
||||||
|
public static int putFloat(byte [] bytes, int offset, float f) { |
||||||
|
return putInt(bytes, offset, Float.floatToRawIntBits(f)); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* @param f float value |
||||||
|
* @return the float represented as byte [] |
||||||
|
*/ |
||||||
|
public static byte [] toBytes(final float f) { |
||||||
|
// Encode it as int
|
||||||
|
return Bytes.toBytes(Float.floatToRawIntBits(f)); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* @param bytes byte array |
||||||
|
* @return Return double made from passed bytes. |
||||||
|
*/ |
||||||
|
public static double toDouble(final byte [] bytes) { |
||||||
|
return toDouble(bytes, 0); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* @param bytes byte array |
||||||
|
* @param offset offset where double is |
||||||
|
* @return Return double made from passed bytes. |
||||||
|
*/ |
||||||
|
public static double toDouble(final byte [] bytes, final int offset) { |
||||||
|
return Double.longBitsToDouble(toLong(bytes, offset, SIZEOF_LONG)); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* @param bytes byte array |
||||||
|
* @param offset offset to write to |
||||||
|
* @param d value |
||||||
|
* @return New offset into array <code>bytes</code> |
||||||
|
*/ |
||||||
|
public static int putDouble(byte [] bytes, int offset, double d) { |
||||||
|
return putLong(bytes, offset, Double.doubleToLongBits(d)); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Serialize a double as the IEEE 754 double format output. The resultant |
||||||
|
* array will be 8 bytes long. |
||||||
|
* |
||||||
|
* @param d value |
||||||
|
* @return the double represented as byte [] |
||||||
|
*/ |
||||||
|
public static byte [] toBytes(final double d) { |
||||||
|
// Encode it as a long
|
||||||
|
return Bytes.toBytes(Double.doubleToRawLongBits(d)); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Convert an int value to a byte array |
||||||
|
* @param val value |
||||||
|
* @return the byte array |
||||||
|
*/ |
||||||
|
public static byte[] toBytes(int val) { |
||||||
|
byte [] b = new byte[4]; |
||||||
|
for(int i = 3; i > 0; i--) { |
||||||
|
b[i] = (byte) val; |
||||||
|
val >>>= 8; |
||||||
|
} |
||||||
|
b[0] = (byte) val; |
||||||
|
return b; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Converts a byte array to an int value |
||||||
|
* @param bytes byte array |
||||||
|
* @return the int value |
||||||
|
*/ |
||||||
|
public static int toInt(byte[] bytes) { |
||||||
|
return toInt(bytes, 0, SIZEOF_INT); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Converts a byte array to an int value |
||||||
|
* @param bytes byte array |
||||||
|
* @param offset offset into array |
||||||
|
* @return the int value |
||||||
|
*/ |
||||||
|
public static int toInt(byte[] bytes, int offset) { |
||||||
|
return toInt(bytes, offset, SIZEOF_INT); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Converts a byte array to an int value |
||||||
|
* @param bytes byte array |
||||||
|
* @param offset offset into array |
||||||
|
* @param length length of int (has to be {@link #SIZEOF_INT}) |
||||||
|
* @return the int value |
||||||
|
* @throws IllegalArgumentException if length is not {@link #SIZEOF_INT} or |
||||||
|
* if there's not enough room in the array at the offset indicated. |
||||||
|
*/ |
||||||
|
public static int toInt(byte[] bytes, int offset, final int length) { |
||||||
|
if (length != SIZEOF_INT || offset + length > bytes.length) { |
||||||
|
throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_INT); |
||||||
|
} |
||||||
|
int n = 0; |
||||||
|
for(int i = offset; i < (offset + length); i++) { |
||||||
|
n <<= 8; |
||||||
|
n ^= bytes[i] & 0xFF; |
||||||
|
} |
||||||
|
return n; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Put an int value out to the specified byte array position. |
||||||
|
* @param bytes the byte array |
||||||
|
* @param offset position in the array |
||||||
|
* @param val int to write out |
||||||
|
* @return incremented offset |
||||||
|
* @throws IllegalArgumentException if the byte array given doesn't have |
||||||
|
* enough room at the offset specified. |
||||||
|
*/ |
||||||
|
public static int putInt(byte[] bytes, int offset, int val) { |
||||||
|
if (bytes.length - offset < SIZEOF_INT) { |
||||||
|
throw new IllegalArgumentException("Not enough room to put an int at" |
||||||
|
+ " offset " + offset + " in a " + bytes.length + " byte array"); |
||||||
|
} |
||||||
|
for(int i= offset + 3; i > offset; i--) { |
||||||
|
bytes[i] = (byte) val; |
||||||
|
val >>>= 8; |
||||||
|
} |
||||||
|
bytes[offset] = (byte) val; |
||||||
|
return offset + SIZEOF_INT; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Convert a short value to a byte array of {@link #SIZEOF_SHORT} bytes long. |
||||||
|
* @param val value |
||||||
|
* @return the byte array |
||||||
|
*/ |
||||||
|
public static byte[] toBytes(short val) { |
||||||
|
byte[] b = new byte[SIZEOF_SHORT]; |
||||||
|
b[1] = (byte) val; |
||||||
|
val >>= 8; |
||||||
|
b[0] = (byte) val; |
||||||
|
return b; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Converts a byte array to a short value |
||||||
|
* @param bytes byte array |
||||||
|
* @return the short value |
||||||
|
*/ |
||||||
|
public static short toShort(byte[] bytes) { |
||||||
|
return toShort(bytes, 0, SIZEOF_SHORT); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Converts a byte array to a short value |
||||||
|
* @param bytes byte array |
||||||
|
* @param offset offset into array |
||||||
|
* @return the short value |
||||||
|
*/ |
||||||
|
public static short toShort(byte[] bytes, int offset) { |
||||||
|
return toShort(bytes, offset, SIZEOF_SHORT); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Converts a byte array to a short value |
||||||
|
* @param bytes byte array |
||||||
|
* @param offset offset into array |
||||||
|
* @param length length, has to be {@link #SIZEOF_SHORT} |
||||||
|
* @return the short value |
||||||
|
* @throws IllegalArgumentException if length is not {@link #SIZEOF_SHORT} |
||||||
|
* or if there's not enough room in the array at the offset indicated. |
||||||
|
*/ |
||||||
|
public static short toShort(byte[] bytes, int offset, final int length) { |
||||||
|
if (length != SIZEOF_SHORT || offset + length > bytes.length) { |
||||||
|
throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_SHORT); |
||||||
|
} |
||||||
|
short n = 0; |
||||||
|
n ^= bytes[offset] & 0xFF; |
||||||
|
n <<= 8; |
||||||
|
n ^= bytes[offset+1] & 0xFF; |
||||||
|
return n; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* This method will get a sequence of bytes from pos -> limit, |
||||||
|
* but will restore pos after. |
||||||
|
* @param buf |
||||||
|
* @return byte array |
||||||
|
*/ |
||||||
|
public static byte[] getBytes(ByteBuffer buf) { |
||||||
|
int savedPos = buf.position(); |
||||||
|
byte [] newBytes = new byte[buf.remaining()]; |
||||||
|
buf.get(newBytes); |
||||||
|
buf.position(savedPos); |
||||||
|
return newBytes; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Put a short value out to the specified byte array position. |
||||||
|
* @param bytes the byte array |
||||||
|
* @param offset position in the array |
||||||
|
* @param val short to write out |
||||||
|
* @return incremented offset |
||||||
|
* @throws IllegalArgumentException if the byte array given doesn't have |
||||||
|
* enough room at the offset specified. |
||||||
|
*/ |
||||||
|
public static int putShort(byte[] bytes, int offset, short val) { |
||||||
|
if (bytes.length - offset < SIZEOF_SHORT) { |
||||||
|
throw new IllegalArgumentException("Not enough room to put a short at" |
||||||
|
+ " offset " + offset + " in a " + bytes.length + " byte array"); |
||||||
|
} |
||||||
|
bytes[offset+1] = (byte) val; |
||||||
|
val >>= 8; |
||||||
|
bytes[offset] = (byte) val; |
||||||
|
return offset + SIZEOF_SHORT; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Convert a BigDecimal value to a byte array |
||||||
|
* |
||||||
|
* @param val |
||||||
|
* @return the byte array |
||||||
|
*/ |
||||||
|
public static byte[] toBytes(BigDecimal val) { |
||||||
|
byte[] valueBytes = val.unscaledValue().toByteArray(); |
||||||
|
byte[] result = new byte[valueBytes.length + SIZEOF_INT]; |
||||||
|
int offset = putInt(result, 0, val.scale()); |
||||||
|
putBytes(result, offset, valueBytes, 0, valueBytes.length); |
||||||
|
return result; |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* Converts a byte array to a BigDecimal |
||||||
|
* |
||||||
|
* @param bytes |
||||||
|
* @return the char value |
||||||
|
*/ |
||||||
|
public static BigDecimal toBigDecimal(byte[] bytes) { |
||||||
|
return toBigDecimal(bytes, 0, bytes.length); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Converts a byte array to a BigDecimal value |
||||||
|
* |
||||||
|
* @param bytes |
||||||
|
* @param offset |
||||||
|
* @param length |
||||||
|
* @return the char value |
||||||
|
*/ |
||||||
|
public static BigDecimal toBigDecimal(byte[] bytes, int offset, final int length) { |
||||||
|
if (bytes == null || length < SIZEOF_INT + 1 || |
||||||
|
(offset + length > bytes.length)) { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
int scale = toInt(bytes, offset); |
||||||
|
byte[] tcBytes = new byte[length - SIZEOF_INT]; |
||||||
|
System.arraycopy(bytes, offset + SIZEOF_INT, tcBytes, 0, length - SIZEOF_INT); |
||||||
|
return new BigDecimal(new BigInteger(tcBytes), scale); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Put a BigDecimal value out to the specified byte array position. |
||||||
|
* |
||||||
|
* @param bytes the byte array |
||||||
|
* @param offset position in the array |
||||||
|
* @param val BigDecimal to write out |
||||||
|
* @return incremented offset |
||||||
|
*/ |
||||||
|
public static int putBigDecimal(byte[] bytes, int offset, BigDecimal val) { |
||||||
|
if (bytes == null) { |
||||||
|
return offset; |
||||||
|
} |
||||||
|
|
||||||
|
byte[] valueBytes = val.unscaledValue().toByteArray(); |
||||||
|
byte[] result = new byte[valueBytes.length + SIZEOF_INT]; |
||||||
|
offset = putInt(result, offset, val.scale()); |
||||||
|
return putBytes(result, offset, valueBytes, 0, valueBytes.length); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* @param a lower half |
||||||
|
* @param b upper half |
||||||
|
* @return New array that has a in lower half and b in upper half. |
||||||
|
*/ |
||||||
|
public static byte [] add(final byte [] a, final byte [] b) { |
||||||
|
return add(a, b, EMPTY_BYTE_ARRAY); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* @param a first third |
||||||
|
* @param b second third |
||||||
|
* @param c third third |
||||||
|
* @return New array made from a, b and c |
||||||
|
*/ |
||||||
|
public static byte [] add(final byte [] a, final byte [] b, final byte [] c) { |
||||||
|
byte [] result = new byte[a.length + b.length + c.length]; |
||||||
|
System.arraycopy(a, 0, result, 0, a.length); |
||||||
|
System.arraycopy(b, 0, result, a.length, b.length); |
||||||
|
System.arraycopy(c, 0, result, a.length + b.length, c.length); |
||||||
|
return result; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* @param a array |
||||||
|
* @param length amount of bytes to grab |
||||||
|
* @return First <code>length</code> bytes from <code>a</code> |
||||||
|
*/ |
||||||
|
public static byte [] head(final byte [] a, final int length) { |
||||||
|
if (a.length < length) { |
||||||
|
return null; |
||||||
|
} |
||||||
|
byte [] result = new byte[length]; |
||||||
|
System.arraycopy(a, 0, result, 0, length); |
||||||
|
return result; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* @param a array |
||||||
|
* @param length amount of bytes to snarf |
||||||
|
* @return Last <code>length</code> bytes from <code>a</code> |
||||||
|
*/ |
||||||
|
public static byte [] tail(final byte [] a, final int length) { |
||||||
|
if (a.length < length) { |
||||||
|
return null; |
||||||
|
} |
||||||
|
byte [] result = new byte[length]; |
||||||
|
System.arraycopy(a, a.length - length, result, 0, length); |
||||||
|
return result; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* @param a array |
||||||
|
* @param length new array size |
||||||
|
* @return Value in <code>a</code> plus <code>length</code> prepended 0 bytes |
||||||
|
*/ |
||||||
|
public static byte [] padHead(final byte [] a, final int length) { |
||||||
|
byte[] padding = getPadding(length); |
||||||
|
return add(padding,a); |
||||||
|
} |
||||||
|
|
||||||
|
private static byte[] getPadding(int length) { |
||||||
|
byte[] padding = new byte[length]; |
||||||
|
for (int i = 0; i < length; i++) { |
||||||
|
padding[i] = 0; |
||||||
|
} |
||||||
|
return padding; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* @param a array |
||||||
|
* @param length new array size |
||||||
|
* @return Value in <code>a</code> plus <code>length</code> appended 0 bytes |
||||||
|
*/ |
||||||
|
public static byte [] padTail(final byte [] a, final int length) { |
||||||
|
byte[] padding = getPadding(length); |
||||||
|
return add(a,padding); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* @param bytes array to hash |
||||||
|
* @param offset offset to start from |
||||||
|
* @param length length to hash |
||||||
|
* */ |
||||||
|
public static int hashCode(byte[] bytes, int offset, int length) { |
||||||
|
int hash = 1; |
||||||
|
for (int i = offset; i < offset + length; i++) { |
||||||
|
hash = (31 * hash) + (int) bytes[i]; |
||||||
|
} |
||||||
|
return hash; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* @param t operands |
||||||
|
* @return Array of byte arrays made from passed array of Text |
||||||
|
*/ |
||||||
|
public static byte [][] toByteArrays(final String [] t) { |
||||||
|
byte [][] result = new byte[t.length][]; |
||||||
|
for (int i = 0; i < t.length; i++) { |
||||||
|
result[i] = Bytes.toBytes(t[i]); |
||||||
|
} |
||||||
|
return result; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* @param column operand |
||||||
|
* @return A byte array of a byte array where first and only entry is |
||||||
|
* <code>column</code> |
||||||
|
*/ |
||||||
|
public static byte [][] toByteArrays(final String column) { |
||||||
|
return toByteArrays(toBytes(column)); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* @param column operand |
||||||
|
* @return A byte array of a byte array where first and only entry is |
||||||
|
* <code>column</code> |
||||||
|
*/ |
||||||
|
public static byte [][] toByteArrays(final byte [] column) { |
||||||
|
byte [][] result = new byte[1][]; |
||||||
|
result[0] = column; |
||||||
|
return result; |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
} |
@ -0,0 +1,292 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.utils; |
||||||
|
|
||||||
|
|
||||||
|
import org.apache.commons.collections.BeanMap; |
||||||
|
import org.apache.commons.lang.StringUtils; |
||||||
|
|
||||||
|
import java.util.*; |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* Provides utility methods and decorators for {@link Collection} instances. |
||||||
|
* <p> |
||||||
|
* Various utility methods might put the input objects into a Set/Map/Bag. In case |
||||||
|
* the input objects override {@link Object#equals(Object)}, it is mandatory that |
||||||
|
* the general contract of the {@link Object#hashCode()} method is maintained. |
||||||
|
* <p> |
||||||
|
* NOTE: From 4.0, method parameters will take {@link Iterable} objects when possible. |
||||||
|
* |
||||||
|
* @version $Id: CollectionUtils.java 1686855 2015-06-22 13:00:27Z tn $ |
||||||
|
* @since 1.0 |
||||||
|
*/ |
||||||
|
public class CollectionUtils { |
||||||
|
|
||||||
|
/** |
||||||
|
* Returns a new {@link Collection} containing <i>a</i> minus a subset of |
||||||
|
* <i>b</i>. Only the elements of <i>b</i> that satisfy the predicate |
||||||
|
* condition, <i>p</i> are subtracted from <i>a</i>. |
||||||
|
* <p> |
||||||
|
* <p>The cardinality of each element <i>e</i> in the returned {@link Collection} |
||||||
|
* that satisfies the predicate condition will be the cardinality of <i>e</i> in <i>a</i> |
||||||
|
* minus the cardinality of <i>e</i> in <i>b</i>, or zero, whichever is greater.</p> |
||||||
|
* <p>The cardinality of each element <i>e</i> in the returned {@link Collection} that does <b>not</b> |
||||||
|
* satisfy the predicate condition will be equal to the cardinality of <i>e</i> in <i>a</i>.</p> |
||||||
|
* |
||||||
|
* @param a the collection to subtract from, must not be null |
||||||
|
* @param b the collection to subtract, must not be null |
||||||
|
* @return a new collection with the results |
||||||
|
* @see Collection#removeAll |
||||||
|
*/ |
||||||
|
public static <T> Collection<T> subtract(Set<T> a, Set<T> b) { |
||||||
|
return org.apache.commons.collections4.CollectionUtils.subtract(a, b); |
||||||
|
} |
||||||
|
|
||||||
|
public static boolean isNotEmpty(Collection coll) { |
||||||
|
return !isEmpty(coll); |
||||||
|
} |
||||||
|
|
||||||
|
public static boolean isEmpty(Collection coll) { |
||||||
|
return coll == null || coll.isEmpty(); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* String to map |
||||||
|
* |
||||||
|
* @param str string |
||||||
|
* @param separator separator |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static Map<String, String> stringToMap(String str, String separator) { |
||||||
|
return stringToMap(str, separator, ""); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* String to map |
||||||
|
* |
||||||
|
* @param str string |
||||||
|
* @param separator separator |
||||||
|
* @param keyPrefix prefix |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static Map<String, String> stringToMap(String str, String separator, String keyPrefix) { |
||||||
|
if (null == str || "".equals(str)) { |
||||||
|
return null; |
||||||
|
} |
||||||
|
if (null == separator || "".equals(separator)) { |
||||||
|
return null; |
||||||
|
} |
||||||
|
String[] strings = str.split(separator); |
||||||
|
int mapLength = strings.length; |
||||||
|
if ((strings.length % 2) != 0) { |
||||||
|
mapLength = mapLength + 1; |
||||||
|
} |
||||||
|
|
||||||
|
Map<String, String> map = new HashMap<>(mapLength); |
||||||
|
for (int i = 0; i < strings.length; i++) { |
||||||
|
String[] strArray = strings[i].split("="); |
||||||
|
//strArray[0] KEY strArray[1] VALUE
|
||||||
|
if (StringUtils.isEmpty(keyPrefix)) { |
||||||
|
map.put(strArray[0], strArray[1]); |
||||||
|
} else { |
||||||
|
map.put(keyPrefix + strArray[0], strArray[1]); |
||||||
|
} |
||||||
|
} |
||||||
|
return map; |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* Helper class to easily access cardinality properties of two collections. |
||||||
|
* |
||||||
|
* @param <O> the element type |
||||||
|
*/ |
||||||
|
private static class CardinalityHelper<O> { |
||||||
|
|
||||||
|
/** |
||||||
|
* Contains the cardinality for each object in collection A. |
||||||
|
*/ |
||||||
|
final Map<O, Integer> cardinalityA; |
||||||
|
|
||||||
|
/** |
||||||
|
* Contains the cardinality for each object in collection B. |
||||||
|
*/ |
||||||
|
final Map<O, Integer> cardinalityB; |
||||||
|
|
||||||
|
/** |
||||||
|
* Create a new CardinalityHelper for two collections. |
||||||
|
* |
||||||
|
* @param a the first collection |
||||||
|
* @param b the second collection |
||||||
|
*/ |
||||||
|
public CardinalityHelper(final Iterable<? extends O> a, final Iterable<? extends O> b) { |
||||||
|
cardinalityA = CollectionUtils.<O>getCardinalityMap(a); |
||||||
|
cardinalityB = CollectionUtils.<O>getCardinalityMap(b); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Returns the maximum frequency of an object. |
||||||
|
* |
||||||
|
* @param obj the object |
||||||
|
* @return the maximum frequency of the object |
||||||
|
*/ |
||||||
|
public final int max(final Object obj) { |
||||||
|
return Math.max(freqA(obj), freqB(obj)); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Returns the minimum frequency of an object. |
||||||
|
* |
||||||
|
* @param obj the object |
||||||
|
* @return the minimum frequency of the object |
||||||
|
*/ |
||||||
|
public final int min(final Object obj) { |
||||||
|
return Math.min(freqA(obj), freqB(obj)); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Returns the frequency of this object in collection A. |
||||||
|
* |
||||||
|
* @param obj the object |
||||||
|
* @return the frequency of the object in collection A |
||||||
|
*/ |
||||||
|
public int freqA(final Object obj) { |
||||||
|
return getFreq(obj, cardinalityA); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Returns the frequency of this object in collection B. |
||||||
|
* |
||||||
|
* @param obj the object |
||||||
|
* @return the frequency of the object in collection B |
||||||
|
*/ |
||||||
|
public int freqB(final Object obj) { |
||||||
|
return getFreq(obj, cardinalityB); |
||||||
|
} |
||||||
|
|
||||||
|
private final int getFreq(final Object obj, final Map<?, Integer> freqMap) { |
||||||
|
final Integer count = freqMap.get(obj); |
||||||
|
if (count != null) { |
||||||
|
return count.intValue(); |
||||||
|
} |
||||||
|
return 0; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* returns {@code true} iff the given {@link Collection}s contain |
||||||
|
* exactly the same elements with exactly the same cardinalities. |
||||||
|
* |
||||||
|
* @param a the first collection |
||||||
|
* @param b the second collection |
||||||
|
* @return Returns true iff the given Collections contain exactly the same elements with exactly the same cardinalities. |
||||||
|
* That is, iff the cardinality of e in a is equal to the cardinality of e in b, for each element e in a or b. |
||||||
|
*/ |
||||||
|
public static boolean equalLists(Collection<?> a, Collection<?> b) { |
||||||
|
if (a == null && b == null) { |
||||||
|
return true; |
||||||
|
} |
||||||
|
|
||||||
|
if ((a == null && b != null) || a != null && b == null) { |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
return isEqualCollection(a, b); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Returns {@code true} iff the given {@link Collection}s contain |
||||||
|
* exactly the same elements with exactly the same cardinalities. |
||||||
|
* <p> |
||||||
|
* That is, iff the cardinality of <i>e</i> in <i>a</i> is |
||||||
|
* equal to the cardinality of <i>e</i> in <i>b</i>, |
||||||
|
* for each element <i>e</i> in <i>a</i> or <i>b</i>. |
||||||
|
* |
||||||
|
* @param a the first collection, must not be null |
||||||
|
* @param b the second collection, must not be null |
||||||
|
* @return <code>true</code> iff the collections contain the same elements with the same cardinalities. |
||||||
|
*/ |
||||||
|
public static boolean isEqualCollection(final Collection<?> a, final Collection<?> b) { |
||||||
|
if (a.size() != b.size()) { |
||||||
|
return false; |
||||||
|
} |
||||||
|
final CardinalityHelper<Object> helper = new CardinalityHelper<Object>(a, b); |
||||||
|
if (helper.cardinalityA.size() != helper.cardinalityB.size()) { |
||||||
|
return false; |
||||||
|
} |
||||||
|
for (final Object obj : helper.cardinalityA.keySet()) { |
||||||
|
if (helper.freqA(obj) != helper.freqB(obj)) { |
||||||
|
return false; |
||||||
|
} |
||||||
|
} |
||||||
|
return true; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Returns a {@link Map} mapping each unique element in the given |
||||||
|
* {@link Collection} to an {@link Integer} representing the number |
||||||
|
* of occurrences of that element in the {@link Collection}. |
||||||
|
* <p> |
||||||
|
* Only those elements present in the collection will appear as |
||||||
|
* keys in the map. |
||||||
|
* |
||||||
|
* @param <O> the type of object in the returned {@link Map}. This is a super type of <I>. |
||||||
|
* @param coll the collection to get the cardinality map for, must not be null |
||||||
|
* @return the populated cardinality map |
||||||
|
*/ |
||||||
|
public static <O> Map<O, Integer> getCardinalityMap(final Iterable<? extends O> coll) { |
||||||
|
final Map<O, Integer> count = new HashMap<O, Integer>(); |
||||||
|
for (final O obj : coll) { |
||||||
|
final Integer c = count.get(obj); |
||||||
|
if (c == null) { |
||||||
|
count.put(obj, Integer.valueOf(1)); |
||||||
|
} else { |
||||||
|
count.put(obj, Integer.valueOf(c.intValue() + 1)); |
||||||
|
} |
||||||
|
} |
||||||
|
return count; |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* Removes certain attributes of each object in the list |
||||||
|
* @param originList |
||||||
|
* @param exclusionSet |
||||||
|
* @param <T> |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static <T extends Object> List<Map<String, Object>> getListByExclusion(List<T> originList, Set<String> exclusionSet) { |
||||||
|
List<Map<String, Object>> instanceList = new ArrayList<>(); |
||||||
|
Map<String, Object> instanceMap; |
||||||
|
for (T instance : originList) { |
||||||
|
Map<String, Object> dataMap = new BeanMap(instance); |
||||||
|
instanceMap = new LinkedHashMap<>(16,0.75f,true); |
||||||
|
for (Object key : dataMap.keySet()) { |
||||||
|
|
||||||
|
if (exclusionSet.contains(key.toString())) { |
||||||
|
continue; |
||||||
|
} |
||||||
|
instanceMap.put(key.toString(), dataMap.get(key)); |
||||||
|
|
||||||
|
} |
||||||
|
instanceList.add(instanceMap); |
||||||
|
} |
||||||
|
return instanceList; |
||||||
|
} |
||||||
|
|
||||||
|
} |
@ -0,0 +1,73 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.utils; |
||||||
|
|
||||||
|
import cn.escheduler.common.Constants; |
||||||
|
import org.apache.commons.lang3.StringUtils; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
import java.io.File; |
||||||
|
|
||||||
|
import static cn.escheduler.common.Constants.*; |
||||||
|
import static cn.escheduler.common.utils.PropertyUtils.getBoolean; |
||||||
|
import static cn.escheduler.common.utils.PropertyUtils.getString; |
||||||
|
|
||||||
|
/** |
||||||
|
* common utils |
||||||
|
*/ |
||||||
|
public class CommonUtils { |
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(CommonUtils.class); |
||||||
|
|
||||||
|
/** |
||||||
|
* @return get the path of system environment variables |
||||||
|
*/ |
||||||
|
public static String getSystemEnvPath() { |
||||||
|
String envPath = getString(ESCHEDULER_ENV_PATH); |
||||||
|
if (StringUtils.isEmpty(envPath)) { |
||||||
|
envPath = System.getProperty("user.home") + File.separator + ".bash_profile"; |
||||||
|
} |
||||||
|
|
||||||
|
return envPath; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* @return get the path of Python system environment variables |
||||||
|
*/ |
||||||
|
public static String getPythonSystemEnvPath() { |
||||||
|
return getString(ESCHEDULER_ENV_PY); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* @return get queue implementation name |
||||||
|
*/ |
||||||
|
public static String getQueueImplValue(){ |
||||||
|
return getString(Constants.SCHEDULER_QUEUE_IMPL); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* |
||||||
|
* @return is develop mode |
||||||
|
*/ |
||||||
|
public static boolean isDevelopMode() { |
||||||
|
return getBoolean(DEVELOPMENT_STATE); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
} |
@ -0,0 +1,313 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.utils; |
||||||
|
|
||||||
|
import cn.escheduler.common.Constants; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
import java.text.ParseException; |
||||||
|
import java.text.SimpleDateFormat; |
||||||
|
import java.util.Calendar; |
||||||
|
import java.util.Date; |
||||||
|
|
||||||
|
/** |
||||||
|
* date utils |
||||||
|
*/ |
||||||
|
public class DateUtils { |
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(DateUtils.class); |
||||||
|
|
||||||
|
/** |
||||||
|
* @return get the formatted date string for the current time |
||||||
|
*/ |
||||||
|
public static String getCurrentTime() { |
||||||
|
return getCurrentTime(Constants.YYYY_MM_DD_HH_MM_SS); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* @param format |
||||||
|
* @return get the date string in the specified format of the current time |
||||||
|
*/ |
||||||
|
public static String getCurrentTime(String format) { |
||||||
|
return new SimpleDateFormat(format).format(new Date()); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* @param date |
||||||
|
* @param format e.g. yyyy-MM-dd HH:mm:ss |
||||||
|
* @return get the formatted date string |
||||||
|
*/ |
||||||
|
public static String format(Date date, String format) { |
||||||
|
return new SimpleDateFormat(format).format(date); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* @param date |
||||||
|
* @return convert time to yyyy-MM-dd HH:mm:ss format |
||||||
|
*/ |
||||||
|
public static String dateToString(Date date){ |
||||||
|
return format(date,Constants.YYYY_MM_DD_HH_MM_SS); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* @param date |
||||||
|
* @return convert string to date and time |
||||||
|
*/ |
||||||
|
public static Date parse(String date,String format){ |
||||||
|
try { |
||||||
|
return new SimpleDateFormat(format).parse(date); |
||||||
|
} catch (ParseException e) { |
||||||
|
logger.error("error while parse date:" + date, e); |
||||||
|
} |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* convert date str to yyyy-MM-dd HH:mm:ss format |
||||||
|
* @param str |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static Date stringToDate(String str){ |
||||||
|
return parse(str,Constants.YYYY_MM_DD_HH_MM_SS); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get seconds between two dates |
||||||
|
* |
||||||
|
* @param d1 |
||||||
|
* @param d2 |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static long differSec(Date d1, Date d2) { |
||||||
|
return (long) Math.ceil(differMs(d1, d2) / 1000.0); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get ms between two dates |
||||||
|
* |
||||||
|
* @param d1 |
||||||
|
* @param d2 |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static long differMs(Date d1, Date d2) { |
||||||
|
return Math.abs(d1.getTime() - d2.getTime()); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* get hours between two dates |
||||||
|
* |
||||||
|
* @param d1 |
||||||
|
* @param d2 |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static long diffHours(Date d1, Date d2) { |
||||||
|
return (long) Math.ceil(diffMin(d1, d2) / 60.0); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get minutes between two dates |
||||||
|
* |
||||||
|
* @param d1 |
||||||
|
* @param d2 |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static long diffMin(Date d1, Date d2) { |
||||||
|
return (long) Math.ceil(differSec(d1, d2) / 60.0); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* get the date of the specified date in the days before and after |
||||||
|
* @param date |
||||||
|
* @param day |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static Date getSomeDay(Date date, int day) { |
||||||
|
Calendar calendar = Calendar.getInstance(); |
||||||
|
calendar.setTime(date); |
||||||
|
calendar.add(Calendar.DATE, day); |
||||||
|
return calendar.getTime(); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* compare two dates |
||||||
|
* |
||||||
|
* @param future |
||||||
|
* @param old |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static boolean compare(Date future, Date old) { |
||||||
|
return future.getTime() > old.getTime(); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* convert schedule string to date |
||||||
|
* @param schedule |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static Date getScheduleDate(String schedule){ |
||||||
|
return stringToDate(schedule); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* format time to readable |
||||||
|
* |
||||||
|
* @param ms |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static String format2Readable(long ms) { |
||||||
|
|
||||||
|
long days = ms / (1000 * 60 * 60 * 24); |
||||||
|
long hours = (ms % (1000 * 60 * 60 * 24)) / (1000 * 60 * 60); |
||||||
|
long minutes = (ms % (1000 * 60 * 60)) / (1000 * 60); |
||||||
|
long seconds = (ms % (1000 * 60)) / 1000; |
||||||
|
|
||||||
|
return String.format("%02d %02d:%02d:%02d", days, hours, minutes, seconds); |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get monday |
||||||
|
* |
||||||
|
* note: Set the first day of the week to Monday, the default is Sunday |
||||||
|
*/ |
||||||
|
public static Date getMonday(Date date) { |
||||||
|
Calendar cal = Calendar.getInstance(); |
||||||
|
|
||||||
|
cal.setTime(date); |
||||||
|
|
||||||
|
cal.setFirstDayOfWeek(Calendar.MONDAY); |
||||||
|
cal.set(Calendar.DAY_OF_WEEK, Calendar.MONDAY); |
||||||
|
|
||||||
|
return cal.getTime(); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get sunday |
||||||
|
* |
||||||
|
* note: Set the first day of the week to Monday, the default is Sunday |
||||||
|
*/ |
||||||
|
public static Date getSunday(Date date) { |
||||||
|
Calendar cal = Calendar.getInstance(); |
||||||
|
cal.setTime(date); |
||||||
|
|
||||||
|
cal.setFirstDayOfWeek(Calendar.MONDAY); |
||||||
|
cal.set(Calendar.DAY_OF_WEEK, Calendar.SUNDAY); |
||||||
|
|
||||||
|
return cal.getTime(); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get first day of month |
||||||
|
*/ |
||||||
|
public static Date getFirstDayOfMonth(Date date) { |
||||||
|
Calendar cal = Calendar.getInstance(); |
||||||
|
|
||||||
|
cal.setTime(date); |
||||||
|
cal.set(Calendar.DAY_OF_MONTH, 1); |
||||||
|
|
||||||
|
return cal.getTime(); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get first day of month |
||||||
|
*/ |
||||||
|
public static Date getSomeHourOfDay(Date date, int hours) { |
||||||
|
Calendar cal = Calendar.getInstance(); |
||||||
|
|
||||||
|
cal.setTime(date); |
||||||
|
cal.set(Calendar.HOUR_OF_DAY, cal.get(Calendar.HOUR_OF_DAY) - hours); |
||||||
|
cal.set(Calendar.MINUTE, 0); |
||||||
|
cal.set(Calendar.SECOND, 0); |
||||||
|
|
||||||
|
return cal.getTime(); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get last day of month |
||||||
|
*/ |
||||||
|
public static Date getLastDayOfMonth(Date date) { |
||||||
|
Calendar cal = Calendar.getInstance(); |
||||||
|
|
||||||
|
cal.setTime(date); |
||||||
|
|
||||||
|
cal.add(Calendar.MONTH, 1); |
||||||
|
cal.set(Calendar.DAY_OF_MONTH, 1); |
||||||
|
cal.add(Calendar.DAY_OF_MONTH, -1); |
||||||
|
|
||||||
|
return cal.getTime(); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* return YYYY-MM-DD 00:00:00 |
||||||
|
* @param inputDay |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static Date getStartOfDay(Date inputDay){ |
||||||
|
Calendar cal = Calendar.getInstance(); |
||||||
|
cal.setTime(inputDay); |
||||||
|
cal.set(Calendar.HOUR_OF_DAY, 0); |
||||||
|
cal.set(Calendar.MINUTE, 0); |
||||||
|
cal.set(Calendar.SECOND, 0); |
||||||
|
return cal.getTime(); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* return YYYY-MM-DD 23:59:59 |
||||||
|
* @param inputDay |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static Date getEndOfDay(Date inputDay){ |
||||||
|
Calendar cal = Calendar.getInstance(); |
||||||
|
cal.setTime(inputDay); |
||||||
|
cal.set(Calendar.HOUR_OF_DAY, 23); |
||||||
|
cal.set(Calendar.MINUTE, 59); |
||||||
|
cal.set(Calendar.SECOND, 59); |
||||||
|
return cal.getTime(); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* return YYYY-MM-DD 00:00:00 |
||||||
|
* @param inputDay |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static Date getStartOfHour(Date inputDay){ |
||||||
|
Calendar cal = Calendar.getInstance(); |
||||||
|
cal.setTime(inputDay); |
||||||
|
cal.set(Calendar.MINUTE, 0); |
||||||
|
cal.set(Calendar.SECOND, 0); |
||||||
|
return cal.getTime(); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* return YYYY-MM-DD 23:59:59 |
||||||
|
* @param inputDay |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static Date getEndOfHour(Date inputDay){ |
||||||
|
Calendar cal = Calendar.getInstance(); |
||||||
|
cal.setTime(inputDay); |
||||||
|
cal.set(Calendar.MINUTE, 59); |
||||||
|
cal.set(Calendar.SECOND, 59); |
||||||
|
return cal.getTime(); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
} |
@ -0,0 +1,135 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.utils; |
||||||
|
|
||||||
|
import cn.escheduler.common.enums.DependResult; |
||||||
|
import cn.escheduler.common.enums.DependentRelation; |
||||||
|
import cn.escheduler.common.model.DateInterval; |
||||||
|
import cn.escheduler.common.utils.dependent.DependentDateUtils; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
import java.util.ArrayList; |
||||||
|
import java.util.Date; |
||||||
|
import java.util.List; |
||||||
|
|
||||||
|
public class DependentUtils { |
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(DependentUtils.class); |
||||||
|
|
||||||
|
public static DependResult getDependResultForRelation(DependentRelation relation, |
||||||
|
List<DependResult> dependResultList){ |
||||||
|
|
||||||
|
DependResult dependResult = DependResult.SUCCESS; |
||||||
|
|
||||||
|
switch (relation){ |
||||||
|
case AND: |
||||||
|
if(dependResultList.contains(DependResult.FAILED)){ |
||||||
|
dependResult = DependResult.FAILED; |
||||||
|
}else if(dependResultList.contains(DependResult.WAITING)){ |
||||||
|
dependResult = DependResult.WAITING; |
||||||
|
}else{ |
||||||
|
dependResult = DependResult.SUCCESS; |
||||||
|
} |
||||||
|
break; |
||||||
|
case OR: |
||||||
|
if(dependResultList.contains(DependResult.SUCCESS)){ |
||||||
|
dependResult = DependResult.SUCCESS; |
||||||
|
}else if(dependResultList.contains(DependResult.WAITING)){ |
||||||
|
dependResult = DependResult.WAITING; |
||||||
|
}else{ |
||||||
|
dependResult = DependResult.FAILED; |
||||||
|
} |
||||||
|
break; |
||||||
|
default: |
||||||
|
break; |
||||||
|
} |
||||||
|
return dependResult; |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* get date interval list by business date and date value. |
||||||
|
* @param businessDate |
||||||
|
* @param dateValue |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static List<DateInterval> getDateIntervalList(Date businessDate, String dateValue){ |
||||||
|
List<DateInterval> result = new ArrayList<>(); |
||||||
|
switch (dateValue){ |
||||||
|
case "last1Hour": |
||||||
|
result = DependentDateUtils.getLastHoursInterval(businessDate, 1); |
||||||
|
break; |
||||||
|
case "last2Hours": |
||||||
|
result = DependentDateUtils.getLastHoursInterval(businessDate, 2); |
||||||
|
break; |
||||||
|
case "last3Hours": |
||||||
|
result = DependentDateUtils.getLastHoursInterval(businessDate, 3); |
||||||
|
break; |
||||||
|
case "last1Days": |
||||||
|
result = DependentDateUtils.getLastDayInterval(businessDate, 1); |
||||||
|
break; |
||||||
|
case "last2Days": |
||||||
|
result = DependentDateUtils.getLastDayInterval(businessDate, 2); |
||||||
|
break; |
||||||
|
case "last3Days": |
||||||
|
result = DependentDateUtils.getLastDayInterval(businessDate, 3); |
||||||
|
break; |
||||||
|
case "last7Days": |
||||||
|
result = DependentDateUtils.getLastDayInterval(businessDate, 7); |
||||||
|
break; |
||||||
|
case "lastWeek": |
||||||
|
result = DependentDateUtils.getLastWeekInterval(businessDate); |
||||||
|
break; |
||||||
|
case "lastMonday": |
||||||
|
result = DependentDateUtils.getLastWeekOneDayInterval(businessDate, 1); |
||||||
|
break; |
||||||
|
case "lastTuesday": |
||||||
|
result = DependentDateUtils.getLastWeekOneDayInterval(businessDate, 2); |
||||||
|
break; |
||||||
|
case "lastWednesday": |
||||||
|
result = DependentDateUtils.getLastWeekOneDayInterval(businessDate, 3); |
||||||
|
break; |
||||||
|
case "lastThursday": |
||||||
|
result = DependentDateUtils.getLastWeekOneDayInterval(businessDate, 4); |
||||||
|
break; |
||||||
|
case "lastFriday": |
||||||
|
result = DependentDateUtils.getLastWeekOneDayInterval(businessDate, 5); |
||||||
|
break; |
||||||
|
case "lastSaturday": |
||||||
|
result = DependentDateUtils.getLastWeekOneDayInterval(businessDate, 6); |
||||||
|
break; |
||||||
|
case "lastSunday": |
||||||
|
result = DependentDateUtils.getLastWeekOneDayInterval(businessDate, 7); |
||||||
|
break; |
||||||
|
case "lastMonth": |
||||||
|
result = DependentDateUtils.getLastMonthInterval(businessDate); |
||||||
|
break; |
||||||
|
case "lastMonthBegin": |
||||||
|
result = DependentDateUtils.getLastMonthBeginInterval(businessDate, true); |
||||||
|
break; |
||||||
|
case "lastMonthEnd": |
||||||
|
result = DependentDateUtils.getLastMonthBeginInterval(businessDate, false); |
||||||
|
break; |
||||||
|
default: |
||||||
|
break; |
||||||
|
} |
||||||
|
return result; |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
} |
@ -0,0 +1,37 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.utils; |
||||||
|
|
||||||
|
import org.apache.commons.codec.digest.DigestUtils; |
||||||
|
import org.apache.commons.lang3.StringUtils; |
||||||
|
|
||||||
|
/** |
||||||
|
* encryption utils |
||||||
|
*/ |
||||||
|
public class EncryptionUtils { |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* |
||||||
|
* @param rawStr |
||||||
|
* @return md5(rawStr) |
||||||
|
*/ |
||||||
|
public static String getMd5(String rawStr) { |
||||||
|
return DigestUtils.md5Hex(null == rawStr ? StringUtils.EMPTY : rawStr); |
||||||
|
} |
||||||
|
|
||||||
|
} |
@ -0,0 +1,36 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.utils; |
||||||
|
|
||||||
|
import org.apache.ibatis.type.EnumOrdinalTypeHandler; |
||||||
|
|
||||||
|
/** |
||||||
|
* enum field util |
||||||
|
*/ |
||||||
|
public class EnumFieldUtil { |
||||||
|
/** |
||||||
|
* Generate a string for the enums field |
||||||
|
* |
||||||
|
* @param field |
||||||
|
* @param enumClass |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static String genFieldStr(String field, Class<?> enumClass) { |
||||||
|
return "#{" + field + ",javaType=" + enumClass.getName() + ",typeHandler=" + EnumOrdinalTypeHandler.class.getName() + "}"; |
||||||
|
} |
||||||
|
|
||||||
|
} |
@ -0,0 +1,372 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.utils; |
||||||
|
|
||||||
|
import org.apache.commons.io.Charsets; |
||||||
|
import org.apache.commons.io.IOUtils; |
||||||
|
import org.apache.commons.lang.StringUtils; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
import java.io.*; |
||||||
|
import java.nio.charset.Charset; |
||||||
|
import java.nio.charset.UnsupportedCharsetException; |
||||||
|
|
||||||
|
import static cn.escheduler.common.Constants.*; |
||||||
|
import static cn.escheduler.common.utils.PropertyUtils.getString; |
||||||
|
|
||||||
|
/** |
||||||
|
* file utils |
||||||
|
*/ |
||||||
|
public class FileUtils { |
||||||
|
public static final Logger logger = LoggerFactory.getLogger(FileUtils.class); |
||||||
|
|
||||||
|
/** |
||||||
|
* get file suffix |
||||||
|
* |
||||||
|
* @param filename |
||||||
|
* @return file suffix |
||||||
|
*/ |
||||||
|
public static String suffix(String filename) { |
||||||
|
|
||||||
|
String fileSuffix = ""; |
||||||
|
if (StringUtils.isNotEmpty(filename)) { |
||||||
|
int lastIndex = filename.lastIndexOf("."); |
||||||
|
if (lastIndex > 0) { |
||||||
|
fileSuffix = filename.substring(lastIndex + 1); |
||||||
|
} |
||||||
|
} |
||||||
|
return fileSuffix; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get download file absolute path and name |
||||||
|
* |
||||||
|
* @param filename |
||||||
|
* @return download file name |
||||||
|
*/ |
||||||
|
public static String getDownloadFilename(String filename) { |
||||||
|
return String.format("%s/%s/%s", getString(DATA_DOWNLOAD_BASEDIR_PATH), DateUtils.getCurrentTime(YYYYMMDDHHMMSS), filename); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get upload file absolute path and name |
||||||
|
* |
||||||
|
* @param tenantCode tenant code |
||||||
|
* @param filename file name |
||||||
|
* @return local file path |
||||||
|
*/ |
||||||
|
public static String getUploadFilename(String tenantCode, String filename) { |
||||||
|
return String.format("%s/%s/resources/%s",getString(DATA_BASEDIR_PATH), tenantCode, filename); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* directory of process execution |
||||||
|
* @param projectId |
||||||
|
* @param processDefineId |
||||||
|
* @param processInstanceId |
||||||
|
* @param taskInstanceId |
||||||
|
* @return directory of process execution |
||||||
|
*/ |
||||||
|
public static String getProcessExecDir(int projectId, int processDefineId, int processInstanceId, int taskInstanceId) { |
||||||
|
|
||||||
|
return String.format("%s/process/%s/%s/%s/%s", getString(PROCESS_EXEC_BASEPATH), Integer.toString(projectId), |
||||||
|
Integer.toString(processDefineId), Integer.toString(processInstanceId),Integer.toString(taskInstanceId)); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* directory of process instances |
||||||
|
* @param projectId |
||||||
|
* @param processDefineId |
||||||
|
* @param processInstanceId |
||||||
|
* @return directory of process instances |
||||||
|
*/ |
||||||
|
public static String getProcessExecDir(int projectId, int processDefineId, int processInstanceId) { |
||||||
|
return String.format("%s/process/%s/%s/%s", getString(PROCESS_EXEC_BASEPATH), Integer.toString(projectId), |
||||||
|
Integer.toString(processDefineId), Integer.toString(processInstanceId)); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* @return get suffixes for resource files that support online viewing |
||||||
|
*/ |
||||||
|
public static String getResourceViewSuffixs() { |
||||||
|
return getString(RESOURCE_VIEW_SUFFIXS); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* create directory and user |
||||||
|
* @param execLocalPath |
||||||
|
* @param userName |
||||||
|
* @param logger |
||||||
|
* @throws IOException |
||||||
|
*/ |
||||||
|
public static void createWorkDirAndUserIfAbsent(String execLocalPath, String userName, Logger logger) throws IOException{ |
||||||
|
//if work dir exists, first delete
|
||||||
|
File execLocalPathFile = new File(execLocalPath); |
||||||
|
|
||||||
|
if (execLocalPathFile.exists()){ |
||||||
|
org.apache.commons.io.FileUtils.forceDelete(execLocalPathFile); |
||||||
|
} |
||||||
|
|
||||||
|
//create work dir
|
||||||
|
org.apache.commons.io.FileUtils.forceMkdir(execLocalPathFile); |
||||||
|
|
||||||
|
|
||||||
|
//if not exists this user,then create
|
||||||
|
if (!OSUtils.getUserList().contains(userName)){ |
||||||
|
String userGroup = OSUtils.getGroup(); |
||||||
|
if (org.apache.commons.lang3.StringUtils.isNotEmpty(userGroup)){ |
||||||
|
logger.info("create os user : {}",userName); |
||||||
|
String cmd = String.format("sudo useradd -g %s %s",userGroup,userName); |
||||||
|
|
||||||
|
logger.info("execute cmd : {}",cmd); |
||||||
|
OSUtils.exeCmd(cmd); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* write content to file ,if parent path not exists, it will do one's utmost to mkdir |
||||||
|
* |
||||||
|
* @param content content |
||||||
|
* @param filePath target file path |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static boolean writeContent2File(String content, String filePath) { |
||||||
|
boolean flag = true; |
||||||
|
BufferedReader bufferedReader = null; |
||||||
|
BufferedWriter bufferedWriter = null; |
||||||
|
try { |
||||||
|
File distFile = new File(filePath); |
||||||
|
if (!distFile.getParentFile().exists()) { |
||||||
|
distFile.getParentFile().mkdirs(); |
||||||
|
} |
||||||
|
bufferedReader = new BufferedReader(new StringReader(content)); |
||||||
|
bufferedWriter = new BufferedWriter(new FileWriter(distFile)); |
||||||
|
char buf[] = new char[1024]; |
||||||
|
int len; |
||||||
|
while ((len = bufferedReader.read(buf)) != -1) { |
||||||
|
bufferedWriter.write(buf, 0, len); |
||||||
|
} |
||||||
|
bufferedWriter.flush(); |
||||||
|
bufferedReader.close(); |
||||||
|
bufferedWriter.close(); |
||||||
|
} catch (IOException e) { |
||||||
|
FileUtils.logger.error(e.getMessage(), e); |
||||||
|
flag = false; |
||||||
|
return flag; |
||||||
|
} finally { |
||||||
|
IOUtils.closeQuietly(bufferedWriter); |
||||||
|
IOUtils.closeQuietly(bufferedReader); |
||||||
|
} |
||||||
|
return flag; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Writes a String to a file creating the file if it does not exist. |
||||||
|
* |
||||||
|
* NOTE: As from v1.3, the parent directories of the file will be created |
||||||
|
* if they do not exist. |
||||||
|
* |
||||||
|
* @param file the file to write |
||||||
|
* @param data the content to write to the file |
||||||
|
* @param encoding the encoding to use, {@code null} means platform default |
||||||
|
* @throws IOException in case of an I/O error |
||||||
|
* @throws java.io.UnsupportedEncodingException if the encoding is not supported by the VM |
||||||
|
* @since 2.4 |
||||||
|
*/ |
||||||
|
public static void writeStringToFile(File file, String data, Charset encoding) throws IOException { |
||||||
|
writeStringToFile(file, data, encoding, false); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Writes a String to a file creating the file if it does not exist. |
||||||
|
* |
||||||
|
* NOTE: As from v1.3, the parent directories of the file will be created |
||||||
|
* if they do not exist. |
||||||
|
* |
||||||
|
* @param file the file to write |
||||||
|
* @param data the content to write to the file |
||||||
|
* @param encoding the encoding to use, {@code null} means platform default |
||||||
|
* @throws IOException in case of an I/O error |
||||||
|
* @throws java.io.UnsupportedEncodingException if the encoding is not supported by the VM |
||||||
|
*/ |
||||||
|
public static void writeStringToFile(File file, String data, String encoding) throws IOException { |
||||||
|
writeStringToFile(file, data, encoding, false); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Writes a String to a file creating the file if it does not exist. |
||||||
|
* |
||||||
|
* @param file the file to write |
||||||
|
* @param data the content to write to the file |
||||||
|
* @param encoding the encoding to use, {@code null} means platform default |
||||||
|
* @param append if {@code true}, then the String will be added to the |
||||||
|
* end of the file rather than overwriting |
||||||
|
* @throws IOException in case of an I/O error |
||||||
|
* @since 2.3 |
||||||
|
*/ |
||||||
|
public static void writeStringToFile(File file, String data, Charset encoding, boolean append) throws IOException { |
||||||
|
OutputStream out = null; |
||||||
|
try { |
||||||
|
out = openOutputStream(file, append); |
||||||
|
IOUtils.write(data, out, encoding); |
||||||
|
out.close(); // don't swallow close Exception if copy completes normally
|
||||||
|
} finally { |
||||||
|
IOUtils.closeQuietly(out); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Writes a String to a file creating the file if it does not exist. |
||||||
|
* |
||||||
|
* @param file the file to write |
||||||
|
* @param data the content to write to the file |
||||||
|
* @param encoding the encoding to use, {@code null} means platform default |
||||||
|
* @param append if {@code true}, then the String will be added to the |
||||||
|
* end of the file rather than overwriting |
||||||
|
* @throws IOException in case of an I/O error |
||||||
|
* @throws UnsupportedCharsetException |
||||||
|
* thrown instead of {@link UnsupportedEncodingException} in version 2.2 if the encoding is not |
||||||
|
* supported by the VM |
||||||
|
* @since 2.1 |
||||||
|
*/ |
||||||
|
public static void writeStringToFile(File file, String data, String encoding, boolean append) throws IOException { |
||||||
|
writeStringToFile(file, data, Charsets.toCharset(encoding), append); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Writes a String to a file creating the file if it does not exist using the default encoding for the VM. |
||||||
|
* |
||||||
|
* @param file the file to write |
||||||
|
* @param data the content to write to the file |
||||||
|
* @throws IOException in case of an I/O error |
||||||
|
*/ |
||||||
|
public static void writeStringToFile(File file, String data) throws IOException { |
||||||
|
writeStringToFile(file, data, Charset.defaultCharset(), false); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Writes a String to a file creating the file if it does not exist using the default encoding for the VM. |
||||||
|
* |
||||||
|
* @param file the file to write |
||||||
|
* @param data the content to write to the file |
||||||
|
* @param append if {@code true}, then the String will be added to the |
||||||
|
* end of the file rather than overwriting |
||||||
|
* @throws IOException in case of an I/O error |
||||||
|
* @since 2.1 |
||||||
|
*/ |
||||||
|
public static void writeStringToFile(File file, String data, boolean append) throws IOException { |
||||||
|
writeStringToFile(file, data, Charset.defaultCharset(), append); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Opens a {@link FileOutputStream} for the specified file, checking and |
||||||
|
* creating the parent directory if it does not exist. |
||||||
|
* <p> |
||||||
|
* At the end of the method either the stream will be successfully opened, |
||||||
|
* or an exception will have been thrown. |
||||||
|
* <p> |
||||||
|
* The parent directory will be created if it does not exist. |
||||||
|
* The file will be created if it does not exist. |
||||||
|
* An exception is thrown if the file object exists but is a directory. |
||||||
|
* An exception is thrown if the file exists but cannot be written to. |
||||||
|
* An exception is thrown if the parent directory cannot be created. |
||||||
|
* |
||||||
|
* @param file the file to open for output, must not be {@code null} |
||||||
|
* @return a new {@link FileOutputStream} for the specified file |
||||||
|
* @throws IOException if the file object is a directory |
||||||
|
* @throws IOException if the file cannot be written to |
||||||
|
* @throws IOException if a parent directory needs creating but that fails |
||||||
|
* @since 1.3 |
||||||
|
*/ |
||||||
|
public static FileOutputStream openOutputStream(File file) throws IOException { |
||||||
|
return openOutputStream(file, false); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Opens a {@link FileOutputStream} for the specified file, checking and |
||||||
|
* creating the parent directory if it does not exist. |
||||||
|
* <p> |
||||||
|
* At the end of the method either the stream will be successfully opened, |
||||||
|
* or an exception will have been thrown. |
||||||
|
* <p> |
||||||
|
* The parent directory will be created if it does not exist. |
||||||
|
* The file will be created if it does not exist. |
||||||
|
* An exception is thrown if the file object exists but is a directory. |
||||||
|
* An exception is thrown if the file exists but cannot be written to. |
||||||
|
* An exception is thrown if the parent directory cannot be created. |
||||||
|
* |
||||||
|
* @param file the file to open for output, must not be {@code null} |
||||||
|
* @param append if {@code true}, then bytes will be added to the |
||||||
|
* end of the file rather than overwriting |
||||||
|
* @return a new {@link FileOutputStream} for the specified file |
||||||
|
* @throws IOException if the file object is a directory |
||||||
|
* @throws IOException if the file cannot be written to |
||||||
|
* @throws IOException if a parent directory needs creating but that fails |
||||||
|
* @since 2.1 |
||||||
|
*/ |
||||||
|
public static FileOutputStream openOutputStream(File file, boolean append) throws IOException { |
||||||
|
if (file.exists()) { |
||||||
|
if (file.isDirectory()) { |
||||||
|
throw new IOException("File '" + file + "' exists but is a directory"); |
||||||
|
} |
||||||
|
if (file.canWrite() == false) { |
||||||
|
throw new IOException("File '" + file + "' cannot be written to"); |
||||||
|
} |
||||||
|
} else { |
||||||
|
File parent = file.getParentFile(); |
||||||
|
if (parent != null) { |
||||||
|
if (!parent.mkdirs() && !parent.isDirectory()) { |
||||||
|
throw new IOException("Directory '" + parent + "' could not be created"); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
return new FileOutputStream(file, append); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* deletes a directory recursively |
||||||
|
* @param dir |
||||||
|
*/ |
||||||
|
|
||||||
|
public static void deleteDir(String dir) throws IOException { |
||||||
|
org.apache.commons.io.FileUtils.deleteDirectory(new File(dir)); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Deletes a file. If file is a directory, delete it and all sub-directories. |
||||||
|
* <p> |
||||||
|
* The difference between File.delete() and this method are: |
||||||
|
* <ul> |
||||||
|
* <li>A directory to be deleted does not have to be empty.</li> |
||||||
|
* <li>You get exceptions when a file or directory cannot be deleted. |
||||||
|
* (java.io.File methods returns a boolean)</li> |
||||||
|
* </ul> |
||||||
|
* |
||||||
|
* @param filename |
||||||
|
* @throws IOException in case deletion is unsuccessful |
||||||
|
*/ |
||||||
|
public static void deleteFile(String filename) throws IOException { |
||||||
|
org.apache.commons.io.FileUtils.forceDelete(new File(filename)); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
} |
@ -0,0 +1,486 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.utils; |
||||||
|
|
||||||
|
import cn.escheduler.common.Constants; |
||||||
|
import cn.escheduler.common.enums.ExecutionStatus; |
||||||
|
import com.alibaba.fastjson.JSON; |
||||||
|
import com.alibaba.fastjson.JSONException; |
||||||
|
import com.alibaba.fastjson.JSONObject; |
||||||
|
import org.apache.commons.io.IOUtils; |
||||||
|
import org.apache.commons.lang3.StringUtils; |
||||||
|
import org.apache.hadoop.conf.Configuration; |
||||||
|
import org.apache.hadoop.fs.FSDataInputStream; |
||||||
|
import org.apache.hadoop.fs.FileSystem; |
||||||
|
import org.apache.hadoop.fs.FileUtil; |
||||||
|
import org.apache.hadoop.fs.Path; |
||||||
|
import org.apache.hadoop.yarn.client.cli.RMAdminCLI; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
import java.io.*; |
||||||
|
import java.util.List; |
||||||
|
import java.util.stream.Collectors; |
||||||
|
import java.util.stream.Stream; |
||||||
|
|
||||||
|
import static cn.escheduler.common.Constants.*; |
||||||
|
import static cn.escheduler.common.utils.PropertyUtils.getInt; |
||||||
|
import static cn.escheduler.common.utils.PropertyUtils.getString; |
||||||
|
|
||||||
|
/** |
||||||
|
* hadoop utils |
||||||
|
* single instance |
||||||
|
*/ |
||||||
|
public class HadoopUtils implements Closeable { |
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(HadoopUtils.class); |
||||||
|
|
||||||
|
private static volatile HadoopUtils instance = new HadoopUtils(); |
||||||
|
private static volatile Configuration configuration; |
||||||
|
private static FileSystem fs; |
||||||
|
|
||||||
|
private HadoopUtils(){ |
||||||
|
init(); |
||||||
|
} |
||||||
|
|
||||||
|
public static HadoopUtils getInstance(){ |
||||||
|
return instance; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* init hadoop configuration |
||||||
|
*/ |
||||||
|
private void init() { |
||||||
|
if (configuration == null) { |
||||||
|
synchronized (HadoopUtils.class) { |
||||||
|
if (configuration == null) { |
||||||
|
try { |
||||||
|
configuration = new Configuration(); |
||||||
|
String defaultFS = configuration.get(FS_DEFAULTFS); |
||||||
|
//first get key from core-site.xml hdfs-site.xml ,if null ,then try to get from properties file
|
||||||
|
// the default is the local file system
|
||||||
|
if(defaultFS.startsWith("file")){ |
||||||
|
String defaultFSProp = getString(FS_DEFAULTFS); |
||||||
|
if(StringUtils.isNotBlank(defaultFSProp)){ |
||||||
|
configuration.set(FS_DEFAULTFS,defaultFSProp); |
||||||
|
}else{ |
||||||
|
logger.error("property:{} can not to be empty, please set!"); |
||||||
|
throw new RuntimeException("property:{} can not to be empty, please set!"); |
||||||
|
} |
||||||
|
}else{ |
||||||
|
logger.info("get property:{} -> {}, from core-site.xml hdfs-site.xml ", FS_DEFAULTFS, defaultFS); |
||||||
|
} |
||||||
|
|
||||||
|
if (fs == null) { |
||||||
|
fs = FileSystem.get(configuration); |
||||||
|
} |
||||||
|
String rmHaIds = getString(YARN_RESOURCEMANAGER_HA_RM_IDS); |
||||||
|
String appAddress = getString(Constants.YARN_APPLICATION_STATUS_ADDRESS); |
||||||
|
if (!StringUtils.isEmpty(rmHaIds)) { |
||||||
|
appAddress = getAppAddress(appAddress, rmHaIds); |
||||||
|
logger.info("appAddress : {}", appAddress); |
||||||
|
} |
||||||
|
configuration.set(Constants.YARN_APPLICATION_STATUS_ADDRESS, appAddress); |
||||||
|
} catch (Exception e) { |
||||||
|
logger.error(e.getMessage(), e); |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* @return Configuration |
||||||
|
*/ |
||||||
|
public Configuration getConfiguration() { |
||||||
|
return configuration; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get application url |
||||||
|
* |
||||||
|
* @param applicationId |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public String getApplicationUrl(String applicationId) { |
||||||
|
return String.format(configuration.get(YARN_APPLICATION_STATUS_ADDRESS), applicationId); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* cat file on hdfs |
||||||
|
* |
||||||
|
* @param hdfsFilePath hdfs file path |
||||||
|
* @return byte[] |
||||||
|
*/ |
||||||
|
public byte[] catFile(String hdfsFilePath) throws IOException { |
||||||
|
|
||||||
|
if(StringUtils.isBlank(hdfsFilePath)){ |
||||||
|
logger.error("hdfs file path:{} is blank",hdfsFilePath); |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
FSDataInputStream fsDataInputStream = fs.open(new Path(hdfsFilePath)); |
||||||
|
return IOUtils.toByteArray(fsDataInputStream); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* cat file on hdfs |
||||||
|
* |
||||||
|
* @param hdfsFilePath hdfs file path |
||||||
|
* @param skipLineNums skip line numbers |
||||||
|
* @param limit read how many lines |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public List<String> catFile(String hdfsFilePath, int skipLineNums, int limit) throws IOException { |
||||||
|
|
||||||
|
if(StringUtils.isBlank(hdfsFilePath)){ |
||||||
|
logger.error("hdfs file path:{} is blank",hdfsFilePath); |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
FSDataInputStream in = fs.open(new Path(hdfsFilePath)); |
||||||
|
BufferedReader br = new BufferedReader(new InputStreamReader(in)); |
||||||
|
Stream<String> stream = br.lines().skip(skipLineNums).limit(limit); |
||||||
|
return stream.collect(Collectors.toList()); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* make the given file and all non-existent parents into |
||||||
|
* directories. Has the semantics of Unix 'mkdir -p'. |
||||||
|
* Existence of the directory hierarchy is not an error. |
||||||
|
* |
||||||
|
* @param hdfsPath path to create |
||||||
|
*/ |
||||||
|
public boolean mkdir(String hdfsPath) throws IOException { |
||||||
|
return fs.mkdirs(new Path(hdfsPath)); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* copy files between FileSystems |
||||||
|
* |
||||||
|
* @param srcPath source hdfs path |
||||||
|
* @param dstPath destination hdfs path |
||||||
|
* @param deleteSource whether to delete the src |
||||||
|
* @param overwrite whether to overwrite an existing file |
||||||
|
* @return 是否成功 |
||||||
|
*/ |
||||||
|
public boolean copy(String srcPath, String dstPath, boolean deleteSource, boolean overwrite) throws IOException { |
||||||
|
return FileUtil.copy(fs, new Path(srcPath), fs, new Path(dstPath), deleteSource, overwrite, fs.getConf()); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* the src file is on the local disk. Add it to FS at |
||||||
|
* the given dst name. |
||||||
|
|
||||||
|
* @param srcFile local file |
||||||
|
* @param dstHdfsPath destination hdfs path |
||||||
|
* @param deleteSource whether to delete the src |
||||||
|
* @param overwrite whether to overwrite an existing file |
||||||
|
*/ |
||||||
|
public boolean copyLocalToHdfs(String srcFile, String dstHdfsPath, boolean deleteSource, boolean overwrite) throws IOException { |
||||||
|
Path srcPath = new Path(srcFile); |
||||||
|
Path dstPath= new Path(dstHdfsPath); |
||||||
|
|
||||||
|
fs.copyFromLocalFile(deleteSource, overwrite, srcPath, dstPath); |
||||||
|
|
||||||
|
return true; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* copy hdfs file to local |
||||||
|
* |
||||||
|
* @param srcHdfsFilePath source hdfs file path |
||||||
|
* @param dstFile destination file |
||||||
|
* @param deleteSource delete source |
||||||
|
* @param overwrite overwrite |
||||||
|
* @return |
||||||
|
* @throws IOException |
||||||
|
*/ |
||||||
|
public boolean copyHdfsToLocal(String srcHdfsFilePath, String dstFile, boolean deleteSource, boolean overwrite) throws IOException { |
||||||
|
Path srcPath = new Path(srcHdfsFilePath); |
||||||
|
File dstPath = new File(dstFile); |
||||||
|
|
||||||
|
if (dstPath.exists()) { |
||||||
|
if (dstPath.isFile()) { |
||||||
|
if (overwrite) { |
||||||
|
dstPath.delete(); |
||||||
|
} |
||||||
|
} else { |
||||||
|
logger.error("destination file must be a file"); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
if(!dstPath.getParentFile().exists()){ |
||||||
|
dstPath.getParentFile().mkdirs(); |
||||||
|
} |
||||||
|
|
||||||
|
return FileUtil.copy(fs, srcPath, dstPath, deleteSource, fs.getConf()); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* |
||||||
|
* delete a file |
||||||
|
* |
||||||
|
* @param hdfsFilePath the path to delete. |
||||||
|
* @param recursive if path is a directory and set to |
||||||
|
* true, the directory is deleted else throws an exception. In |
||||||
|
* case of a file the recursive can be set to either true or false. |
||||||
|
* @return true if delete is successful else false. |
||||||
|
* @throws IOException |
||||||
|
*/ |
||||||
|
public boolean delete(String hdfsFilePath, boolean recursive) throws IOException { |
||||||
|
return fs.delete(new Path(hdfsFilePath), recursive); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* check if exists |
||||||
|
* |
||||||
|
* @param hdfsFilePath source file path |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public boolean exists(String hdfsFilePath) throws IOException { |
||||||
|
return fs.exists(new Path(hdfsFilePath)); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Renames Path src to Path dst. Can take place on local fs |
||||||
|
* or remote DFS. |
||||||
|
* @param src path to be renamed |
||||||
|
* @param dst new path after rename |
||||||
|
* @throws IOException on failure |
||||||
|
* @return true if rename is successful |
||||||
|
*/ |
||||||
|
public boolean rename(String src, String dst) throws IOException { |
||||||
|
return fs.rename(new Path(src), new Path(dst)); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* get the state of an application |
||||||
|
* |
||||||
|
* @param applicationId |
||||||
|
* @return the return may be null or there may be other parse exceptions |
||||||
|
* @throws JSONException |
||||||
|
* @throws IOException |
||||||
|
*/ |
||||||
|
public ExecutionStatus getApplicationStatus(String applicationId) throws JSONException { |
||||||
|
if (StringUtils.isEmpty(applicationId)) { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
String applicationUrl = getApplicationUrl(applicationId); |
||||||
|
|
||||||
|
String responseContent = HttpUtils.get(applicationUrl); |
||||||
|
|
||||||
|
JSONObject jsonObject = JSONObject.parseObject(responseContent); |
||||||
|
String result = jsonObject.getJSONObject("app").getString("finalStatus"); |
||||||
|
|
||||||
|
switch (result) { |
||||||
|
case ACCEPTED: |
||||||
|
return ExecutionStatus.SUBMITTED_SUCCESS; |
||||||
|
case SUCCEEDED: |
||||||
|
return ExecutionStatus.SUCCESS; |
||||||
|
case NEW: |
||||||
|
case NEW_SAVING: |
||||||
|
case SUBMITTED: |
||||||
|
case FAILED: |
||||||
|
return ExecutionStatus.FAILURE; |
||||||
|
case KILLED: |
||||||
|
return ExecutionStatus.KILL; |
||||||
|
|
||||||
|
case RUNNING: |
||||||
|
default: |
||||||
|
return ExecutionStatus.RUNNING_EXEUTION; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* |
||||||
|
* @return data hdfs path |
||||||
|
*/ |
||||||
|
public static String getHdfsDataBasePath() { |
||||||
|
return getString(DATA_STORE_2_HDFS_BASEPATH); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* hdfs resource dir |
||||||
|
* |
||||||
|
* @param tenantCode tenant code |
||||||
|
* @return hdfs resource dir |
||||||
|
*/ |
||||||
|
public static String getHdfsDir(String tenantCode) { |
||||||
|
return String.format("%s/resources", getHdfsTenantDir(tenantCode)); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get udf dir on hdfs |
||||||
|
* |
||||||
|
* @param tenantCode tenant code |
||||||
|
* @return get udf dir on hdfs |
||||||
|
*/ |
||||||
|
public static String getHdfsUdfDir(String tenantCode) { |
||||||
|
return String.format("%s/udfs", getHdfsTenantDir(tenantCode)); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get absolute path and name for file on hdfs |
||||||
|
* |
||||||
|
* @param tenantCode tenant code |
||||||
|
* @param filename file name |
||||||
|
* @return get absolute path and name for file on hdfs |
||||||
|
*/ |
||||||
|
public static String getHdfsFilename(String tenantCode, String filename) { |
||||||
|
return String.format("%s/%s", getHdfsDir(tenantCode), filename); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get absolute path and name for udf file on hdfs |
||||||
|
* |
||||||
|
* @param tenantCode tenant code |
||||||
|
* @param filename file name |
||||||
|
* @return get absolute path and name for udf file on hdfs |
||||||
|
*/ |
||||||
|
public static String getHdfsUdfFilename(String tenantCode, String filename) { |
||||||
|
return String.format("%s/%s", getHdfsUdfDir(tenantCode), filename); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* @return file directory of tenants on hdfs |
||||||
|
*/ |
||||||
|
private static String getHdfsTenantDir(String tenantCode) { |
||||||
|
return String.format("%s/%s", getString(DATA_STORE_2_HDFS_BASEPATH), tenantCode); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* getAppAddress |
||||||
|
* |
||||||
|
* @param appAddress |
||||||
|
* @param rmHa |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static String getAppAddress(String appAddress, String rmHa) { |
||||||
|
|
||||||
|
//get active ResourceManager
|
||||||
|
String activeRM = YarnHAAdminUtils.getAcitveRMName(rmHa); |
||||||
|
|
||||||
|
String[] split1 = appAddress.split(DOUBLE_SLASH); |
||||||
|
|
||||||
|
if (split1.length != 2) { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
String start = split1[0] + DOUBLE_SLASH; |
||||||
|
String[] split2 = split1[1].split(COLON); |
||||||
|
|
||||||
|
if (split2.length != 2) { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
String end = COLON + split2[1]; |
||||||
|
|
||||||
|
return start + activeRM + end; |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
@Override |
||||||
|
public void close() throws IOException { |
||||||
|
if (fs != null) { |
||||||
|
try { |
||||||
|
fs.close(); |
||||||
|
} catch (IOException e) { |
||||||
|
logger.error("Close HadoopUtils instance failed", e); |
||||||
|
throw new IOException("Close HadoopUtils instance failed", e); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* yarn ha admin utils |
||||||
|
*/ |
||||||
|
private static final class YarnHAAdminUtils extends RMAdminCLI { |
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(YarnHAAdminUtils.class); |
||||||
|
|
||||||
|
/** |
||||||
|
* get active resourcemanager |
||||||
|
* |
||||||
|
* @param rmIds |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static String getAcitveRMName(String rmIds) { |
||||||
|
|
||||||
|
String[] rmIdArr = rmIds.split(COMMA); |
||||||
|
|
||||||
|
int activeResourceManagerPort = getInt(HADOOP_RESOURCE_MANAGER_HTTPADDRESS_PORT, 8088); |
||||||
|
|
||||||
|
String yarnUrl = "http://%s:" + activeResourceManagerPort + "/ws/v1/cluster/info"; |
||||||
|
|
||||||
|
String state = null; |
||||||
|
try { |
||||||
|
/** |
||||||
|
* send http get request to rm1 |
||||||
|
*/ |
||||||
|
state = getRMState(String.format(yarnUrl, rmIdArr[0])); |
||||||
|
|
||||||
|
if (HADOOP_RM_STATE_ACTIVE.equals(state)) { |
||||||
|
return rmIdArr[0]; |
||||||
|
} else if (HADOOP_RM_STATE_STANDBY.equals(state)) { |
||||||
|
state = getRMState(String.format(yarnUrl, rmIdArr[1])); |
||||||
|
if (HADOOP_RM_STATE_ACTIVE.equals(state)) { |
||||||
|
return rmIdArr[1]; |
||||||
|
} |
||||||
|
} else { |
||||||
|
return null; |
||||||
|
} |
||||||
|
} catch (Exception e) { |
||||||
|
state = getRMState(String.format(yarnUrl, rmIdArr[1])); |
||||||
|
if (HADOOP_RM_STATE_ACTIVE.equals(state)) { |
||||||
|
return rmIdArr[0]; |
||||||
|
} |
||||||
|
} |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* get ResourceManager state |
||||||
|
* |
||||||
|
* @param url |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static String getRMState(String url) { |
||||||
|
|
||||||
|
String retStr = HttpUtils.get(url); |
||||||
|
|
||||||
|
if (StringUtils.isEmpty(retStr)) { |
||||||
|
return null; |
||||||
|
} |
||||||
|
//to json
|
||||||
|
JSONObject jsonObject = JSON.parseObject(retStr); |
||||||
|
|
||||||
|
//get ResourceManager state
|
||||||
|
String state = jsonObject.getJSONObject("clusterInfo").getString("haState"); |
||||||
|
return state; |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
|
} |
@ -0,0 +1,100 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.utils; |
||||||
|
|
||||||
|
import cn.escheduler.common.Constants; |
||||||
|
import org.apache.http.HttpEntity; |
||||||
|
import org.apache.http.client.config.RequestConfig; |
||||||
|
import org.apache.http.client.methods.CloseableHttpResponse; |
||||||
|
import org.apache.http.client.methods.HttpGet; |
||||||
|
import org.apache.http.impl.client.CloseableHttpClient; |
||||||
|
import org.apache.http.impl.client.HttpClients; |
||||||
|
import org.apache.http.util.EntityUtils; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
import java.io.IOException; |
||||||
|
|
||||||
|
/** |
||||||
|
* http utils |
||||||
|
*/ |
||||||
|
public class HttpUtils { |
||||||
|
|
||||||
|
|
||||||
|
public static final Logger logger = LoggerFactory.getLogger(HttpUtils.class); |
||||||
|
|
||||||
|
/** |
||||||
|
* get http request content |
||||||
|
* @param url |
||||||
|
* @return http response |
||||||
|
*/ |
||||||
|
public static String get(String url){ |
||||||
|
CloseableHttpClient httpclient = HttpClients.createDefault(); |
||||||
|
|
||||||
|
HttpGet httpget = new HttpGet(url); |
||||||
|
/** set timeout、request time、socket timeout */ |
||||||
|
RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(Constants.HTTP_CONNECT_TIMEOUT) |
||||||
|
.setConnectionRequestTimeout(Constants.HTTP_CONNECTION_REQUEST_TIMEOUT) |
||||||
|
.setSocketTimeout(Constants.SOCKET_TIMEOUT) |
||||||
|
.setRedirectsEnabled(true) |
||||||
|
.build(); |
||||||
|
httpget.setConfig(requestConfig); |
||||||
|
String responseContent = null; |
||||||
|
CloseableHttpResponse response = null; |
||||||
|
|
||||||
|
try { |
||||||
|
response = httpclient.execute(httpget); |
||||||
|
//check response status is 200
|
||||||
|
if (response.getStatusLine().getStatusCode() == 200) { |
||||||
|
HttpEntity entity = response.getEntity(); |
||||||
|
if (entity != null) { |
||||||
|
responseContent = EntityUtils.toString(entity, Constants.UTF_8); |
||||||
|
}else{ |
||||||
|
logger.warn("http entity is null"); |
||||||
|
} |
||||||
|
}else{ |
||||||
|
logger.error("htt get:{} response status code is not 200!"); |
||||||
|
} |
||||||
|
}catch (Exception e){ |
||||||
|
logger.error(e.getMessage(),e); |
||||||
|
}finally { |
||||||
|
try { |
||||||
|
if (response != null) { |
||||||
|
EntityUtils.consume(response.getEntity()); |
||||||
|
response.close(); |
||||||
|
} |
||||||
|
} catch (IOException e) { |
||||||
|
logger.error(e.getMessage(),e); |
||||||
|
} |
||||||
|
|
||||||
|
if (httpget != null && !httpget.isAborted()) { |
||||||
|
httpget.releaseConnection(); |
||||||
|
httpget.abort(); |
||||||
|
} |
||||||
|
|
||||||
|
if (httpclient != null) { |
||||||
|
try { |
||||||
|
httpclient.close(); |
||||||
|
} catch (IOException e) { |
||||||
|
logger.error(e.getMessage(),e); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
return responseContent; |
||||||
|
} |
||||||
|
|
||||||
|
} |
@ -0,0 +1,258 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.utils; |
||||||
|
|
||||||
|
import com.alibaba.fastjson.JSONArray; |
||||||
|
import com.alibaba.fastjson.JSONObject; |
||||||
|
import com.alibaba.fastjson.TypeReference; |
||||||
|
import com.fasterxml.jackson.core.JsonGenerator; |
||||||
|
import com.fasterxml.jackson.core.JsonParser; |
||||||
|
import com.fasterxml.jackson.databind.*; |
||||||
|
import org.apache.commons.lang.StringUtils; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
import java.io.IOException; |
||||||
|
import java.util.HashMap; |
||||||
|
import java.util.List; |
||||||
|
import java.util.Map; |
||||||
|
import java.util.TimeZone; |
||||||
|
|
||||||
|
/** |
||||||
|
* json utils |
||||||
|
*/ |
||||||
|
public class JSONUtils { |
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(JSONUtils.class); |
||||||
|
|
||||||
|
/** |
||||||
|
* can use static singleton, inject: just make sure to reuse! |
||||||
|
*/ |
||||||
|
private static final ObjectMapper objectMapper = new ObjectMapper(); |
||||||
|
|
||||||
|
/** |
||||||
|
* init |
||||||
|
*/ |
||||||
|
private static final JSONUtils instance = new JSONUtils(); |
||||||
|
|
||||||
|
|
||||||
|
private JSONUtils() { |
||||||
|
//Feature that determines whether encountering of unknown properties, false means not analyzer unknown properties
|
||||||
|
objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false).setTimeZone(TimeZone.getDefault()); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* json representation of object |
||||||
|
* @param object |
||||||
|
* @return object to json string |
||||||
|
*/ |
||||||
|
public static String toJson(Object object) { |
||||||
|
try{ |
||||||
|
return JSONObject.toJSONString(object,false); |
||||||
|
} catch (Exception e) { |
||||||
|
logger.error("object to json exception!",e); |
||||||
|
} |
||||||
|
|
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* |
||||||
|
* This method deserializes the specified Json into an object of the specified class. It is not |
||||||
|
* suitable to use if the specified class is a generic type since it will not have the generic |
||||||
|
* type information because of the Type Erasure feature of Java. Therefore, this method should not |
||||||
|
* be used if the desired type is a generic type. Note that this method works fine if the any of |
||||||
|
* the fields of the specified object are generics, just the object itself should not be a |
||||||
|
* generic type. |
||||||
|
* |
||||||
|
* @param json the string from which the object is to be deserialized |
||||||
|
* @param clazz the class of T |
||||||
|
* @return an object of type T from the string |
||||||
|
* classOfT |
||||||
|
*/ |
||||||
|
public static <T> T parseObject(String json, Class<T> clazz) { |
||||||
|
if (StringUtils.isEmpty(json)) { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
try { |
||||||
|
return JSONObject.parseObject(json, clazz); |
||||||
|
} catch (Exception e) { |
||||||
|
logger.error("parse object exception!",e); |
||||||
|
} |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* json to list |
||||||
|
* |
||||||
|
* @param json |
||||||
|
* @param clazz c |
||||||
|
* @param <T> |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static <T> List<T> toList(String json, Class<T> clazz) { |
||||||
|
if (StringUtils.isEmpty(json)) { |
||||||
|
return null; |
||||||
|
} |
||||||
|
try { |
||||||
|
return JSONArray.parseArray(json, clazz); |
||||||
|
} catch (Exception e) { |
||||||
|
logger.error("JSONArray.parseArray exception!",e); |
||||||
|
} |
||||||
|
|
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* check json object valid |
||||||
|
* |
||||||
|
* @param json |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static boolean checkJsonVaild(String json) { |
||||||
|
|
||||||
|
if (StringUtils.isEmpty(json)) { |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
try { |
||||||
|
objectMapper.readTree(json); |
||||||
|
return true; |
||||||
|
} catch (IOException e) { |
||||||
|
logger.error("check json object valid exception!",e); |
||||||
|
} |
||||||
|
|
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* Method for finding a JSON Object field with specified name in this |
||||||
|
* node or its child nodes, and returning value it has. |
||||||
|
* If no matching field is found in this node or its descendants, returns null. |
||||||
|
* |
||||||
|
* @param fieldName Name of field to look for |
||||||
|
* |
||||||
|
* @return Value of first matching node found, if any; null if none |
||||||
|
*/ |
||||||
|
public static String findValue(JsonNode jsonNode, String fieldName) { |
||||||
|
JsonNode node = jsonNode.findValue(fieldName); |
||||||
|
|
||||||
|
if (node == null) { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
return node.toString(); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* json to map |
||||||
|
* |
||||||
|
* {@link #toMap(String, Class, Class)} |
||||||
|
* |
||||||
|
* @param json |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static Map<String, String> toMap(String json) { |
||||||
|
if (StringUtils.isEmpty(json)) { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
try { |
||||||
|
return JSONObject.parseObject(json, new TypeReference<HashMap<String, String>>(){}); |
||||||
|
} catch (Exception e) { |
||||||
|
logger.error("json to map exception!",e); |
||||||
|
} |
||||||
|
|
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* |
||||||
|
* json to map |
||||||
|
* |
||||||
|
* <pre> |
||||||
|
* String jsonStr = "{\"id\":\"1001\",\"name\":\"Jobs\"}"; |
||||||
|
* Map<String,String> models = JSONUtils.toMap(jsonStr, String.class,String.class); |
||||||
|
* </pre> |
||||||
|
* @param json |
||||||
|
* @param classK |
||||||
|
* @param classV |
||||||
|
* @param <K> |
||||||
|
* @param <V> |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static <K, V> Map<K, V> toMap(String json, Class<K> classK, Class<V> classV) { |
||||||
|
if (StringUtils.isEmpty(json)) { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
try { |
||||||
|
return JSONObject.parseObject(json, new TypeReference<HashMap<K, V>>() {}); |
||||||
|
} catch (Exception e) { |
||||||
|
logger.error("json to map exception!",e); |
||||||
|
} |
||||||
|
|
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* 对象装json字符串 <p> |
||||||
|
* |
||||||
|
* @return json string |
||||||
|
*/ |
||||||
|
public static String toJsonString(Object object) { |
||||||
|
try{ |
||||||
|
return JSONObject.toJSONString(object,false); |
||||||
|
} catch (Exception e) { |
||||||
|
throw new RuntimeException("Json deserialization exception.", e); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* json serializer |
||||||
|
*/ |
||||||
|
public static class JsonDataSerializer extends JsonSerializer<String> { |
||||||
|
|
||||||
|
@Override |
||||||
|
public void serialize(String value, JsonGenerator gen, SerializerProvider provider) throws IOException { |
||||||
|
gen.writeRawValue(value); |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* json data deserializer |
||||||
|
*/ |
||||||
|
public static class JsonDataDeserializer extends JsonDeserializer<String> { |
||||||
|
|
||||||
|
@Override |
||||||
|
public String deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { |
||||||
|
JsonNode node = p.getCodec().readTree(p); |
||||||
|
return node.toString(); |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
|
} |
@ -0,0 +1,297 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.utils; |
||||||
|
|
||||||
|
import cn.escheduler.common.Constants; |
||||||
|
import cn.escheduler.common.shell.ShellExecutor; |
||||||
|
import org.apache.commons.configuration.Configuration; |
||||||
|
import org.apache.commons.lang3.StringUtils; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
import oshi.SystemInfo; |
||||||
|
import oshi.hardware.CentralProcessor; |
||||||
|
import oshi.hardware.GlobalMemory; |
||||||
|
import oshi.hardware.HardwareAbstractionLayer; |
||||||
|
|
||||||
|
import java.io.BufferedReader; |
||||||
|
import java.io.FileInputStream; |
||||||
|
import java.io.IOException; |
||||||
|
import java.io.InputStreamReader; |
||||||
|
import java.lang.management.ManagementFactory; |
||||||
|
import java.lang.management.RuntimeMXBean; |
||||||
|
import java.math.RoundingMode; |
||||||
|
import java.net.InetAddress; |
||||||
|
import java.net.UnknownHostException; |
||||||
|
import java.text.DecimalFormat; |
||||||
|
import java.util.ArrayList; |
||||||
|
import java.util.List; |
||||||
|
|
||||||
|
/** |
||||||
|
* os utils |
||||||
|
* |
||||||
|
*/ |
||||||
|
public class OSUtils { |
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(OSUtils.class); |
||||||
|
|
||||||
|
private static final SystemInfo SI = new SystemInfo(); |
||||||
|
public static final String TWO_DECIMAL = "0.00"; |
||||||
|
|
||||||
|
private static HardwareAbstractionLayer hal = SI.getHardware(); |
||||||
|
|
||||||
|
private OSUtils() {} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* get memory usage |
||||||
|
* Keep 2 decimal |
||||||
|
* @return percent % |
||||||
|
*/ |
||||||
|
public static double memoryUsage() { |
||||||
|
GlobalMemory memory = hal.getMemory(); |
||||||
|
double memoryUsage = (memory.getTotal() - memory.getAvailable() - memory.getSwapUsed()) * 0.1 / memory.getTotal() * 10; |
||||||
|
|
||||||
|
DecimalFormat df = new DecimalFormat(TWO_DECIMAL); |
||||||
|
df.setRoundingMode(RoundingMode.HALF_UP); |
||||||
|
return Double.parseDouble(df.format(memoryUsage)); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* get available physical memory size |
||||||
|
* |
||||||
|
* Keep 2 decimal |
||||||
|
* @return available Physical Memory Size, unit: G |
||||||
|
*/ |
||||||
|
public static double availablePhysicalMemorySize() { |
||||||
|
GlobalMemory memory = hal.getMemory(); |
||||||
|
double availablePhysicalMemorySize = (memory.getAvailable() + memory.getSwapUsed()) /1024.0/1024/1024; |
||||||
|
|
||||||
|
DecimalFormat df = new DecimalFormat(TWO_DECIMAL); |
||||||
|
df.setRoundingMode(RoundingMode.HALF_UP); |
||||||
|
return Double.parseDouble(df.format(availablePhysicalMemorySize)); |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get total physical memory size |
||||||
|
* |
||||||
|
* Keep 2 decimal |
||||||
|
* @return available Physical Memory Size, unit: G |
||||||
|
*/ |
||||||
|
public static double totalMemorySize() { |
||||||
|
GlobalMemory memory = hal.getMemory(); |
||||||
|
double availablePhysicalMemorySize = memory.getTotal() /1024.0/1024/1024; |
||||||
|
|
||||||
|
DecimalFormat df = new DecimalFormat(TWO_DECIMAL); |
||||||
|
df.setRoundingMode(RoundingMode.HALF_UP); |
||||||
|
return Double.parseDouble(df.format(availablePhysicalMemorySize)); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* load average |
||||||
|
* |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static double loadAverage() { |
||||||
|
double loadAverage = hal.getProcessor().getSystemLoadAverage(); |
||||||
|
|
||||||
|
DecimalFormat df = new DecimalFormat(TWO_DECIMAL); |
||||||
|
|
||||||
|
df.setRoundingMode(RoundingMode.HALF_UP); |
||||||
|
return Double.parseDouble(df.format(loadAverage)); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get cpu usage |
||||||
|
* |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static double cpuUsage() { |
||||||
|
CentralProcessor processor = hal.getProcessor(); |
||||||
|
double cpuUsage = processor.getSystemCpuLoad(); |
||||||
|
|
||||||
|
DecimalFormat df = new DecimalFormat(TWO_DECIMAL); |
||||||
|
df.setRoundingMode(RoundingMode.HALF_UP); |
||||||
|
|
||||||
|
return Double.parseDouble(df.format(cpuUsage)); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* get user list |
||||||
|
* |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static List<String> getUserList() { |
||||||
|
List<String> userList = new ArrayList<>(); |
||||||
|
BufferedReader bufferedReader = null; |
||||||
|
|
||||||
|
try { |
||||||
|
bufferedReader = new BufferedReader(new InputStreamReader(new FileInputStream("/etc/passwd"))); |
||||||
|
String line; |
||||||
|
|
||||||
|
while ((line = bufferedReader.readLine()) != null) { |
||||||
|
if (line.contains(":")) { |
||||||
|
String[] userInfo = line.split(":"); |
||||||
|
userList.add(userInfo[0]); |
||||||
|
} |
||||||
|
} |
||||||
|
} catch (Exception e) { |
||||||
|
logger.error(e.getMessage(), e); |
||||||
|
} finally { |
||||||
|
try { |
||||||
|
bufferedReader.close(); |
||||||
|
} catch (IOException e) { |
||||||
|
logger.error(e.getMessage(), e); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
return userList; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get system group information |
||||||
|
* @return |
||||||
|
* @throws IOException |
||||||
|
*/ |
||||||
|
public static String getGroup() throws IOException { |
||||||
|
String result = exeCmd("groups"); |
||||||
|
|
||||||
|
if (StringUtils.isNotEmpty(result)) { |
||||||
|
String[] groupInfo = StringUtils.split(result); |
||||||
|
return groupInfo[0]; |
||||||
|
} |
||||||
|
|
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Execute the corresponding command of Linux or Windows |
||||||
|
* |
||||||
|
* @param command |
||||||
|
* @return |
||||||
|
* @throws IOException |
||||||
|
*/ |
||||||
|
public static String exeCmd(String command) throws IOException { |
||||||
|
BufferedReader br = null; |
||||||
|
|
||||||
|
try { |
||||||
|
Process p = Runtime.getRuntime().exec(command); |
||||||
|
br = new BufferedReader(new InputStreamReader(p.getInputStream())); |
||||||
|
String line; |
||||||
|
StringBuilder sb = new StringBuilder(); |
||||||
|
|
||||||
|
while ((line = br.readLine()) != null) { |
||||||
|
sb.append(line + "\n"); |
||||||
|
} |
||||||
|
|
||||||
|
return sb.toString(); |
||||||
|
} finally { |
||||||
|
if (br != null) { |
||||||
|
try { |
||||||
|
br.close(); |
||||||
|
} catch (Exception e) { |
||||||
|
logger.error(e.getMessage(), e); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Execute the shell |
||||||
|
* @param command |
||||||
|
* @return |
||||||
|
* @throws IOException |
||||||
|
*/ |
||||||
|
public static String exeShell(String command) throws IOException { |
||||||
|
return ShellExecutor.execCommand("groups"); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get process id |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static int getProcessID() { |
||||||
|
RuntimeMXBean runtimeMXBean = ManagementFactory.getRuntimeMXBean(); |
||||||
|
return Integer.parseInt(runtimeMXBean.getName().split("@")[0]); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get local host |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static String getHost(){ |
||||||
|
try { |
||||||
|
return InetAddress.getLocalHost().getHostAddress(); |
||||||
|
} catch (UnknownHostException e) { |
||||||
|
logger.error(e.getMessage(),e); |
||||||
|
} |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* whether is macOS |
||||||
|
*/ |
||||||
|
public static boolean isMacOS() { |
||||||
|
String os = System.getProperty("os.name"); |
||||||
|
return os.startsWith("Mac"); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* whether is windows |
||||||
|
*/ |
||||||
|
public static boolean isWindows() { |
||||||
|
String os = System.getProperty("os.name"); |
||||||
|
return os.startsWith("Windows"); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* check memory and cpu usage |
||||||
|
* @param conf |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static Boolean checkResource(Configuration conf, Boolean isMaster){ |
||||||
|
double systemCpuLoad; |
||||||
|
double systemReservedMemory; |
||||||
|
|
||||||
|
if(isMaster){ |
||||||
|
systemCpuLoad = conf.getDouble(Constants.MASTER_MAX_CPULOAD_AVG, Constants.defaultMasterCpuLoad); |
||||||
|
systemReservedMemory = conf.getDouble(Constants.MASTER_RESERVED_MEMORY, Constants.defaultMasterReservedMemory); |
||||||
|
}else{ |
||||||
|
systemCpuLoad = conf.getDouble(Constants.WORKER_MAX_CPULOAD_AVG, Constants.defaultWorkerCpuLoad); |
||||||
|
systemReservedMemory = conf.getDouble(Constants.WORKER_RESERVED_MEMORY, Constants.defaultWorkerReservedMemory); |
||||||
|
} |
||||||
|
|
||||||
|
// judging usage
|
||||||
|
double loadAverage = OSUtils.loadAverage(); |
||||||
|
//
|
||||||
|
double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize(); |
||||||
|
|
||||||
|
if(loadAverage > systemCpuLoad || availablePhysicalMemorySize < systemReservedMemory){ |
||||||
|
logger.warn("load or availablePhysicalMemorySize(G) is too high, it's availablePhysicalMemorySize(G):{},loadAvg:{}", availablePhysicalMemorySize , loadAverage); |
||||||
|
return false; |
||||||
|
}else{ |
||||||
|
return true; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
} |
@ -0,0 +1,162 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.utils; |
||||||
|
|
||||||
|
import cn.escheduler.common.Constants; |
||||||
|
import cn.escheduler.common.enums.CommandType; |
||||||
|
import cn.escheduler.common.enums.DataType; |
||||||
|
import cn.escheduler.common.process.Property; |
||||||
|
import cn.escheduler.common.utils.placeholder.BusinessTimeUtils; |
||||||
|
import cn.escheduler.common.utils.placeholder.PlaceholderUtils; |
||||||
|
import cn.escheduler.common.utils.placeholder.TimePlaceholderUtils; |
||||||
|
import com.alibaba.fastjson.JSONObject; |
||||||
|
import org.apache.commons.lang.StringUtils; |
||||||
|
import org.apache.commons.lang.time.DateUtils; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
import java.sql.PreparedStatement; |
||||||
|
import java.text.ParseException; |
||||||
|
import java.util.*; |
||||||
|
|
||||||
|
/** |
||||||
|
* parameter parse utils |
||||||
|
*/ |
||||||
|
public class ParameterUtils { |
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(ParameterUtils.class); |
||||||
|
|
||||||
|
/** |
||||||
|
* convert parameters place holders |
||||||
|
* |
||||||
|
* @param parameterString |
||||||
|
* @param parameterMap |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static String convertParameterPlaceholders(String parameterString, Map<String, String> parameterMap) { |
||||||
|
if (StringUtils.isEmpty(parameterString)) { |
||||||
|
return parameterString; |
||||||
|
} |
||||||
|
|
||||||
|
//Get current time, schedule execute time
|
||||||
|
String cronTimeStr = parameterMap.get(Constants.PARAMETER_DATETIME); |
||||||
|
|
||||||
|
Date cronTime = null; |
||||||
|
|
||||||
|
if (StringUtils.isNotEmpty(cronTimeStr)) { |
||||||
|
try { |
||||||
|
cronTime = DateUtils.parseDate(cronTimeStr, new String[]{Constants.PARAMETER_FORMAT_TIME}); |
||||||
|
} catch (ParseException e) { |
||||||
|
logger.error(String.format("parse %s exception", cronTimeStr), e); |
||||||
|
} |
||||||
|
} else { |
||||||
|
cronTime = new Date(); |
||||||
|
} |
||||||
|
|
||||||
|
// replace variable ${} form,refers to the replacement of system variables and custom variables
|
||||||
|
parameterString = PlaceholderUtils.replacePlaceholders(parameterString, parameterMap, true); |
||||||
|
|
||||||
|
// replace time $[...] form, eg. $[yyyyMMdd]
|
||||||
|
if (cronTime != null) { |
||||||
|
parameterString = TimePlaceholderUtils.replacePlaceholders(parameterString, cronTime, true); |
||||||
|
} |
||||||
|
|
||||||
|
return parameterString; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* set in parameter |
||||||
|
* @param index |
||||||
|
* @param stmt |
||||||
|
* @param dataType |
||||||
|
* @param value |
||||||
|
* @throws Exception |
||||||
|
*/ |
||||||
|
public static void setInParameter(int index, PreparedStatement stmt, DataType dataType, String value)throws Exception{ |
||||||
|
if (dataType.equals(DataType.VARCHAR)){ |
||||||
|
stmt.setString(index,value); |
||||||
|
}else if (dataType.equals(DataType.INTEGER)){ |
||||||
|
stmt.setInt(index, Integer.parseInt(value)); |
||||||
|
}else if (dataType.equals(DataType.LONG)){ |
||||||
|
stmt.setLong(index, Long.parseLong(value)); |
||||||
|
}else if (dataType.equals(DataType.FLOAT)){ |
||||||
|
stmt.setFloat(index, Float.parseFloat(value)); |
||||||
|
}else if (dataType.equals(DataType.DOUBLE)){ |
||||||
|
stmt.setDouble(index, Double.parseDouble(value)); |
||||||
|
}else if (dataType.equals(DataType.DATE)){ |
||||||
|
stmt.setString(index, value); |
||||||
|
}else if (dataType.equals(DataType.TIME)){ |
||||||
|
stmt.setString(index, value); |
||||||
|
}else if (dataType.equals(DataType.TIMESTAMP)){ |
||||||
|
stmt.setString(index, value); |
||||||
|
}else if (dataType.equals(DataType.BOOLEAN)){ |
||||||
|
stmt.setBoolean(index,Boolean.parseBoolean(value)); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* curing user define parameters |
||||||
|
* |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static String curingGlobalParams(Map<String,String> globalParamMap, List<Property> globalParamList, |
||||||
|
CommandType commandType, Date scheduleTime){ |
||||||
|
Map<String, String> globalMap = new HashMap<>(); |
||||||
|
if(globalParamMap!= null){ |
||||||
|
globalMap.putAll(globalParamMap); |
||||||
|
} |
||||||
|
Map<String,String> allParamMap = new HashMap<>(); |
||||||
|
//如果是补数,需要传入一个补数时间,根据任务类型
|
||||||
|
Map<String,String> timeParams = BusinessTimeUtils |
||||||
|
.getBusinessTime(commandType,scheduleTime); |
||||||
|
|
||||||
|
if (timeParams != null) { |
||||||
|
allParamMap.putAll(timeParams); |
||||||
|
} |
||||||
|
|
||||||
|
if (globalMap != null) { |
||||||
|
allParamMap.putAll(globalMap); |
||||||
|
} |
||||||
|
|
||||||
|
Set<Map.Entry<String, String>> entries = allParamMap.entrySet(); |
||||||
|
|
||||||
|
Map<String,String> resolveMap = new HashMap<>(); |
||||||
|
for (Map.Entry<String,String> entry : entries){ |
||||||
|
String val = entry.getValue(); |
||||||
|
if (val.startsWith("$")){ |
||||||
|
String str = ParameterUtils.convertParameterPlaceholders(val, allParamMap); |
||||||
|
resolveMap.put(entry.getKey(),str); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
if (globalMap != null){ |
||||||
|
globalMap.putAll(resolveMap); |
||||||
|
} |
||||||
|
|
||||||
|
if (globalParamList != null && globalParamList.size() > 0){ |
||||||
|
|
||||||
|
for (Property property : globalParamList){ |
||||||
|
String val = globalMap.get(property.getProp()); |
||||||
|
if (val != null){ |
||||||
|
property.setValue(val); |
||||||
|
} |
||||||
|
} |
||||||
|
return JSONObject.toJSONString(globalParamList); |
||||||
|
} |
||||||
|
return null; |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,192 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.utils; |
||||||
|
|
||||||
|
import org.apache.commons.io.IOUtils; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
import java.io.IOException; |
||||||
|
import java.io.InputStream; |
||||||
|
import java.util.Properties; |
||||||
|
|
||||||
|
import static cn.escheduler.common.Constants.COMMON_PROPERTIES_PATH; |
||||||
|
import static cn.escheduler.common.Constants.HADOOP_PROPERTIES_PATH; |
||||||
|
|
||||||
|
/** |
||||||
|
* property utils |
||||||
|
* single instance |
||||||
|
*/ |
||||||
|
public class PropertyUtils { |
||||||
|
|
||||||
|
/** |
||||||
|
* logger |
||||||
|
*/ |
||||||
|
private static final Logger logger = LoggerFactory.getLogger(PropertyUtils.class); |
||||||
|
|
||||||
|
private static final Properties properties = new Properties(); |
||||||
|
|
||||||
|
private static final PropertyUtils propertyUtils = new PropertyUtils(); |
||||||
|
|
||||||
|
private PropertyUtils(){ |
||||||
|
init(); |
||||||
|
} |
||||||
|
|
||||||
|
private void init(){ |
||||||
|
String[] propertyFiles = new String[]{HADOOP_PROPERTIES_PATH,COMMON_PROPERTIES_PATH}; |
||||||
|
for (String fileName : propertyFiles) { |
||||||
|
InputStream fis = null; |
||||||
|
try { |
||||||
|
fis = PropertyUtils.class.getResourceAsStream(fileName); |
||||||
|
properties.load(fis); |
||||||
|
|
||||||
|
} catch (IOException e) { |
||||||
|
logger.error(e.getMessage(), e); |
||||||
|
System.exit(1); |
||||||
|
} finally { |
||||||
|
IOUtils.closeQuietly(fis); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/* |
||||||
|
public static PropertyUtils getInstance(){ |
||||||
|
return propertyUtils; |
||||||
|
} |
||||||
|
*/ |
||||||
|
|
||||||
|
/** |
||||||
|
* get property value |
||||||
|
* |
||||||
|
* @param key property name |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static String getString(String key) { |
||||||
|
return properties.getProperty(key.trim()); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get property value |
||||||
|
* |
||||||
|
* @param key property name |
||||||
|
* @return get property int value , if key == null, then return -1 |
||||||
|
*/ |
||||||
|
public static int getInt(String key) { |
||||||
|
return getInt(key, -1); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* |
||||||
|
* @param key |
||||||
|
* @param defaultValue |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static int getInt(String key, int defaultValue) { |
||||||
|
String value = getString(key); |
||||||
|
if (value == null) { |
||||||
|
return defaultValue; |
||||||
|
} |
||||||
|
|
||||||
|
try { |
||||||
|
return Integer.parseInt(value); |
||||||
|
} catch (NumberFormatException e) { |
||||||
|
logger.info(e.getMessage(),e); |
||||||
|
} |
||||||
|
return defaultValue; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get property value |
||||||
|
* |
||||||
|
* @param key property name |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static Boolean getBoolean(String key) { |
||||||
|
String value = properties.getProperty(key.trim()); |
||||||
|
if(null != value){ |
||||||
|
return Boolean.parseBoolean(value); |
||||||
|
} |
||||||
|
|
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get property long value |
||||||
|
* @param key |
||||||
|
* @param defaultVal |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static long getLong(String key, long defaultVal) { |
||||||
|
String val = getString(key); |
||||||
|
return val == null ? defaultVal : Long.parseLong(val); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* |
||||||
|
* @param key |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static long getLong(String key) { |
||||||
|
return getLong(key,-1); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* |
||||||
|
* @param key |
||||||
|
* @param defaultVal |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public double getDouble(String key, double defaultVal) { |
||||||
|
String val = getString(key); |
||||||
|
return val == null ? defaultVal : Double.parseDouble(val); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* get array |
||||||
|
* @param key property name |
||||||
|
* @param splitStr separator |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static String[] getArray(String key, String splitStr) { |
||||||
|
String value = getString(key); |
||||||
|
if (value == null) { |
||||||
|
return null; |
||||||
|
} |
||||||
|
try { |
||||||
|
String[] propertyArray = value.split(splitStr); |
||||||
|
return propertyArray; |
||||||
|
} catch (NumberFormatException e) { |
||||||
|
logger.info(e.getMessage(),e); |
||||||
|
} |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* |
||||||
|
* @param key |
||||||
|
* @param type |
||||||
|
* @param defaultValue |
||||||
|
* @param <T> |
||||||
|
* @return get enum value |
||||||
|
*/ |
||||||
|
public <T extends Enum<T>> T getEnum(String key, Class<T> type, |
||||||
|
T defaultValue) { |
||||||
|
String val = getString(key); |
||||||
|
return val == null ? defaultValue : Enum.valueOf(type, val); |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,74 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.utils; |
||||||
|
|
||||||
|
import cn.escheduler.common.enums.TaskType; |
||||||
|
import cn.escheduler.common.task.AbstractParameters; |
||||||
|
import cn.escheduler.common.task.dependent.DependentParameters; |
||||||
|
import cn.escheduler.common.task.mr.MapreduceParameters; |
||||||
|
import cn.escheduler.common.task.procedure.ProcedureParameters; |
||||||
|
import cn.escheduler.common.task.python.PythonParameters; |
||||||
|
import cn.escheduler.common.task.shell.ShellParameters; |
||||||
|
import cn.escheduler.common.task.spark.SparkParameters; |
||||||
|
import cn.escheduler.common.task.sql.SqlParameters; |
||||||
|
import cn.escheduler.common.task.subprocess.SubProcessParameters; |
||||||
|
import org.apache.commons.lang3.EnumUtils; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* task parameters utils |
||||||
|
*/ |
||||||
|
public class TaskParametersUtils { |
||||||
|
|
||||||
|
private static Logger logger = LoggerFactory.getLogger(TaskParametersUtils.class); |
||||||
|
|
||||||
|
/** |
||||||
|
* get task parameters |
||||||
|
* @param taskType |
||||||
|
* @param parameter |
||||||
|
* @return task parameters |
||||||
|
*/ |
||||||
|
public static AbstractParameters getParameters(String taskType, String parameter) { |
||||||
|
try { |
||||||
|
switch (EnumUtils.getEnum(TaskType.class,taskType)) { |
||||||
|
case SUB_PROCESS: |
||||||
|
return JSONUtils.parseObject(parameter, SubProcessParameters.class); |
||||||
|
case SHELL: |
||||||
|
return JSONUtils.parseObject(parameter, ShellParameters.class); |
||||||
|
case PROCEDURE: |
||||||
|
return JSONUtils.parseObject(parameter, ProcedureParameters.class); |
||||||
|
case SQL: |
||||||
|
return JSONUtils.parseObject(parameter, SqlParameters.class); |
||||||
|
case MR: |
||||||
|
return JSONUtils.parseObject(parameter, MapreduceParameters.class); |
||||||
|
case SPARK: |
||||||
|
return JSONUtils.parseObject(parameter, SparkParameters.class); |
||||||
|
case PYTHON: |
||||||
|
return JSONUtils.parseObject(parameter, PythonParameters.class); |
||||||
|
case DEPENDENT: |
||||||
|
return JSONUtils.parseObject(parameter, DependentParameters.class); |
||||||
|
default: |
||||||
|
return null; |
||||||
|
} |
||||||
|
} catch (Exception e) { |
||||||
|
logger.error(e.getMessage(), e); |
||||||
|
} |
||||||
|
return null; |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,134 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.utils.dependent; |
||||||
|
|
||||||
|
import cn.escheduler.common.model.DateInterval; |
||||||
|
import cn.escheduler.common.utils.DateUtils; |
||||||
|
|
||||||
|
import java.util.ArrayList; |
||||||
|
import java.util.Date; |
||||||
|
import java.util.List; |
||||||
|
|
||||||
|
public class DependentDateUtils { |
||||||
|
|
||||||
|
/** |
||||||
|
* get last day interval list |
||||||
|
* @param businessDate |
||||||
|
* @param hourNumber |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static List<DateInterval> getLastHoursInterval(Date businessDate, int hourNumber){ |
||||||
|
List<DateInterval> dateIntervals = new ArrayList<>(); |
||||||
|
for(int index = hourNumber; index > 0; index--){ |
||||||
|
Date lastHour = DateUtils.getSomeHourOfDay(businessDate, index); |
||||||
|
Date beginTime = DateUtils.getStartOfHour(lastHour); |
||||||
|
Date endTime = DateUtils.getEndOfHour(lastHour); |
||||||
|
dateIntervals.add(new DateInterval(beginTime, endTime)); |
||||||
|
} |
||||||
|
return dateIntervals; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get last day interval list |
||||||
|
* @param businessDate |
||||||
|
* @param someDay |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static List<DateInterval> getLastDayInterval(Date businessDate, int someDay){ |
||||||
|
|
||||||
|
List<DateInterval> dateIntervals = new ArrayList<>(); |
||||||
|
for(int index = someDay; index > 0; index--){ |
||||||
|
Date lastDay = DateUtils.getSomeDay(businessDate, -index); |
||||||
|
|
||||||
|
Date beginTime = DateUtils.getStartOfDay(lastDay); |
||||||
|
Date endTime = DateUtils.getEndOfDay(lastDay); |
||||||
|
dateIntervals.add(new DateInterval(beginTime, endTime)); |
||||||
|
} |
||||||
|
return dateIntervals; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get interval between last month first day and last day |
||||||
|
* @param businessDate |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static List<DateInterval> getLastMonthInterval(Date businessDate) { |
||||||
|
|
||||||
|
Date firstDayThisMonth = DateUtils.getFirstDayOfMonth(businessDate); |
||||||
|
Date lastDay = DateUtils.getSomeDay(firstDayThisMonth, -1); |
||||||
|
Date firstDay = DateUtils.getFirstDayOfMonth(lastDay); |
||||||
|
return getDateIntervalListBetweenTwoDates( firstDay, lastDay); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* get interval on first/last day of the last month |
||||||
|
* @param businessDate |
||||||
|
* @param isBeginDay |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static List<DateInterval> getLastMonthBeginInterval(Date businessDate, boolean isBeginDay) { |
||||||
|
|
||||||
|
Date firstDayThisMonth = DateUtils.getFirstDayOfMonth(businessDate); |
||||||
|
Date lastDay = DateUtils.getSomeDay(firstDayThisMonth, -1); |
||||||
|
Date firstDay = DateUtils.getFirstDayOfMonth(lastDay); |
||||||
|
if(isBeginDay){ |
||||||
|
return getDateIntervalListBetweenTwoDates(firstDay, firstDay); |
||||||
|
}else{ |
||||||
|
return getDateIntervalListBetweenTwoDates(lastDay, lastDay); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get interval between monday to sunday of last week |
||||||
|
* default set monday the first day of week |
||||||
|
* @param businessDate |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static List<DateInterval> getLastWeekInterval(Date businessDate) { |
||||||
|
Date mondayThisWeek = DateUtils.getMonday(businessDate); |
||||||
|
Date sunday = DateUtils.getSomeDay(mondayThisWeek, -1); |
||||||
|
Date monday = DateUtils.getMonday(sunday); |
||||||
|
return getDateIntervalListBetweenTwoDates(monday, sunday); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get interval on the day of last week |
||||||
|
* default set monday the first day of week |
||||||
|
* @param businessDate |
||||||
|
* @param dayOfWeek monday:1,tuesday:2,wednesday:3,thursday:4,friday:5,saturday:6,sunday:7 |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static List<DateInterval> getLastWeekOneDayInterval(Date businessDate, int dayOfWeek) { |
||||||
|
Date mondayThisWeek = DateUtils.getMonday(businessDate); |
||||||
|
Date sunday = DateUtils.getSomeDay(mondayThisWeek, -1); |
||||||
|
Date monday = DateUtils.getMonday(sunday); |
||||||
|
Date destDay = DateUtils.getSomeDay(monday, dayOfWeek -1); |
||||||
|
return getDateIntervalListBetweenTwoDates(destDay, destDay); |
||||||
|
} |
||||||
|
|
||||||
|
public static List<DateInterval> getDateIntervalListBetweenTwoDates(Date firstDay, Date lastDay) { |
||||||
|
List<DateInterval> dateIntervals = new ArrayList<>(); |
||||||
|
while(!firstDay.after(lastDay)){ |
||||||
|
Date beginTime = DateUtils.getStartOfDay(firstDay); |
||||||
|
Date endTime = DateUtils.getEndOfDay(firstDay); |
||||||
|
dateIntervals.add(new DateInterval(beginTime, endTime)); |
||||||
|
firstDay = DateUtils.getSomeDay(firstDay, 1); |
||||||
|
} |
||||||
|
return dateIntervals; |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,65 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.utils.placeholder; |
||||||
|
|
||||||
|
import cn.escheduler.common.Constants; |
||||||
|
import cn.escheduler.common.enums.CommandType; |
||||||
|
|
||||||
|
import java.util.Date; |
||||||
|
import java.util.HashMap; |
||||||
|
import java.util.Map; |
||||||
|
|
||||||
|
import static cn.escheduler.common.Constants.PARAMETER_FORMAT_DATE; |
||||||
|
import static cn.escheduler.common.Constants.PARAMETER_FORMAT_TIME; |
||||||
|
import static cn.escheduler.common.utils.DateUtils.format; |
||||||
|
import static org.apache.commons.lang.time.DateUtils.addDays; |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* business time utils |
||||||
|
*/ |
||||||
|
public class BusinessTimeUtils { |
||||||
|
|
||||||
|
/** |
||||||
|
* get business time in parameters by different command types |
||||||
|
* |
||||||
|
* @param commandType command type |
||||||
|
* @param runTime run time or schedule time |
||||||
|
*/ |
||||||
|
public static Map<String, String> getBusinessTime(CommandType commandType, Date runTime) { |
||||||
|
Date businessDate = runTime; |
||||||
|
switch (commandType) { |
||||||
|
case COMPLEMENT_DATA: |
||||||
|
break; |
||||||
|
case START_PROCESS: |
||||||
|
case START_CURRENT_TASK_PROCESS: |
||||||
|
case RECOVER_TOLERANCE_FAULT_PROCESS: |
||||||
|
case RECOVER_SUSPENDED_PROCESS: |
||||||
|
case START_FAILURE_TASK_PROCESS: |
||||||
|
case SCHEDULER: |
||||||
|
default: |
||||||
|
businessDate = addDays(new Date(), -1); |
||||||
|
break; |
||||||
|
} |
||||||
|
Date businessCurrentDate = addDays(businessDate, 1); |
||||||
|
Map<String, String> result = new HashMap<>(); |
||||||
|
result.put(Constants.PARAMETER_CURRENT_DATE, format(businessCurrentDate, PARAMETER_FORMAT_DATE)); |
||||||
|
result.put(Constants.PARAMETER_BUSINESS_DATE, format(businessDate, PARAMETER_FORMAT_DATE)); |
||||||
|
result.put(Constants.PARAMETER_DATETIME, format(businessCurrentDate, PARAMETER_FORMAT_TIME)); |
||||||
|
return result; |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,99 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.utils.placeholder; |
||||||
|
|
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
import java.util.Map; |
||||||
|
|
||||||
|
/** |
||||||
|
* placeholder utils |
||||||
|
*/ |
||||||
|
public class PlaceholderUtils { |
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(PlaceholderUtils.class); |
||||||
|
|
||||||
|
/** |
||||||
|
* Prefix of the position to be replaced |
||||||
|
*/ |
||||||
|
public static final String placeholderPrefix = "${"; |
||||||
|
|
||||||
|
/** |
||||||
|
* The suffix of the position to be replaced |
||||||
|
*/ |
||||||
|
public static final String placeholderSuffix = "}"; |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* Replaces all placeholders of format {@code ${name}} with the value returned |
||||||
|
* from the supplied {@link PropertyPlaceholderHelper.PlaceholderResolver}. |
||||||
|
* |
||||||
|
* @param value the value containing the placeholders to be replaced |
||||||
|
* @param paramsMap placeholder data dictionary |
||||||
|
* @return the supplied value with placeholders replaced inline |
||||||
|
*/ |
||||||
|
public static String replacePlaceholders(String value, Map<String, String> paramsMap, boolean ignoreUnresolvablePlaceholders) { |
||||||
|
//replacement tool, parameter key will be replaced by value,if can't match , will throw an exception
|
||||||
|
PropertyPlaceholderHelper strictHelper = getPropertyPlaceholderHelper(false); |
||||||
|
|
||||||
|
//Non-strict replacement tool implementation, when the position to be replaced does not get the corresponding value, the current position is ignored, and the next position is replaced.
|
||||||
|
PropertyPlaceholderHelper nonStrictHelper = getPropertyPlaceholderHelper(true); |
||||||
|
|
||||||
|
PropertyPlaceholderHelper helper = (ignoreUnresolvablePlaceholders ? nonStrictHelper : strictHelper); |
||||||
|
|
||||||
|
//the PlaceholderResolver to use for replacement
|
||||||
|
return helper.replacePlaceholders(value, new PropertyPlaceholderResolver(value, paramsMap)); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Creates a new {@code PropertyPlaceholderHelper} that uses the supplied prefix and suffix. |
||||||
|
* @param ignoreUnresolvablePlaceholders indicates whether unresolvable placeholders should |
||||||
|
* be ignored ({@code true}) or cause an exception ({@code false}) |
||||||
|
*/ |
||||||
|
public static PropertyPlaceholderHelper getPropertyPlaceholderHelper(boolean ignoreUnresolvablePlaceholders) { |
||||||
|
|
||||||
|
return new PropertyPlaceholderHelper(placeholderPrefix, placeholderSuffix, null, ignoreUnresolvablePlaceholders); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Placeholder replacement resolver |
||||||
|
*/ |
||||||
|
private static class PropertyPlaceholderResolver implements PropertyPlaceholderHelper.PlaceholderResolver { |
||||||
|
|
||||||
|
private final String value; |
||||||
|
|
||||||
|
private final Map<String, String> paramsMap; |
||||||
|
|
||||||
|
public PropertyPlaceholderResolver(String value, Map<String, String> paramsMap) { |
||||||
|
this.value = value; |
||||||
|
this.paramsMap = paramsMap; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public String resolvePlaceholder(String placeholderName) { |
||||||
|
try { |
||||||
|
return paramsMap.get(placeholderName); |
||||||
|
} catch (Exception ex) { |
||||||
|
logger.error(String.format("resolve placeholder '%s' in [ %s ]" , placeholderName, value), ex); |
||||||
|
return null; |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
} |
@ -0,0 +1,254 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.utils.placeholder; |
||||||
|
|
||||||
|
|
||||||
|
import org.apache.commons.logging.Log; |
||||||
|
import org.apache.commons.logging.LogFactory; |
||||||
|
|
||||||
|
import java.util.*; |
||||||
|
|
||||||
|
/** |
||||||
|
* Utility class for working with Strings that have placeholder values in them. A placeholder takes the form |
||||||
|
* {@code ${name}}. Using {@code PropertyPlaceholderHelper} these placeholders can be substituted for |
||||||
|
* user-supplied values. <p> Values for substitution can be supplied using a {@link Properties} instance or |
||||||
|
* using a {@link PlaceholderResolver}. |
||||||
|
* |
||||||
|
* @author Juergen Hoeller |
||||||
|
* @author Rob Harrop |
||||||
|
* @since 3.0 |
||||||
|
*/ |
||||||
|
public class PropertyPlaceholderHelper { |
||||||
|
|
||||||
|
private static final Log logger = LogFactory.getLog(PropertyPlaceholderHelper.class); |
||||||
|
|
||||||
|
private static final Map<String, String> wellKnownSimplePrefixes = new HashMap<String, String>(4); |
||||||
|
|
||||||
|
static { |
||||||
|
wellKnownSimplePrefixes.put("}", "{"); |
||||||
|
wellKnownSimplePrefixes.put("]", "["); |
||||||
|
wellKnownSimplePrefixes.put(")", "("); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
private final String placeholderPrefix; |
||||||
|
|
||||||
|
private final String placeholderSuffix; |
||||||
|
|
||||||
|
private final String simplePrefix; |
||||||
|
|
||||||
|
private final String valueSeparator; |
||||||
|
|
||||||
|
private final boolean ignoreUnresolvablePlaceholders; |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* Creates a new {@code PropertyPlaceholderHelper} that uses the supplied prefix and suffix. |
||||||
|
* Unresolvable placeholders are ignored. |
||||||
|
* @param placeholderPrefix the prefix that denotes the start of a placeholder |
||||||
|
* @param placeholderSuffix the suffix that denotes the end of a placeholder |
||||||
|
*/ |
||||||
|
public PropertyPlaceholderHelper(String placeholderPrefix, String placeholderSuffix) { |
||||||
|
this(placeholderPrefix, placeholderSuffix, null, true); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Creates a new {@code PropertyPlaceholderHelper} that uses the supplied prefix and suffix. |
||||||
|
* @param placeholderPrefix the prefix that denotes the start of a placeholder |
||||||
|
* @param placeholderSuffix the suffix that denotes the end of a placeholder |
||||||
|
* @param valueSeparator the separating character between the placeholder variable |
||||||
|
* and the associated default value, if any |
||||||
|
* @param ignoreUnresolvablePlaceholders indicates whether unresolvable placeholders should |
||||||
|
* be ignored ({@code true}) or cause an exception ({@code false}) |
||||||
|
*/ |
||||||
|
public PropertyPlaceholderHelper(String placeholderPrefix, String placeholderSuffix, |
||||||
|
String valueSeparator, boolean ignoreUnresolvablePlaceholders) { |
||||||
|
|
||||||
|
notNull(placeholderPrefix, "'placeholderPrefix' must not be null"); |
||||||
|
notNull(placeholderSuffix, "'placeholderSuffix' must not be null"); |
||||||
|
this.placeholderPrefix = placeholderPrefix; |
||||||
|
this.placeholderSuffix = placeholderSuffix; |
||||||
|
String simplePrefixForSuffix = wellKnownSimplePrefixes.get(this.placeholderSuffix); |
||||||
|
if (simplePrefixForSuffix != null && this.placeholderPrefix.endsWith(simplePrefixForSuffix)) { |
||||||
|
this.simplePrefix = simplePrefixForSuffix; |
||||||
|
} |
||||||
|
else { |
||||||
|
this.simplePrefix = this.placeholderPrefix; |
||||||
|
} |
||||||
|
this.valueSeparator = valueSeparator; |
||||||
|
this.ignoreUnresolvablePlaceholders = ignoreUnresolvablePlaceholders; |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* Replaces all placeholders of format {@code ${name}} with the corresponding |
||||||
|
* property from the supplied {@link Properties}. |
||||||
|
* @param value the value containing the placeholders to be replaced |
||||||
|
* @param properties the {@code Properties} to use for replacement |
||||||
|
* @return the supplied value with placeholders replaced inline |
||||||
|
*/ |
||||||
|
public String replacePlaceholders(String value, final Properties properties) { |
||||||
|
notNull(properties, "'properties' must not be null"); |
||||||
|
return replacePlaceholders(value, new PlaceholderResolver() { |
||||||
|
@Override |
||||||
|
public String resolvePlaceholder(String placeholderName) { |
||||||
|
return properties.getProperty(placeholderName); |
||||||
|
} |
||||||
|
}); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Replaces all placeholders of format {@code ${name}} with the value returned |
||||||
|
* from the supplied {@link PlaceholderResolver}. |
||||||
|
* @param value the value containing the placeholders to be replaced |
||||||
|
* @param placeholderResolver the {@code PlaceholderResolver} to use for replacement |
||||||
|
* @return the supplied value with placeholders replaced inline |
||||||
|
*/ |
||||||
|
public String replacePlaceholders(String value, PlaceholderResolver placeholderResolver) { |
||||||
|
notNull(value, "'value' must not be null"); |
||||||
|
return parseStringValue(value, placeholderResolver, new HashSet<String>()); |
||||||
|
} |
||||||
|
|
||||||
|
protected String parseStringValue( |
||||||
|
String value, PlaceholderResolver placeholderResolver, Set<String> visitedPlaceholders) { |
||||||
|
|
||||||
|
StringBuilder result = new StringBuilder(value); |
||||||
|
|
||||||
|
int startIndex = value.indexOf(this.placeholderPrefix); |
||||||
|
while (startIndex != -1) { |
||||||
|
int endIndex = findPlaceholderEndIndex(result, startIndex); |
||||||
|
if (endIndex != -1) { |
||||||
|
String placeholder = result.substring(startIndex + this.placeholderPrefix.length(), endIndex); |
||||||
|
String originalPlaceholder = placeholder; |
||||||
|
if (!visitedPlaceholders.add(originalPlaceholder)) { |
||||||
|
throw new IllegalArgumentException( |
||||||
|
"Circular placeholder reference '" + originalPlaceholder + "' in property definitions"); |
||||||
|
} |
||||||
|
// Recursive invocation, parsing placeholders contained in the placeholder key.
|
||||||
|
placeholder = parseStringValue(placeholder, placeholderResolver, visitedPlaceholders); |
||||||
|
// Now obtain the value for the fully resolved key...
|
||||||
|
String propVal = placeholderResolver.resolvePlaceholder(placeholder); |
||||||
|
if (propVal == null && this.valueSeparator != null) { |
||||||
|
int separatorIndex = placeholder.indexOf(this.valueSeparator); |
||||||
|
if (separatorIndex != -1) { |
||||||
|
String actualPlaceholder = placeholder.substring(0, separatorIndex); |
||||||
|
String defaultValue = placeholder.substring(separatorIndex + this.valueSeparator.length()); |
||||||
|
propVal = placeholderResolver.resolvePlaceholder(actualPlaceholder); |
||||||
|
if (propVal == null) { |
||||||
|
propVal = defaultValue; |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
if (propVal != null) { |
||||||
|
// Recursive invocation, parsing placeholders contained in the
|
||||||
|
// previously resolved placeholder value.
|
||||||
|
propVal = parseStringValue(propVal, placeholderResolver, visitedPlaceholders); |
||||||
|
result.replace(startIndex, endIndex + this.placeholderSuffix.length(), propVal); |
||||||
|
if (logger.isTraceEnabled()) { |
||||||
|
logger.trace("Resolved placeholder '" + placeholder + "'"); |
||||||
|
} |
||||||
|
startIndex = result.indexOf(this.placeholderPrefix, startIndex + propVal.length()); |
||||||
|
} |
||||||
|
else if (this.ignoreUnresolvablePlaceholders) { |
||||||
|
// Proceed with unprocessed value.
|
||||||
|
startIndex = result.indexOf(this.placeholderPrefix, endIndex + this.placeholderSuffix.length()); |
||||||
|
} |
||||||
|
else { |
||||||
|
throw new IllegalArgumentException("Could not resolve placeholder '" + |
||||||
|
placeholder + "'" + " in value \"" + value + "\""); |
||||||
|
} |
||||||
|
visitedPlaceholders.remove(originalPlaceholder); |
||||||
|
} |
||||||
|
else { |
||||||
|
startIndex = -1; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
return result.toString(); |
||||||
|
} |
||||||
|
|
||||||
|
private int findPlaceholderEndIndex(CharSequence buf, int startIndex) { |
||||||
|
int index = startIndex + this.placeholderPrefix.length(); |
||||||
|
int withinNestedPlaceholder = 0; |
||||||
|
while (index < buf.length()) { |
||||||
|
if (substringMatch(buf, index, this.placeholderSuffix)) { |
||||||
|
if (withinNestedPlaceholder > 0) { |
||||||
|
withinNestedPlaceholder--; |
||||||
|
index = index + this.placeholderSuffix.length(); |
||||||
|
} |
||||||
|
else { |
||||||
|
return index; |
||||||
|
} |
||||||
|
} |
||||||
|
else if (substringMatch(buf, index, this.simplePrefix)) { |
||||||
|
withinNestedPlaceholder++; |
||||||
|
index = index + this.simplePrefix.length(); |
||||||
|
} |
||||||
|
else { |
||||||
|
index++; |
||||||
|
} |
||||||
|
} |
||||||
|
return -1; |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* Strategy interface used to resolve replacement values for placeholders contained in Strings. |
||||||
|
*/ |
||||||
|
public interface PlaceholderResolver { |
||||||
|
|
||||||
|
/** |
||||||
|
* Resolve the supplied placeholder name to the replacement value. |
||||||
|
* @param placeholderName the name of the placeholder to resolve |
||||||
|
* @return the replacement value, or {@code null} if no replacement is to be made |
||||||
|
*/ |
||||||
|
String resolvePlaceholder(String placeholderName); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Test whether the given string matches the given substring |
||||||
|
* at the given index. |
||||||
|
* @param str the original string (or StringBuilder) |
||||||
|
* @param index the index in the original string to start matching against |
||||||
|
* @param substring the substring to match at the given index |
||||||
|
*/ |
||||||
|
public static boolean substringMatch(CharSequence str, int index, CharSequence substring) { |
||||||
|
for (int j = 0; j < substring.length(); j++) { |
||||||
|
int i = index + j; |
||||||
|
if (i >= str.length() || str.charAt(i) != substring.charAt(j)) { |
||||||
|
return false; |
||||||
|
} |
||||||
|
} |
||||||
|
return true; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Assert that an object is not {@code null}. |
||||||
|
* <pre class="code">Assert.notNull(clazz, "The class must not be null");</pre> |
||||||
|
* @param object the object to check |
||||||
|
* @param message the exception message to use if the assertion fails |
||||||
|
* @throws IllegalArgumentException if the object is {@code null} |
||||||
|
*/ |
||||||
|
public static void notNull(Object object, String message) { |
||||||
|
if (object == null) { |
||||||
|
throw new IllegalArgumentException(message); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
} |
||||||
|
|
@ -0,0 +1,512 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.utils.placeholder; |
||||||
|
|
||||||
|
import cn.escheduler.common.Constants; |
||||||
|
import cn.escheduler.common.utils.DateUtils; |
||||||
|
import org.apache.commons.lang.StringUtils; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
import java.util.*; |
||||||
|
|
||||||
|
import static cn.escheduler.common.Constants.*; |
||||||
|
|
||||||
|
/** |
||||||
|
* time place holder utils |
||||||
|
*/ |
||||||
|
public class TimePlaceholderUtils { |
||||||
|
private static final Logger logger = LoggerFactory.getLogger(TimePlaceholderUtils.class); |
||||||
|
|
||||||
|
/** |
||||||
|
* Prefix of the position to be replaced |
||||||
|
*/ |
||||||
|
public static final String placeholderPrefix = "$["; |
||||||
|
|
||||||
|
/** |
||||||
|
* The suffix of the position to be replaced |
||||||
|
*/ |
||||||
|
public static final String placeholderSuffix = "]"; |
||||||
|
|
||||||
|
/** |
||||||
|
* Replaces all placeholders of format {@code ${name}} with the value returned |
||||||
|
* from the supplied {@link PropertyPlaceholderHelper.PlaceholderResolver}. |
||||||
|
* |
||||||
|
* @param value the value containing the placeholders to be replaced |
||||||
|
* @param date custom date |
||||||
|
* @param ignoreUnresolvablePlaceholders |
||||||
|
* @return the supplied value with placeholders replaced inline |
||||||
|
*/ |
||||||
|
public static String replacePlaceholders(String value, Date date, boolean ignoreUnresolvablePlaceholders) { |
||||||
|
PropertyPlaceholderHelper strictHelper = getPropertyPlaceholderHelper(false); |
||||||
|
PropertyPlaceholderHelper nonStrictHelper = getPropertyPlaceholderHelper(true); |
||||||
|
|
||||||
|
PropertyPlaceholderHelper helper = (ignoreUnresolvablePlaceholders ? nonStrictHelper : strictHelper); |
||||||
|
return helper.replacePlaceholders(value, new TimePlaceholderResolver(value, date)); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* Creates a new {@code PropertyPlaceholderHelper} that uses the supplied prefix and suffix. |
||||||
|
* @param ignoreUnresolvablePlaceholders indicates whether unresolvable placeholders should |
||||||
|
* be ignored ({@code true}) or cause an exception ({@code false}) |
||||||
|
*/ |
||||||
|
private static PropertyPlaceholderHelper getPropertyPlaceholderHelper(boolean ignoreUnresolvablePlaceholders) { |
||||||
|
return new PropertyPlaceholderHelper(placeholderPrefix, placeholderSuffix, null, ignoreUnresolvablePlaceholders); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* calculate expression's value |
||||||
|
* |
||||||
|
* @param expression |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static Integer calculate(String expression) { |
||||||
|
expression = StringUtils.trim(expression); |
||||||
|
expression = convert(expression); |
||||||
|
|
||||||
|
List<String> result = string2List(expression); |
||||||
|
result = convert2SuffixList(result); |
||||||
|
|
||||||
|
return calculate(result); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Change the sign in the expression to P (positive) N (negative) |
||||||
|
* |
||||||
|
* @param expression |
||||||
|
* @return eg. "-3+-6*(+8)-(-5) -> S3+S6*(P8)-(S5)" |
||||||
|
*/ |
||||||
|
private static String convert(String expression) { |
||||||
|
char[] arr = expression.toCharArray(); |
||||||
|
|
||||||
|
for (int i = 0; i < arr.length; i++) { |
||||||
|
if (arr[i] == SUBTRACT_CHAR) { |
||||||
|
if (i == 0) { |
||||||
|
arr[i] = N; |
||||||
|
} else { |
||||||
|
char c = arr[i - 1]; |
||||||
|
if (c == ADD_CHAR || c == SUBTRACT_CHAR || c == MULTIPLY_CHAR || c == DIVISION_CHAR || c == LEFT_BRACE_CHAR) { |
||||||
|
arr[i] = N; |
||||||
|
} |
||||||
|
} |
||||||
|
} else if (arr[i] == ADD_CHAR) { |
||||||
|
if (i == 0) { |
||||||
|
arr[i] = P; |
||||||
|
} else { |
||||||
|
char c = arr[i - 1]; |
||||||
|
if (c == ADD_CHAR || c == SUBTRACT_CHAR || c == MULTIPLY_CHAR || c == DIVISION_CHAR || c == LEFT_BRACE_CHAR) { |
||||||
|
arr[i] = P; |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
return new String(arr); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* to suffix expression |
||||||
|
* |
||||||
|
* @param srcList |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
private static List<String> convert2SuffixList(List<String> srcList) { |
||||||
|
List<String> result = new ArrayList<>(); |
||||||
|
Stack<String> stack = new Stack<>(); |
||||||
|
|
||||||
|
for (int i = 0; i < srcList.size(); i++) { |
||||||
|
if (Character.isDigit(srcList.get(i).charAt(0))) { |
||||||
|
result.add(srcList.get(i)); |
||||||
|
} else { |
||||||
|
switch (srcList.get(i).charAt(0)) { |
||||||
|
case LEFT_BRACE_CHAR: |
||||||
|
stack.push(srcList.get(i)); |
||||||
|
break; |
||||||
|
case RIGHT_BRACE_CHAR: |
||||||
|
while (!LEFT_BRACE_STRING.equals(stack.peek())) { |
||||||
|
result.add(stack.pop()); |
||||||
|
} |
||||||
|
stack.pop(); |
||||||
|
break; |
||||||
|
default: |
||||||
|
while (!stack.isEmpty() && compare(stack.peek(), srcList.get(i))) { |
||||||
|
result.add(stack.pop()); |
||||||
|
} |
||||||
|
stack.push(srcList.get(i)); |
||||||
|
break; |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
while (!stack.isEmpty()) { |
||||||
|
result.add(stack.pop()); |
||||||
|
} |
||||||
|
|
||||||
|
return result; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Calculate the suffix expression |
||||||
|
* |
||||||
|
* @param result |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
private static Integer calculate(List<String> result) { |
||||||
|
Stack<Integer> stack = new Stack<>(); |
||||||
|
for (int i = 0; i < result.size(); i++) { |
||||||
|
if (Character.isDigit(result.get(i).charAt(0))) { |
||||||
|
stack.push(Integer.parseInt(result.get(i))); |
||||||
|
} else { |
||||||
|
Integer backInt = stack.pop(); |
||||||
|
Integer frontInt = 0; |
||||||
|
char op = result.get(i).charAt(0); |
||||||
|
|
||||||
|
if (!(op == P || op == N)) { |
||||||
|
frontInt = stack.pop(); |
||||||
|
} |
||||||
|
|
||||||
|
Integer res = 0; |
||||||
|
switch (result.get(i).charAt(0)) { |
||||||
|
case P: |
||||||
|
res = frontInt + backInt; |
||||||
|
break; |
||||||
|
case N: |
||||||
|
res = frontInt - backInt; |
||||||
|
break; |
||||||
|
case ADD_CHAR: |
||||||
|
res = frontInt + backInt; |
||||||
|
break; |
||||||
|
case SUBTRACT_CHAR: |
||||||
|
res = frontInt - backInt; |
||||||
|
break; |
||||||
|
case MULTIPLY_CHAR: |
||||||
|
res = frontInt * backInt; |
||||||
|
break; |
||||||
|
case DIVISION_CHAR: |
||||||
|
res = frontInt / backInt; |
||||||
|
break; |
||||||
|
default: |
||||||
|
break; |
||||||
|
} |
||||||
|
stack.push(res); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
return stack.pop(); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* string to list |
||||||
|
* |
||||||
|
* @param expression |
||||||
|
* @return list |
||||||
|
*/ |
||||||
|
private static List<String> string2List(String expression) { |
||||||
|
List<String> result = new ArrayList<>(); |
||||||
|
String num = ""; |
||||||
|
for (int i = 0; i < expression.length(); i++) { |
||||||
|
if (Character.isDigit(expression.charAt(i))) { |
||||||
|
num = num + expression.charAt(i); |
||||||
|
} else { |
||||||
|
if (!num.isEmpty()) { |
||||||
|
result.add(num); |
||||||
|
} |
||||||
|
result.add(expression.charAt(i) + ""); |
||||||
|
num = ""; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
if (!num.isEmpty()) { |
||||||
|
result.add(num); |
||||||
|
} |
||||||
|
|
||||||
|
return result; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* compare loginUser level |
||||||
|
* |
||||||
|
* @param peek |
||||||
|
* @param cur |
||||||
|
* @return true or false |
||||||
|
*/ |
||||||
|
private static boolean compare(String peek, String cur) { |
||||||
|
if (MULTIPLY_STRING.equals(peek) && (DIVISION_STRING.equals(cur) || MULTIPLY_STRING.equals(cur) || ADD_STRING.equals(cur) || SUBTRACT_STRING.equals(cur))) { |
||||||
|
return true; |
||||||
|
} else if (DIVISION_STRING.equals(peek) && (DIVISION_STRING.equals(cur) || MULTIPLY_STRING.equals(cur) || ADD_STRING.equals(cur) || SUBTRACT_STRING.equals(cur))) { |
||||||
|
return true; |
||||||
|
} else if (ADD_STRING.equals(peek) && (ADD_STRING.equals(cur) || SUBTRACT_STRING.equals(cur))) { |
||||||
|
return true; |
||||||
|
} else { |
||||||
|
return SUBTRACT_STRING.equals(peek) && (ADD_STRING.equals(cur) || SUBTRACT_STRING.equals(cur)); |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Placeholder replacement resolver |
||||||
|
*/ |
||||||
|
private static class TimePlaceholderResolver implements |
||||||
|
PropertyPlaceholderHelper.PlaceholderResolver { |
||||||
|
|
||||||
|
private final String value; |
||||||
|
|
||||||
|
private final Date date; |
||||||
|
|
||||||
|
public TimePlaceholderResolver(String value, Date date) { |
||||||
|
this.value = value; |
||||||
|
this.date = date; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public String resolvePlaceholder(String placeholderName) { |
||||||
|
try { |
||||||
|
return calculateTime(placeholderName, date); |
||||||
|
} catch (Exception ex) { |
||||||
|
logger.error(String.format("resolve placeholder '%s' in [ %s ]" , placeholderName, value), ex); |
||||||
|
return null; |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* calculate time |
||||||
|
* |
||||||
|
* @param date date |
||||||
|
* @return calculate time |
||||||
|
*/ |
||||||
|
private static String calculateTime(String expression, Date date) { |
||||||
|
// After N years: $[add_months(yyyyMMdd,12*N)], the first N months: $[add_months(yyyyMMdd,-N)], etc
|
||||||
|
String value; |
||||||
|
|
||||||
|
try { |
||||||
|
if (expression.startsWith(Constants.TIMESTAMP)) { |
||||||
|
String timeExpression = expression.substring(Constants.TIMESTAMP.length() + 1, expression.length() - 1); |
||||||
|
|
||||||
|
Map.Entry<Date, String> entry = calcTimeExpression(timeExpression, date); |
||||||
|
|
||||||
|
String dateStr = DateUtils.format(entry.getKey(), entry.getValue()); |
||||||
|
|
||||||
|
Date timestamp = DateUtils.parse(dateStr, Constants.PARAMETER_FORMAT_TIME); |
||||||
|
|
||||||
|
value = String.valueOf(timestamp.getTime() / 1000); |
||||||
|
} else { |
||||||
|
Map.Entry<Date, String> entry = calcTimeExpression(expression, date); |
||||||
|
value = DateUtils.format(entry.getKey(), entry.getValue()); |
||||||
|
} |
||||||
|
} catch (Exception e) { |
||||||
|
logger.error(e.getMessage(), e); |
||||||
|
throw e; |
||||||
|
} |
||||||
|
|
||||||
|
return value; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* calculate time expresstion |
||||||
|
* |
||||||
|
* @return <date, date format> |
||||||
|
*/ |
||||||
|
public static Map.Entry<Date, String> calcTimeExpression(String expression, Date date) { |
||||||
|
Map.Entry<Date, String> resultEntry; |
||||||
|
|
||||||
|
if (expression.startsWith(Constants.ADD_MONTHS)) { |
||||||
|
resultEntry = calcMonths(expression, date); |
||||||
|
} else if (expression.startsWith(Constants.MONTH_BEGIN)) { |
||||||
|
resultEntry = calcMonthBegin(expression, date); |
||||||
|
} else if (expression.startsWith(Constants.MONTH_END)) { |
||||||
|
resultEntry = calcMonthEnd(expression, date); |
||||||
|
} else if (expression.startsWith(Constants.WEEK_BEGIN)) { |
||||||
|
resultEntry = calcWeekStart(expression, date); |
||||||
|
} else if (expression.startsWith(Constants.WEEK_END)) { |
||||||
|
resultEntry = calcWeekEnd(expression, date); |
||||||
|
} else { |
||||||
|
resultEntry = calcMinutes(expression, date); |
||||||
|
} |
||||||
|
|
||||||
|
return resultEntry; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get first day of month |
||||||
|
* |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static Map.Entry<Date, String> calcMonthBegin(String expression, Date date) { |
||||||
|
String addMonthExpr = expression.substring(Constants.MONTH_BEGIN.length() + 1, expression.length() - 1); |
||||||
|
String[] params = addMonthExpr.split(Constants.COMMA); |
||||||
|
|
||||||
|
if (params.length == 2) { |
||||||
|
String dateFormat = params[0]; |
||||||
|
String dayExpr = params[1]; |
||||||
|
Integer day = calculate(dayExpr); |
||||||
|
Date targetDate = DateUtils.getFirstDayOfMonth(date); |
||||||
|
targetDate = org.apache.commons.lang.time.DateUtils.addDays(targetDate, day); |
||||||
|
|
||||||
|
return new AbstractMap.SimpleImmutableEntry<>(targetDate, dateFormat); |
||||||
|
} |
||||||
|
|
||||||
|
throw new RuntimeException("expression not valid"); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get last day of month |
||||||
|
* |
||||||
|
*/ |
||||||
|
public static Map.Entry<Date, String> calcMonthEnd(String expression, Date date) { |
||||||
|
String addMonthExpr = expression.substring(Constants.MONTH_END.length() + 1, expression.length() - 1); |
||||||
|
String[] params = addMonthExpr.split(Constants.COMMA); |
||||||
|
|
||||||
|
if (params.length == 2) { |
||||||
|
String dateFormat = params[0]; |
||||||
|
String dayExpr = params[1]; |
||||||
|
Integer day = calculate(dayExpr); |
||||||
|
Date targetDate = DateUtils.getLastDayOfMonth(date); |
||||||
|
targetDate = org.apache.commons.lang.time.DateUtils.addDays(targetDate, day); |
||||||
|
|
||||||
|
return new AbstractMap.SimpleImmutableEntry<>(targetDate, dateFormat); |
||||||
|
} |
||||||
|
|
||||||
|
throw new RuntimeException("expression not valid"); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get first day of week |
||||||
|
* |
||||||
|
* @return monday |
||||||
|
*/ |
||||||
|
public static Map.Entry<Date, String> calcWeekStart(String expression, Date date) { |
||||||
|
String addMonthExpr = expression.substring(Constants.WEEK_BEGIN.length() + 1, expression.length() - 1); |
||||||
|
String[] params = addMonthExpr.split(Constants.COMMA); |
||||||
|
|
||||||
|
if (params.length == 2) { |
||||||
|
String dateFormat = params[0]; |
||||||
|
String dayExpr = params[1]; |
||||||
|
Integer day = calculate(dayExpr); |
||||||
|
Date targetDate = DateUtils.getMonday(date); |
||||||
|
targetDate = org.apache.commons.lang.time.DateUtils.addDays(targetDate, day); |
||||||
|
return new AbstractMap.SimpleImmutableEntry<>(targetDate, dateFormat); |
||||||
|
} |
||||||
|
|
||||||
|
throw new RuntimeException("expression not valid"); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get last day of week |
||||||
|
* |
||||||
|
*/ |
||||||
|
public static Map.Entry<Date, String> calcWeekEnd(String expression, Date date) { |
||||||
|
String addMonthExpr = expression.substring(Constants.WEEK_END.length() + 1, expression.length() - 1); |
||||||
|
String[] params = addMonthExpr.split(Constants.COMMA); |
||||||
|
|
||||||
|
if (params.length == 2) { |
||||||
|
String dateFormat = params[0]; |
||||||
|
String dayExpr = params[1]; |
||||||
|
Integer day = calculate(dayExpr); |
||||||
|
Date targetDate = DateUtils.getSunday(date); |
||||||
|
targetDate = org.apache.commons.lang.time.DateUtils.addDays(targetDate, day); |
||||||
|
|
||||||
|
return new AbstractMap.SimpleImmutableEntry<>(targetDate, dateFormat); |
||||||
|
} |
||||||
|
|
||||||
|
throw new RuntimeException("Expression not valid"); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* calc months expression |
||||||
|
* |
||||||
|
* @return <date, format> |
||||||
|
*/ |
||||||
|
public static Map.Entry<Date, String> calcMonths(String expression, Date date) { |
||||||
|
String addMonthExpr = expression.substring(Constants.ADD_MONTHS.length() + 1, expression.length() - 1); |
||||||
|
String[] params = addMonthExpr.split(Constants.COMMA); |
||||||
|
|
||||||
|
if (params.length == 2) { |
||||||
|
String dateFormat = params[0]; |
||||||
|
String monthExpr = params[1]; |
||||||
|
Integer addMonth = calculate(monthExpr); |
||||||
|
Date targetDate = org.apache.commons.lang.time.DateUtils.addMonths(date, addMonth); |
||||||
|
|
||||||
|
return new AbstractMap.SimpleImmutableEntry<>(targetDate, dateFormat); |
||||||
|
} |
||||||
|
|
||||||
|
throw new RuntimeException("expression not valid"); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* calculate time expression |
||||||
|
* |
||||||
|
* |
||||||
|
* @return <date, format> |
||||||
|
*/ |
||||||
|
public static Map.Entry<Date, String> calcMinutes(String expression, Date date) { |
||||||
|
if (expression.contains("+")) { |
||||||
|
int index = expression.lastIndexOf('+'); |
||||||
|
|
||||||
|
if (Character.isDigit(expression.charAt(index + 1))) { |
||||||
|
String addMinuteExpr = expression.substring(index + 1); |
||||||
|
Date targetDate = org.apache.commons.lang.time.DateUtils |
||||||
|
.addMinutes(date, calcMinutes(addMinuteExpr)); |
||||||
|
String dateFormat = expression.substring(0, index); |
||||||
|
|
||||||
|
return new AbstractMap.SimpleImmutableEntry<>(targetDate, dateFormat); |
||||||
|
} |
||||||
|
} else if (expression.contains("-")) { |
||||||
|
int index = expression.lastIndexOf('-'); |
||||||
|
|
||||||
|
if (Character.isDigit(expression.charAt(index + 1))) { |
||||||
|
String addMinuteExpr = expression.substring(index + 1); |
||||||
|
Date targetDate = org.apache.commons.lang.time.DateUtils |
||||||
|
.addMinutes(date, 0 - calcMinutes(addMinuteExpr)); |
||||||
|
String dateFormat = expression.substring(0, index); |
||||||
|
|
||||||
|
return new AbstractMap.SimpleImmutableEntry<>(targetDate, dateFormat); |
||||||
|
} |
||||||
|
|
||||||
|
// yyyy-MM-dd/HH:mm:ss
|
||||||
|
return new AbstractMap.SimpleImmutableEntry<>(date, expression); |
||||||
|
} |
||||||
|
|
||||||
|
// $[HHmmss]
|
||||||
|
return new AbstractMap.SimpleImmutableEntry<>(date, expression); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* calculate need minutes |
||||||
|
* |
||||||
|
* @param minuteExpression |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public static Integer calcMinutes(String minuteExpression) { |
||||||
|
int index = minuteExpression.indexOf("/"); |
||||||
|
|
||||||
|
String calcExpression; |
||||||
|
|
||||||
|
if (index == -1) { |
||||||
|
calcExpression = String.format("60*24*(%s)", minuteExpression); |
||||||
|
} else { |
||||||
|
|
||||||
|
calcExpression = String.format("60*24*(%s)%s", minuteExpression.substring(0, index), |
||||||
|
minuteExpression.substring(index)); |
||||||
|
} |
||||||
|
|
||||||
|
return calculate(calcExpression); |
||||||
|
} |
||||||
|
|
||||||
|
} |
@ -0,0 +1,340 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.zk; |
||||||
|
|
||||||
|
import cn.escheduler.common.Constants; |
||||||
|
import cn.escheduler.common.IStoppable; |
||||||
|
import cn.escheduler.common.utils.DateUtils; |
||||||
|
import cn.escheduler.common.utils.OSUtils; |
||||||
|
import org.apache.commons.configuration.Configuration; |
||||||
|
import org.apache.commons.configuration.ConfigurationException; |
||||||
|
import org.apache.commons.configuration.PropertiesConfiguration; |
||||||
|
import org.apache.curator.RetryPolicy; |
||||||
|
import org.apache.curator.framework.CuratorFramework; |
||||||
|
import org.apache.curator.framework.CuratorFrameworkFactory; |
||||||
|
import org.apache.curator.framework.imps.CuratorFrameworkState; |
||||||
|
import org.apache.curator.framework.state.ConnectionState; |
||||||
|
import org.apache.curator.framework.state.ConnectionStateListener; |
||||||
|
import org.apache.curator.retry.ExponentialBackoffRetry; |
||||||
|
import org.apache.zookeeper.CreateMode; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
import java.util.ArrayList; |
||||||
|
import java.util.Date; |
||||||
|
import java.util.List; |
||||||
|
|
||||||
|
import static cn.escheduler.common.Constants.*; |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* abstract zookeeper client |
||||||
|
*/ |
||||||
|
public abstract class AbstractZKClient { |
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(AbstractZKClient.class); |
||||||
|
|
||||||
|
/** |
||||||
|
* load configuration file |
||||||
|
*/ |
||||||
|
protected static Configuration conf; |
||||||
|
|
||||||
|
protected CuratorFramework zkClient = null; |
||||||
|
|
||||||
|
/** |
||||||
|
* server node parent path |
||||||
|
*/ |
||||||
|
protected String deadServerZNodeParentPath = null; |
||||||
|
|
||||||
|
/** |
||||||
|
* master node parent path |
||||||
|
*/ |
||||||
|
protected String masterZNodeParentPath = null; |
||||||
|
|
||||||
|
/** |
||||||
|
* worker node parent path |
||||||
|
*/ |
||||||
|
protected String workerZNodeParentPath = null; |
||||||
|
|
||||||
|
/** |
||||||
|
* server stop or not |
||||||
|
*/ |
||||||
|
protected IStoppable stoppable = null; |
||||||
|
|
||||||
|
|
||||||
|
static { |
||||||
|
try { |
||||||
|
conf = new PropertiesConfiguration(Constants.ZOOKEEPER_PROPERTIES_PATH); |
||||||
|
}catch (ConfigurationException e){ |
||||||
|
logger.error("load configuration failed : " + e.getMessage(),e); |
||||||
|
System.exit(1); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
public AbstractZKClient() { |
||||||
|
StringBuilder sb = new StringBuilder(); |
||||||
|
String[] zookeeperParamslist = conf.getStringArray(Constants.ZOOKEEPER_QUORUM); |
||||||
|
for (String param : zookeeperParamslist) { |
||||||
|
sb.append(param).append(Constants.COMMA); |
||||||
|
} |
||||||
|
|
||||||
|
if(sb.length() > 0){ |
||||||
|
sb.deleteCharAt(sb.length() - 1); |
||||||
|
} |
||||||
|
|
||||||
|
// retry strategy
|
||||||
|
RetryPolicy retryPolicy = new ExponentialBackoffRetry( |
||||||
|
Integer.parseInt(conf.getString(Constants.ZOOKEEPER_RETRY_SLEEP)), |
||||||
|
Integer.parseInt(conf.getString(Constants.ZOOKEEPER_RETRY_MAXTIME))); |
||||||
|
|
||||||
|
try{ |
||||||
|
// crate zookeeper client
|
||||||
|
zkClient = CuratorFrameworkFactory.builder() |
||||||
|
.connectString(sb.toString()) |
||||||
|
.retryPolicy(retryPolicy) |
||||||
|
.sessionTimeoutMs(1000 * Integer.parseInt(conf.getString(Constants.ZOOKEEPER_SESSION_TIMEOUT))) |
||||||
|
.connectionTimeoutMs(1000 * Integer.parseInt(conf.getString(Constants.ZOOKEEPER_CONNECTION_TIMEOUT))) |
||||||
|
.build(); |
||||||
|
|
||||||
|
zkClient.start(); |
||||||
|
initStateLister(); |
||||||
|
|
||||||
|
}catch(Exception e){ |
||||||
|
logger.error("create zookeeper connect failed : " + e.getMessage(),e); |
||||||
|
System.exit(-1); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* |
||||||
|
* register status monitoring events for zookeeper clients |
||||||
|
*/ |
||||||
|
public void initStateLister(){ |
||||||
|
if(zkClient == null) { |
||||||
|
return; |
||||||
|
} |
||||||
|
// add ConnectionStateListener monitoring zookeeper connection state
|
||||||
|
ConnectionStateListener csLister = new ConnectionStateListener() { |
||||||
|
|
||||||
|
@Override |
||||||
|
public void stateChanged(CuratorFramework client, ConnectionState newState) { |
||||||
|
logger.info("state changed , current state : " + newState.name()); |
||||||
|
/** |
||||||
|
* probably session expired |
||||||
|
*/ |
||||||
|
if(newState == ConnectionState.LOST){ |
||||||
|
// if lost , then exit
|
||||||
|
logger.info("current zookeepr connection state : connection lost "); |
||||||
|
} |
||||||
|
} |
||||||
|
}; |
||||||
|
|
||||||
|
zkClient.getConnectionStateListenable().addListener(csLister); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
public void start() { |
||||||
|
zkClient.start(); |
||||||
|
logger.info("zookeeper start ..."); |
||||||
|
} |
||||||
|
|
||||||
|
public void close() { |
||||||
|
zkClient.getZookeeperClient().close(); |
||||||
|
zkClient.close(); |
||||||
|
logger.info("zookeeper close ..."); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* heartbeat for zookeeper |
||||||
|
* @param znode |
||||||
|
*/ |
||||||
|
public void heartBeatForZk(String znode, String serverType){ |
||||||
|
try { |
||||||
|
|
||||||
|
//check dead or not in zookeeper
|
||||||
|
if(zkClient.getState() == CuratorFrameworkState.STOPPED || checkIsDeadServer(znode, serverType)){ |
||||||
|
stoppable.stop("i was judged to death, release resources and stop myself"); |
||||||
|
return; |
||||||
|
} |
||||||
|
|
||||||
|
byte[] bytes = zkClient.getData().forPath(znode); |
||||||
|
String resInfoStr = new String(bytes); |
||||||
|
String[] splits = resInfoStr.split(Constants.COMMA); |
||||||
|
if (splits.length != Constants.HEARTBEAT_FOR_ZOOKEEPER_INFO_LENGTH){ |
||||||
|
return; |
||||||
|
} |
||||||
|
String str = splits[0] + Constants.COMMA +splits[1] + Constants.COMMA |
||||||
|
+ OSUtils.cpuUsage() + Constants.COMMA |
||||||
|
+ OSUtils.memoryUsage() + Constants.COMMA |
||||||
|
+ splits[4] + Constants.COMMA |
||||||
|
+ DateUtils.dateToString(new Date()); |
||||||
|
zkClient.setData().forPath(znode,str.getBytes()); |
||||||
|
|
||||||
|
} catch (Exception e) { |
||||||
|
logger.error("heartbeat for zk failed : " + e.getMessage(), e); |
||||||
|
stoppable.stop("heartbeat for zk exception, release resources and stop myself"); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* check dead server or not , if dead, stop self |
||||||
|
* |
||||||
|
* @param zNode node path |
||||||
|
* @param serverType master or worker prefix |
||||||
|
* @throws Exception |
||||||
|
*/ |
||||||
|
protected boolean checkIsDeadServer(String zNode, String serverType) throws Exception { |
||||||
|
//ip_sequenceno
|
||||||
|
String[] zNodesPath = zNode.split("\\/"); |
||||||
|
String ipSeqNo = zNodesPath[zNodesPath.length - 1]; |
||||||
|
|
||||||
|
String type = serverType.equals(MASTER_PREFIX) ? MASTER_PREFIX : WORKER_PREFIX; |
||||||
|
String deadServerPath = deadServerZNodeParentPath + SINGLE_SLASH + type + UNDERLINE + ipSeqNo; |
||||||
|
|
||||||
|
if(zkClient.checkExists().forPath(zNode) == null || |
||||||
|
zkClient.checkExists().forPath(deadServerPath) != null ){ |
||||||
|
return true; |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* init system znode |
||||||
|
*/ |
||||||
|
protected void initSystemZNode(){ |
||||||
|
try { |
||||||
|
// read master node parent path from conf
|
||||||
|
masterZNodeParentPath = conf.getString(Constants.ZOOKEEPER_ESCHEDULER_MASTERS); |
||||||
|
// read worker node parent path from conf
|
||||||
|
workerZNodeParentPath = conf.getString(Constants.ZOOKEEPER_ESCHEDULER_WORKERS); |
||||||
|
|
||||||
|
// read server node parent path from conf
|
||||||
|
deadServerZNodeParentPath = conf.getString(ZOOKEEPER_ESCHEDULER_DEAD_SERVERS); |
||||||
|
|
||||||
|
if(zkClient.checkExists().forPath(deadServerZNodeParentPath) == null){ |
||||||
|
// create persistent dead server parent node
|
||||||
|
zkClient.create().creatingParentContainersIfNeeded() |
||||||
|
.withMode(CreateMode.PERSISTENT).forPath(deadServerZNodeParentPath); |
||||||
|
} |
||||||
|
|
||||||
|
if(zkClient.checkExists().forPath(masterZNodeParentPath) == null){ |
||||||
|
// create persistent master parent node
|
||||||
|
zkClient.create().creatingParentContainersIfNeeded() |
||||||
|
.withMode(CreateMode.PERSISTENT).forPath(masterZNodeParentPath); |
||||||
|
} |
||||||
|
|
||||||
|
if(zkClient.checkExists().forPath(workerZNodeParentPath) == null){ |
||||||
|
// create persistent worker parent node
|
||||||
|
zkClient.create().creatingParentContainersIfNeeded() |
||||||
|
.withMode(CreateMode.PERSISTENT).forPath(workerZNodeParentPath); |
||||||
|
} |
||||||
|
|
||||||
|
} catch (Exception e) { |
||||||
|
logger.error("init system znode failed : " + e.getMessage(),e); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
public void removeDeadServerByHost(String host, String serverType) throws Exception { |
||||||
|
List<String> deadServers = zkClient.getChildren().forPath(deadServerZNodeParentPath); |
||||||
|
for(String serverPath : deadServers){ |
||||||
|
if(serverPath.startsWith(serverType+UNDERLINE+host)){ |
||||||
|
|
||||||
|
String server = deadServerZNodeParentPath + SINGLE_SLASH + serverPath; |
||||||
|
zkClient.delete().forPath(server); |
||||||
|
logger.info("{} server {} deleted from zk dead server path success" , serverType , host); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* opType(add): if find dead server , then add to zk deadServerPath |
||||||
|
* opType(delete): delete path from zk |
||||||
|
* |
||||||
|
* @param zNode node path |
||||||
|
* @param serverType master or worker prefix |
||||||
|
* @param opType delete or add |
||||||
|
* @throws Exception |
||||||
|
*/ |
||||||
|
public void handleDeadServer(String zNode, String serverType, String opType) throws Exception { |
||||||
|
//ip_sequenceno
|
||||||
|
String[] zNodesPath = zNode.split("\\/"); |
||||||
|
String ipSeqNo = zNodesPath[zNodesPath.length - 1]; |
||||||
|
|
||||||
|
String type = serverType.equals(MASTER_PREFIX) ? MASTER_PREFIX : WORKER_PREFIX; |
||||||
|
|
||||||
|
|
||||||
|
//check server restart, if restart , dead server path in zk should be delete
|
||||||
|
if(opType.equals(DELETE_ZK_OP)){ |
||||||
|
String[] ipAndSeqNo = ipSeqNo.split(UNDERLINE); |
||||||
|
String ip = ipAndSeqNo[0]; |
||||||
|
removeDeadServerByHost(ip, serverType); |
||||||
|
|
||||||
|
}else if(opType.equals(ADD_ZK_OP)){ |
||||||
|
String deadServerPath = deadServerZNodeParentPath + SINGLE_SLASH + type + UNDERLINE + ipSeqNo; |
||||||
|
if(zkClient.checkExists().forPath(deadServerPath) == null){ |
||||||
|
//add dead server info to zk dead server path : /dead-servers/
|
||||||
|
|
||||||
|
zkClient.create().forPath(deadServerPath,(type + UNDERLINE + ipSeqNo).getBytes()); |
||||||
|
|
||||||
|
logger.info("{} server dead , and {} added to zk dead server path success" , serverType, zNode); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* for stop server |
||||||
|
* @param serverStoppable |
||||||
|
*/ |
||||||
|
public void setStoppable(IStoppable serverStoppable){ |
||||||
|
this.stoppable = serverStoppable; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get active master num |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public int getActiveMasterNum(){ |
||||||
|
List<String> childrenList = new ArrayList<>(); |
||||||
|
try { |
||||||
|
// read master node parent path from conf
|
||||||
|
masterZNodeParentPath = conf.getString(Constants.ZOOKEEPER_ESCHEDULER_MASTERS); |
||||||
|
if(zkClient.checkExists().forPath(masterZNodeParentPath) != null){ |
||||||
|
childrenList = zkClient.getChildren().forPath(masterZNodeParentPath); |
||||||
|
} |
||||||
|
} catch (Exception e) { |
||||||
|
logger.warn(e.getMessage(),e); |
||||||
|
return childrenList.size(); |
||||||
|
} |
||||||
|
return childrenList.size(); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public String toString() { |
||||||
|
return "AbstractZKClient{" + |
||||||
|
"zkClient=" + zkClient + |
||||||
|
", deadServerZNodeParentPath='" + deadServerZNodeParentPath + '\'' + |
||||||
|
", masterZNodeParentPath='" + masterZNodeParentPath + '\'' + |
||||||
|
", workerZNodeParentPath='" + workerZNodeParentPath + '\'' + |
||||||
|
", stoppable=" + stoppable + |
||||||
|
'}'; |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,110 @@ |
|||||||
|
#!/usr/bin/env bash |
||||||
|
|
||||||
|
usage="Usage: escheduler-daemon.sh (start|stop) <command> " |
||||||
|
|
||||||
|
# if no args specified, show usage |
||||||
|
if [ $# -le 1 ]; then |
||||||
|
echo $usage |
||||||
|
exit 1 |
||||||
|
fi |
||||||
|
|
||||||
|
startStop=$1 |
||||||
|
shift |
||||||
|
command=$1 |
||||||
|
shift |
||||||
|
|
||||||
|
echo "Begin $startStop $command......" |
||||||
|
|
||||||
|
BIN_DIR=`dirname $0` |
||||||
|
BIN_DIR=`cd "$BIN_DIR"; pwd` |
||||||
|
ESCHEDULER_HOME=$BIN_DIR/.. |
||||||
|
|
||||||
|
export JAVA_HOME=$JAVA_HOME |
||||||
|
#export JAVA_HOME=/opt/soft/jdk |
||||||
|
export HOSTNAME=`hostname` |
||||||
|
|
||||||
|
export ESCHEDULER_PID_DIR=/tmp/ |
||||||
|
export ESCHEDULER_LOG_DIR=$ESCHEDULER_HOME/logs |
||||||
|
export ESCHEDULER_CONF_DIR=$ESCHEDULER_HOME/conf |
||||||
|
export ESCHEDULER_LIB_JARS=$ESCHEDULER_HOME/lib/* |
||||||
|
|
||||||
|
export ESCHEDULER_OPTS="-server -Xmx16g -Xms4g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70" |
||||||
|
export STOP_TIMEOUT=5 |
||||||
|
|
||||||
|
if [ ! -d "$ESCHEDULER_LOG_DIR" ]; then |
||||||
|
mkdir $ESCHEDULER_LOG_DIR |
||||||
|
fi |
||||||
|
|
||||||
|
log=$ESCHEDULER_LOG_DIR/escheduler-$command-$HOSTNAME.out |
||||||
|
pid=$ESCHEDULER_LOG_DIR/escheduler-$command.pid |
||||||
|
|
||||||
|
cd $ESCHEDULER_HOME |
||||||
|
|
||||||
|
if [ "$command" = "api-server" ]; then |
||||||
|
LOG_FILE="-Dlogging.config=conf/apiserver_logback.xml" |
||||||
|
CLASS=cn.escheduler.api.ApiApplicationServer |
||||||
|
elif [ "$command" = "master-server" ]; then |
||||||
|
LOG_FILE="-Dspring.config.location=conf/application_master.properties -Ddruid.mysql.usePingMethod=false" |
||||||
|
CLASS=cn.escheduler.server.master.MasterServer |
||||||
|
elif [ "$command" = "worker-server" ]; then |
||||||
|
LOG_FILE="-Dlogback.configurationFile=conf/worker_logback.xml -Ddruid.mysql.usePingMethod=false" |
||||||
|
CLASS=cn.escheduler.server.worker.WorkerServer |
||||||
|
elif [ "$command" = "alert-server" ]; then |
||||||
|
LOG_FILE="-Dlogback.configurationFile=conf/alert_logback.xml" |
||||||
|
CLASS=cn.escheduler.alert.AlertServer |
||||||
|
elif [ "$command" = "logger-server" ]; then |
||||||
|
CLASS=cn.escheduler.server.rpc.LoggerServer |
||||||
|
else |
||||||
|
echo "Error: No command named \`$command' was found." |
||||||
|
exit 1 |
||||||
|
fi |
||||||
|
|
||||||
|
case $startStop in |
||||||
|
(start) |
||||||
|
[ -w "$ESCHEDULER_PID_DIR" ] || mkdir -p "$ESCHEDULER_PID_DIR" |
||||||
|
|
||||||
|
if [ -f $pid ]; then |
||||||
|
if kill -0 `cat $pid` > /dev/null 2>&1; then |
||||||
|
echo $command running as process `cat $pid`. Stop it first. |
||||||
|
exit 1 |
||||||
|
fi |
||||||
|
fi |
||||||
|
|
||||||
|
echo starting $command, logging to $log |
||||||
|
|
||||||
|
exec_command="$LOG_FILE $ESCHEDULER_OPTS -classpath $ESCHEDULER_CONF_DIR:$ESCHEDULER_LIB_JARS $CLASS" |
||||||
|
|
||||||
|
echo "nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 < /dev/null &" |
||||||
|
nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 < /dev/null & |
||||||
|
echo $! > $pid |
||||||
|
;; |
||||||
|
|
||||||
|
(stop) |
||||||
|
|
||||||
|
if [ -f $pid ]; then |
||||||
|
TARGET_PID=`cat $pid` |
||||||
|
if kill -0 $TARGET_PID > /dev/null 2>&1; then |
||||||
|
echo stopping $command |
||||||
|
kill $TARGET_PID |
||||||
|
sleep $STOP_TIMEOUT |
||||||
|
if kill -0 $TARGET_PID > /dev/null 2>&1; then |
||||||
|
echo "$command did not stop gracefully after $STOP_TIMEOUT seconds: killing with kill -9" |
||||||
|
kill -9 $TARGET_PID |
||||||
|
fi |
||||||
|
else |
||||||
|
echo no $command to stop |
||||||
|
fi |
||||||
|
rm -f $pid |
||||||
|
else |
||||||
|
echo no $command to stop |
||||||
|
fi |
||||||
|
;; |
||||||
|
|
||||||
|
(*) |
||||||
|
echo $usage |
||||||
|
exit 1 |
||||||
|
;; |
||||||
|
|
||||||
|
esac |
||||||
|
|
||||||
|
echo "End $startStop $command." |
@ -0,0 +1,28 @@ |
|||||||
|
#task queue implementation, default "zookeeper" |
||||||
|
escheduler.queue.impl=zookeeper |
||||||
|
|
||||||
|
# user data directory path, self configuration, please make sure the directory exists and have read write permissions |
||||||
|
data.basedir.path=/tmp/escheduler |
||||||
|
|
||||||
|
# directory path for user data download. self configuration, please make sure the directory exists and have read write permissions |
||||||
|
data.download.basedir.path=/tmp/escheduler/download |
||||||
|
|
||||||
|
# process execute directory. self configuration, please make sure the directory exists and have read write permissions |
||||||
|
process.exec.basepath=/tmp/escheduler/exec |
||||||
|
|
||||||
|
# data base dir, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/escheduler" is recommended |
||||||
|
data.store2hdfs.basepath=/escheduler |
||||||
|
|
||||||
|
# whether hdfs starts |
||||||
|
hdfs.startup.state=true |
||||||
|
|
||||||
|
# system env path. self configuration, please make sure the directory and file exists and have read write execute permissions |
||||||
|
escheduler.env.path=/opt/.escheduler_env.sh |
||||||
|
escheduler.env.py=/opt/escheduler_env.py |
||||||
|
|
||||||
|
#resource.view.suffixs |
||||||
|
resource.view.suffixs=txt,log,sh,conf,cfg,py,java,sql,hql,xml |
||||||
|
|
||||||
|
# is development state? default "false" |
||||||
|
development.state=false |
||||||
|
|
@ -0,0 +1,8 @@ |
|||||||
|
# ha or single namenode,If namenode ha needs to copy core-site.xml and hdfs-site.xml to the conf directory |
||||||
|
fs.defaultFS=hdfs://mycluster:8020 |
||||||
|
|
||||||
|
#resourcemanager ha note this need ips , this empty if single |
||||||
|
yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx |
||||||
|
|
||||||
|
# If it is a single resourcemanager, you only need to configure one host name. If it is resourcemanager HA, the default configuration is fine |
||||||
|
yarn.application.status.address=http://ark1:8088/ws/v1/cluster/apps/%s |
@ -0,0 +1,39 @@ |
|||||||
|
#============================================================================ |
||||||
|
# Configure Main Scheduler Properties |
||||||
|
#============================================================================ |
||||||
|
org.quartz.scheduler.instanceName = EasyScheduler |
||||||
|
org.quartz.scheduler.instanceId = AUTO |
||||||
|
org.quartz.scheduler.makeSchedulerThreadDaemon = true |
||||||
|
org.quartz.jobStore.useProperties = false |
||||||
|
|
||||||
|
#============================================================================ |
||||||
|
# Configure ThreadPool |
||||||
|
#============================================================================ |
||||||
|
|
||||||
|
org.quartz.threadPool.class = org.quartz.simpl.SimpleThreadPool |
||||||
|
org.quartz.threadPool.makeThreadsDaemons = true |
||||||
|
org.quartz.threadPool.threadCount = 25 |
||||||
|
org.quartz.threadPool.threadPriority = 5 |
||||||
|
|
||||||
|
#============================================================================ |
||||||
|
# Configure JobStore |
||||||
|
#============================================================================ |
||||||
|
|
||||||
|
org.quartz.jobStore.class = org.quartz.impl.jdbcjobstore.JobStoreTX |
||||||
|
org.quartz.jobStore.driverDelegateClass = org.quartz.impl.jdbcjobstore.StdJDBCDelegate |
||||||
|
org.quartz.jobStore.tablePrefix = QRTZ_ |
||||||
|
org.quartz.jobStore.isClustered = true |
||||||
|
org.quartz.jobStore.misfireThreshold = 60000 |
||||||
|
org.quartz.jobStore.clusterCheckinInterval = 5000 |
||||||
|
org.quartz.jobStore.dataSource = myDs |
||||||
|
|
||||||
|
#============================================================================ |
||||||
|
# Configure Datasources |
||||||
|
#============================================================================ |
||||||
|
|
||||||
|
org.quartz.dataSource.myDs.driver = com.mysql.jdbc.Driver |
||||||
|
org.quartz.dataSource.myDs.URL = jdbc:mysql://192.168.xx.xx:3306/escheduler?characterEncoding=utf8 |
||||||
|
org.quartz.dataSource.myDs.user = xx |
||||||
|
org.quartz.dataSource.myDs.password = xx |
||||||
|
org.quartz.dataSource.myDs.maxConnections = 10 |
||||||
|
org.quartz.dataSource.myDs.validationQuery = select 1 |
@ -0,0 +1,24 @@ |
|||||||
|
#zookeeper cluster |
||||||
|
zookeeper.quorum=192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181 |
||||||
|
|
||||||
|
#escheduler root directory |
||||||
|
zookeeper.escheduler.root=/escheduler |
||||||
|
|
||||||
|
#zookeeper server dirctory |
||||||
|
zookeeper.escheduler.dead.servers=/escheduler/dead-servers |
||||||
|
zookeeper.escheduler.masters=/escheduler/masters |
||||||
|
zookeeper.escheduler.workers=/escheduler/workers |
||||||
|
|
||||||
|
#zookeeper lock dirctory |
||||||
|
zookeeper.escheduler.lock.masters=/escheduler/lock/masters |
||||||
|
zookeeper.escheduler.lock.workers=/escheduler/lock/workers |
||||||
|
|
||||||
|
#escheduler failover directory |
||||||
|
zookeeper.escheduler.lock.failover.masters=/escheduler/lock/failover/masters |
||||||
|
zookeeper.escheduler.lock.failover.workers=/escheduler/lock/failover/workers |
||||||
|
|
||||||
|
#escheduler failover directory |
||||||
|
zookeeper.session.timeout=300 |
||||||
|
zookeeper.connection.timeout=300 |
||||||
|
zookeeper.retry.sleep=1000 |
||||||
|
zookeeper.retry.maxtime=5 |
@ -0,0 +1,355 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.graph; |
||||||
|
|
||||||
|
import org.junit.After; |
||||||
|
import org.junit.Before; |
||||||
|
import org.junit.Test; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
import java.util.ArrayList; |
||||||
|
import java.util.Arrays; |
||||||
|
import java.util.List; |
||||||
|
|
||||||
|
import static org.junit.Assert.*; |
||||||
|
|
||||||
|
public class DAGTest { |
||||||
|
private DAG<Integer, String, String> graph; |
||||||
|
private static final Logger logger = LoggerFactory.getLogger(DAGTest.class); |
||||||
|
|
||||||
|
@Before |
||||||
|
public void setup() { |
||||||
|
graph = new DAG<>(); |
||||||
|
} |
||||||
|
|
||||||
|
@After |
||||||
|
public void tearDown() { |
||||||
|
clear(); |
||||||
|
} |
||||||
|
|
||||||
|
private void clear() { |
||||||
|
graph = null; |
||||||
|
graph = new DAG<>(); |
||||||
|
|
||||||
|
assertEquals(graph.getNodesCount(), 0); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
private void makeGraph() { |
||||||
|
clear(); |
||||||
|
|
||||||
|
// 1->2
|
||||||
|
// 2->5
|
||||||
|
// 3->5
|
||||||
|
// 4->6
|
||||||
|
// 5->6
|
||||||
|
// 6->7
|
||||||
|
|
||||||
|
for (int i = 1; i <= 7; ++i) { |
||||||
|
graph.addNode(i, "v(" + i + ")"); |
||||||
|
} |
||||||
|
|
||||||
|
// 构造边
|
||||||
|
assertTrue(graph.addEdge(1, 2)); |
||||||
|
|
||||||
|
assertTrue(graph.addEdge(2, 5)); |
||||||
|
|
||||||
|
assertTrue(graph.addEdge(3, 5)); |
||||||
|
|
||||||
|
assertTrue(graph.addEdge(4, 6)); |
||||||
|
|
||||||
|
assertTrue(graph.addEdge(5, 6)); |
||||||
|
|
||||||
|
assertTrue(graph.addEdge(6, 7)); |
||||||
|
|
||||||
|
assertEquals(graph.getNodesCount(), 7); |
||||||
|
assertEquals(graph.getEdgesCount(), 6); |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* 测试增加顶点 |
||||||
|
*/ |
||||||
|
@Test |
||||||
|
public void testAddNode() { |
||||||
|
clear(); |
||||||
|
|
||||||
|
graph.addNode(1, "v(1)"); |
||||||
|
graph.addNode(2, null); |
||||||
|
graph.addNode(5, "v(5)"); |
||||||
|
|
||||||
|
assertEquals(graph.getNodesCount(), 3); |
||||||
|
|
||||||
|
assertEquals(graph.getNode(1), "v(1)"); |
||||||
|
assertTrue(graph.containsNode(1)); |
||||||
|
|
||||||
|
assertFalse(graph.containsNode(10)); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* 添加边 |
||||||
|
*/ |
||||||
|
@Test |
||||||
|
public void testAddEdge() { |
||||||
|
clear(); |
||||||
|
|
||||||
|
assertFalse(graph.addEdge(1, 2, "edge(1 -> 2)", false)); |
||||||
|
|
||||||
|
graph.addNode(1, "v(1)"); |
||||||
|
|
||||||
|
assertTrue(graph.addEdge(1, 2, "edge(1 -> 2)",true)); |
||||||
|
|
||||||
|
graph.addNode(2, "v(2)"); |
||||||
|
|
||||||
|
assertTrue(graph.addEdge(1, 2, "edge(1 -> 2)",true)); |
||||||
|
|
||||||
|
assertFalse(graph.containsEdge(1, 3)); |
||||||
|
|
||||||
|
assertTrue(graph.containsEdge(1, 2)); |
||||||
|
assertEquals(graph.getEdgesCount(), 1); |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* 测试后续结点 |
||||||
|
*/ |
||||||
|
@Test |
||||||
|
public void testSubsequentNodes() { |
||||||
|
makeGraph(); |
||||||
|
|
||||||
|
assertEquals(graph.getSubsequentNodes(1).size(), 1); |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* 测试入度 |
||||||
|
*/ |
||||||
|
@Test |
||||||
|
public void testIndegree() { |
||||||
|
makeGraph(); |
||||||
|
|
||||||
|
assertEquals(graph.getIndegree(1), 0); |
||||||
|
assertEquals(graph.getIndegree(2), 1); |
||||||
|
assertEquals(graph.getIndegree(3), 0); |
||||||
|
assertEquals(graph.getIndegree(4), 0); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* 测试起点 |
||||||
|
*/ |
||||||
|
@Test |
||||||
|
public void testBeginNode() { |
||||||
|
makeGraph(); |
||||||
|
|
||||||
|
assertEquals(graph.getBeginNode().size(), 3); |
||||||
|
|
||||||
|
assertTrue(graph.getBeginNode().contains(1)); |
||||||
|
assertTrue(graph.getBeginNode().contains(3)); |
||||||
|
assertTrue(graph.getBeginNode().contains(4)); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* 测试终点 |
||||||
|
*/ |
||||||
|
@Test |
||||||
|
public void testEndNode() { |
||||||
|
makeGraph(); |
||||||
|
|
||||||
|
assertEquals(graph.getEndNode().size(), 1); |
||||||
|
|
||||||
|
assertTrue(graph.getEndNode().contains(7)); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* 测试环 |
||||||
|
*/ |
||||||
|
@Test |
||||||
|
public void testCycle() { |
||||||
|
clear(); |
||||||
|
|
||||||
|
// 构造顶点
|
||||||
|
for (int i = 1; i <= 5; ++i) { |
||||||
|
graph.addNode(i, "v(" + i + ")"); |
||||||
|
} |
||||||
|
|
||||||
|
// 构造边, 1->2, 2->3, 3->4
|
||||||
|
try { |
||||||
|
graph.addEdge(1, 2); |
||||||
|
graph.addEdge(2, 3); |
||||||
|
graph.addEdge(3, 4); |
||||||
|
|
||||||
|
assertFalse(graph.hasCycle()); |
||||||
|
} catch (Exception e) { |
||||||
|
e.printStackTrace(); |
||||||
|
fail(); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
try { |
||||||
|
boolean addResult = graph.addEdge(4, 1);//有环,添加失败
|
||||||
|
|
||||||
|
if(!addResult){//有环,添加失败
|
||||||
|
assertTrue(true); |
||||||
|
} |
||||||
|
|
||||||
|
graph.addEdge(5, 1); |
||||||
|
|
||||||
|
assertFalse(graph.hasCycle()); |
||||||
|
} catch (Exception e) { |
||||||
|
e.printStackTrace(); |
||||||
|
fail(); |
||||||
|
} |
||||||
|
|
||||||
|
// 重新清空
|
||||||
|
clear(); |
||||||
|
|
||||||
|
// 构造顶点
|
||||||
|
for (int i = 1; i <= 5; ++i) { |
||||||
|
graph.addNode(i, "v(" + i +")"); |
||||||
|
} |
||||||
|
|
||||||
|
// 构造边, 1->2, 2->3, 3->4
|
||||||
|
try { |
||||||
|
graph.addEdge(1, 2); |
||||||
|
graph.addEdge(2, 3); |
||||||
|
graph.addEdge(3, 4); |
||||||
|
graph.addEdge(4, 5); |
||||||
|
graph.addEdge(5, 2);//会失败,添加不进去,所以下一步无环
|
||||||
|
|
||||||
|
assertFalse(graph.hasCycle()); |
||||||
|
} catch (Exception e) { |
||||||
|
e.printStackTrace(); |
||||||
|
fail(); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
@Test |
||||||
|
public void testTopologicalSort(){ |
||||||
|
makeGraph(); |
||||||
|
|
||||||
|
try { |
||||||
|
List<Integer> topoList = new ArrayList<>();//一种拓扑结果是1 3 4 2 5 6 7
|
||||||
|
topoList.add(1); |
||||||
|
topoList.add(3); |
||||||
|
topoList.add(4); |
||||||
|
topoList.add(2); |
||||||
|
topoList.add(5); |
||||||
|
topoList.add(6); |
||||||
|
topoList.add(7); |
||||||
|
|
||||||
|
assertEquals(graph.topologicalSort(),topoList); |
||||||
|
} catch (Exception e) { |
||||||
|
e.printStackTrace(); |
||||||
|
fail(); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
@Test |
||||||
|
public void testTopologicalSort2() { |
||||||
|
clear(); |
||||||
|
|
||||||
|
graph.addEdge(1, 2, null, true); |
||||||
|
graph.addEdge(2, 3, null, true); |
||||||
|
graph.addEdge(3, 4, null, true); |
||||||
|
graph.addEdge(4, 5, null, true); |
||||||
|
graph.addEdge(5, 1, null, false); //因环会添加失败,ERROR级别日志输出
|
||||||
|
|
||||||
|
try { |
||||||
|
List<Integer> topoList = new ArrayList<>();//拓扑结果是1 2 3 4 5
|
||||||
|
topoList.add(1); |
||||||
|
topoList.add(2); |
||||||
|
topoList.add(3); |
||||||
|
topoList.add(4); |
||||||
|
topoList.add(5); |
||||||
|
|
||||||
|
assertEquals(graph.topologicalSort(),topoList); |
||||||
|
|
||||||
|
} catch (Exception e) { |
||||||
|
e.printStackTrace(); |
||||||
|
fail(); |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* |
||||||
|
*/ |
||||||
|
@Test |
||||||
|
public void testTopologicalSort3() throws Exception { |
||||||
|
clear(); |
||||||
|
|
||||||
|
// 1->2
|
||||||
|
// 1->3
|
||||||
|
// 2->5
|
||||||
|
// 3->4
|
||||||
|
// 4->6
|
||||||
|
// 5->6
|
||||||
|
// 6->7
|
||||||
|
// 6->8
|
||||||
|
|
||||||
|
for (int i = 1; i <= 8; ++i) { |
||||||
|
graph.addNode(i, "v(" + i + ")"); |
||||||
|
} |
||||||
|
|
||||||
|
// 构造边
|
||||||
|
assertTrue(graph.addEdge(1, 2)); |
||||||
|
|
||||||
|
assertTrue(graph.addEdge(1, 3)); |
||||||
|
|
||||||
|
assertTrue(graph.addEdge(2, 5)); |
||||||
|
assertTrue(graph.addEdge(3, 4)); |
||||||
|
|
||||||
|
assertTrue(graph.addEdge(4, 6)); |
||||||
|
|
||||||
|
assertTrue(graph.addEdge(5, 6)); |
||||||
|
|
||||||
|
assertTrue(graph.addEdge(6, 7)); |
||||||
|
assertTrue(graph.addEdge(6, 8)); |
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
assertEquals(graph.getNodesCount(), 8); |
||||||
|
|
||||||
|
logger.info(Arrays.toString(graph.topologicalSort().toArray())); |
||||||
|
|
||||||
|
List<Integer> expectedList = new ArrayList<>(); |
||||||
|
|
||||||
|
for (int i = 1; i <= 8; ++i) { |
||||||
|
expectedList.add(i); |
||||||
|
|
||||||
|
logger.info(i + " subsequentNodes : " + graph.getSubsequentNodes(i)); |
||||||
|
} |
||||||
|
|
||||||
|
assertArrayEquals(expectedList.toArray(),graph.topologicalSort().toArray()); |
||||||
|
|
||||||
|
logger.info(6 + " previousNodesb: " + graph.getPreviousNodes(6)); |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
} |
@ -0,0 +1,140 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.os; |
||||||
|
|
||||||
|
|
||||||
|
import cn.escheduler.common.utils.OSUtils; |
||||||
|
import org.junit.Test; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
import oshi.hardware.GlobalMemory; |
||||||
|
|
||||||
|
import java.math.RoundingMode; |
||||||
|
import java.text.DecimalFormat; |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* OSUtilsTest |
||||||
|
*/ |
||||||
|
public class OSUtilsTest { |
||||||
|
|
||||||
|
private static Logger logger = LoggerFactory.getLogger(OSUtilsTest.class); |
||||||
|
// static SystemInfo si = new SystemInfo();
|
||||||
|
// static HardwareAbstractionLayer hal = si.getHardware();
|
||||||
|
|
||||||
|
|
||||||
|
@Test |
||||||
|
public void memoryUsage() { |
||||||
|
logger.info("memoryUsage : {}", OSUtils.memoryUsage());// 0.3361799418926239
|
||||||
|
// printMemory(hal.getMemory());// 35 %
|
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void availablePhysicalMemorySize() { |
||||||
|
logger.info("availablePhysicalMemorySize : {}", OSUtils.availablePhysicalMemorySize()); |
||||||
|
logger.info("availablePhysicalMemorySize : {}", OSUtils.totalMemorySize() / 10); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
@Test |
||||||
|
public void loadAverage() { |
||||||
|
logger.info("memoryUsage : {}", OSUtils.loadAverage()); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
private void printMemory(GlobalMemory memory) { |
||||||
|
logger.info("memoryUsage : {} %" , (memory.getTotal() - memory.getAvailable()) * 100 / memory.getTotal() ); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
@Test |
||||||
|
public void cpuUsage() throws Exception { |
||||||
|
logger.info("cpuUsage : {}", OSUtils.cpuUsage()); |
||||||
|
Thread.sleep(1000l); |
||||||
|
logger.info("cpuUsage : {}", OSUtils.cpuUsage()); |
||||||
|
|
||||||
|
double cpuUsage = OSUtils.cpuUsage(); |
||||||
|
|
||||||
|
DecimalFormat df = new DecimalFormat("0.00"); |
||||||
|
|
||||||
|
df.setRoundingMode(RoundingMode.HALF_UP); |
||||||
|
|
||||||
|
logger.info("cpuUsage1 : {}", df.format(cpuUsage)); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
//
|
||||||
|
// @Test
|
||||||
|
// public void getUserList() {
|
||||||
|
// logger.info("getUserList : {}", OSUtils.getUserList());
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
//
|
||||||
|
// @Test
|
||||||
|
// public void getGroup() throws Exception {
|
||||||
|
// logger.info("getGroup : {}", OSUtils.getGroup());
|
||||||
|
// logger.info("getGroup : {}", OSUtils.exeShell("groups"));
|
||||||
|
//
|
||||||
|
//
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
//
|
||||||
|
// @Test
|
||||||
|
// public void getProcessID() {
|
||||||
|
// logger.info("getProcessID : {}", OSUtils.getProcessID());
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
//
|
||||||
|
// @Test
|
||||||
|
// public void getHost() {
|
||||||
|
// logger.info("getHost : {}", OSUtils.getHost());
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
//
|
||||||
|
//
|
||||||
|
// @Test
|
||||||
|
// public void anotherGetOsInfoTest() throws InterruptedException {
|
||||||
|
// OperatingSystemMXBean os = ManagementFactory.getPlatformMXBean(OperatingSystemMXBean.class);
|
||||||
|
// final MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean();
|
||||||
|
//
|
||||||
|
// MemoryUsage memoryUsage = memoryMXBean.getHeapMemoryUsage();
|
||||||
|
// double usage = (double)memoryUsage.getUsed() / (double)memoryUsage.getCommitted();
|
||||||
|
// logger.info("memory usage : {}",usage);
|
||||||
|
//
|
||||||
|
// if (os instanceof UnixOperatingSystemMXBean) {
|
||||||
|
// UnixOperatingSystemMXBean unixOs = (UnixOperatingSystemMXBean) os;
|
||||||
|
// logger.info("getMaxFileDescriptorCount : {}" ,unixOs.getMaxFileDescriptorCount()); //10240
|
||||||
|
// logger.info("getOpenFileDescriptorCount : {}",unixOs.getOpenFileDescriptorCount()); //241
|
||||||
|
// logger.info("getAvailableProcessors : {}",unixOs.getAvailableProcessors()); //8
|
||||||
|
//
|
||||||
|
// logger.info("getSystemLoadAverage : {}",unixOs.getSystemLoadAverage()); //1.36083984375
|
||||||
|
//
|
||||||
|
// logger.info("getFreePhysicalMemorySize : {}",unixOs.getFreePhysicalMemorySize()); //209768448
|
||||||
|
//
|
||||||
|
// logger.info("getTotalPhysicalMemorySize : {}",unixOs.getTotalPhysicalMemorySize()); //17179869184 16G
|
||||||
|
//
|
||||||
|
// for(int i = 0; i < 3; i++) {
|
||||||
|
// logger.info("getSystemCpuLoad : {}", unixOs.getSystemCpuLoad()); //0.0
|
||||||
|
//
|
||||||
|
// logger.info("getProcessCpuLoad : {}", unixOs.getProcessCpuLoad() * 10); //0.0
|
||||||
|
// Thread.sleep(1000l);
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
|
||||||
|
} |
@ -0,0 +1,112 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.os; |
||||||
|
|
||||||
|
|
||||||
|
import org.junit.Test; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
import oshi.SystemInfo; |
||||||
|
import oshi.hardware.CentralProcessor; |
||||||
|
import oshi.hardware.CentralProcessor.TickType; |
||||||
|
import oshi.hardware.GlobalMemory; |
||||||
|
import oshi.hardware.HardwareAbstractionLayer; |
||||||
|
import oshi.util.FormatUtil; |
||||||
|
import oshi.util.Util; |
||||||
|
|
||||||
|
import java.util.Arrays; |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* os information test |
||||||
|
*/ |
||||||
|
public class OshiTest { |
||||||
|
|
||||||
|
private static Logger logger = LoggerFactory.getLogger(OshiTest.class); |
||||||
|
|
||||||
|
|
||||||
|
@Test |
||||||
|
public void test() { |
||||||
|
|
||||||
|
SystemInfo si = new SystemInfo(); |
||||||
|
|
||||||
|
HardwareAbstractionLayer hal = si.getHardware(); |
||||||
|
|
||||||
|
logger.info("Checking Memory..."); |
||||||
|
printMemory(hal.getMemory()); |
||||||
|
|
||||||
|
|
||||||
|
logger.info("Checking CPU..."); |
||||||
|
printCpu(hal.getProcessor()); |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
private static void printMemory(GlobalMemory memory) { |
||||||
|
|
||||||
|
logger.info("memory avail:{} MB" , memory.getAvailable() / 1024 / 1024 );//memory avail:6863 MB
|
||||||
|
logger.info("memory total:{} MB" , memory.getTotal() / 1024 / 1024 );//memory total:16384 MB
|
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
private static void printCpu(CentralProcessor processor) { |
||||||
|
logger.info(String.format("CPU load: %.1f%% (OS MXBean)%n", processor.getSystemCpuLoad() * 100));//CPU load: 24.9% (OS MXBean)
|
||||||
|
logger.info("CPU load averages : {}", processor.getSystemLoadAverage());//CPU load averages : 1.5234375
|
||||||
|
|
||||||
|
|
||||||
|
logger.info("Uptime: " + FormatUtil.formatElapsedSecs(processor.getSystemUptime())); |
||||||
|
logger.info("Context Switches/Interrupts: " + processor.getContextSwitches() + " / " + processor.getInterrupts()); |
||||||
|
|
||||||
|
|
||||||
|
long[] prevTicks = processor.getSystemCpuLoadTicks(); |
||||||
|
logger.info("CPU, IOWait, and IRQ ticks @ 0 sec:" + Arrays.toString(prevTicks)); |
||||||
|
//Wait a second...
|
||||||
|
Util.sleep(1000); |
||||||
|
long[] ticks = processor.getSystemCpuLoadTicks(); |
||||||
|
logger.info("CPU, IOWait, and IRQ ticks @ 1 sec:" + Arrays.toString(ticks)); |
||||||
|
long user = ticks[TickType.USER.getIndex()] - prevTicks[TickType.USER.getIndex()]; |
||||||
|
long nice = ticks[TickType.NICE.getIndex()] - prevTicks[TickType.NICE.getIndex()]; |
||||||
|
long sys = ticks[TickType.SYSTEM.getIndex()] - prevTicks[TickType.SYSTEM.getIndex()]; |
||||||
|
long idle = ticks[TickType.IDLE.getIndex()] - prevTicks[TickType.IDLE.getIndex()]; |
||||||
|
long iowait = ticks[TickType.IOWAIT.getIndex()] - prevTicks[TickType.IOWAIT.getIndex()]; |
||||||
|
long irq = ticks[TickType.IRQ.getIndex()] - prevTicks[TickType.IRQ.getIndex()]; |
||||||
|
long softirq = ticks[TickType.SOFTIRQ.getIndex()] - prevTicks[TickType.SOFTIRQ.getIndex()]; |
||||||
|
long steal = ticks[TickType.STEAL.getIndex()] - prevTicks[TickType.STEAL.getIndex()]; |
||||||
|
long totalCpu = user + nice + sys + idle + iowait + irq + softirq + steal; |
||||||
|
|
||||||
|
logger.info(String.format( |
||||||
|
"User: %.1f%% Nice: %.1f%% System: %.1f%% Idle: %.1f%% IOwait: %.1f%% IRQ: %.1f%% SoftIRQ: %.1f%% Steal: %.1f%%%n", |
||||||
|
100d * user / totalCpu, 100d * nice / totalCpu, 100d * sys / totalCpu, 100d * idle / totalCpu, |
||||||
|
100d * iowait / totalCpu, 100d * irq / totalCpu, 100d * softirq / totalCpu, 100d * steal / totalCpu)); |
||||||
|
logger.info(String.format("CPU load: %.1f%% (counting ticks)%n", processor.getSystemCpuLoadBetweenTicks() * 100)); |
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
double[] loadAverage = processor.getSystemLoadAverage(3); |
||||||
|
logger.info("CPU load averages:" + (loadAverage[0] < 0 ? " N/A" : String.format(" %.2f", loadAverage[0])) |
||||||
|
+ (loadAverage[1] < 0 ? " N/A" : String.format(" %.2f", loadAverage[1])) |
||||||
|
+ (loadAverage[2] < 0 ? " N/A" : String.format(" %.2f", loadAverage[2]))); |
||||||
|
// per core CPU
|
||||||
|
StringBuilder procCpu = new StringBuilder("CPU load per processor:"); |
||||||
|
double[] load = processor.getProcessorCpuLoadBetweenTicks(); |
||||||
|
for (double avg : load) { |
||||||
|
procCpu.append(String.format(" %.1f%%", avg * 100)); |
||||||
|
} |
||||||
|
logger.info(procCpu.toString()); |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,112 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.queue; |
||||||
|
|
||||||
|
import cn.escheduler.common.Constants; |
||||||
|
import org.junit.Assert; |
||||||
|
import org.junit.Test; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
import java.util.Arrays; |
||||||
|
import java.util.Random; |
||||||
|
|
||||||
|
import static org.junit.Assert.assertEquals; |
||||||
|
|
||||||
|
/** |
||||||
|
* task queue test |
||||||
|
*/ |
||||||
|
public class TaskQueueImplTest { |
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(TaskQueueImplTest.class); |
||||||
|
|
||||||
|
|
||||||
|
@Test |
||||||
|
public void testTaskQueue(){ |
||||||
|
|
||||||
|
ITaskQueue tasksQueue = TaskQueueFactory.getTaskQueueInstance(); |
||||||
|
//clear all data
|
||||||
|
tasksQueue.delete(); |
||||||
|
|
||||||
|
//add
|
||||||
|
tasksQueue.add(Constants.SCHEDULER_TASKS_QUEUE,"1"); |
||||||
|
tasksQueue.add(Constants.SCHEDULER_TASKS_QUEUE,"2"); |
||||||
|
tasksQueue.add(Constants.SCHEDULER_TASKS_QUEUE,"3"); |
||||||
|
tasksQueue.add(Constants.SCHEDULER_TASKS_QUEUE,"4"); |
||||||
|
|
||||||
|
//pop
|
||||||
|
String node1 = tasksQueue.poll(Constants.SCHEDULER_TASKS_QUEUE); |
||||||
|
assertEquals(node1,"1"); |
||||||
|
String node2 = tasksQueue.poll(Constants.SCHEDULER_TASKS_QUEUE); |
||||||
|
assertEquals(node2,"2"); |
||||||
|
|
||||||
|
//sadd
|
||||||
|
String task1 = "1.1.1.1-1-mr"; |
||||||
|
String task2 = "1.1.1.2-2-mr"; |
||||||
|
String task3 = "1.1.1.3-3-mr"; |
||||||
|
String task4 = "1.1.1.4-4-mr"; |
||||||
|
String task5 = "1.1.1.5-5-mr"; |
||||||
|
|
||||||
|
tasksQueue.sadd(Constants.SCHEDULER_TASKS_KILL,task1); |
||||||
|
tasksQueue.sadd(Constants.SCHEDULER_TASKS_KILL,task2); |
||||||
|
tasksQueue.sadd(Constants.SCHEDULER_TASKS_KILL,task3); |
||||||
|
tasksQueue.sadd(Constants.SCHEDULER_TASKS_KILL,task4); |
||||||
|
tasksQueue.sadd(Constants.SCHEDULER_TASKS_KILL,task5); |
||||||
|
tasksQueue.sadd(Constants.SCHEDULER_TASKS_KILL,task5); //repeat task
|
||||||
|
|
||||||
|
Assert.assertEquals(tasksQueue.smembers(Constants.SCHEDULER_TASKS_KILL).size(),5); |
||||||
|
logger.info(Arrays.toString(tasksQueue.smembers(Constants.SCHEDULER_TASKS_KILL).toArray())); |
||||||
|
//srem
|
||||||
|
tasksQueue.srem(Constants.SCHEDULER_TASKS_KILL,task5); |
||||||
|
//smembers
|
||||||
|
Assert.assertEquals(tasksQueue.smembers(Constants.SCHEDULER_TASKS_KILL).size(),4); |
||||||
|
logger.info(Arrays.toString(tasksQueue.smembers(Constants.SCHEDULER_TASKS_KILL).toArray())); |
||||||
|
|
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* test one million data from zookeeper queue |
||||||
|
*/ |
||||||
|
@Test |
||||||
|
public void extremeTest(){ |
||||||
|
ITaskQueue tasksQueue = TaskQueueFactory.getTaskQueueInstance(); |
||||||
|
//clear all data
|
||||||
|
tasksQueue.delete(); |
||||||
|
int total = 30 * 10000; |
||||||
|
|
||||||
|
for(int i = 0; i < total; i++) |
||||||
|
{ |
||||||
|
for(int j = 0; j < total; j++) { |
||||||
|
//${processInstancePriority}_${processInstanceId}_${taskInstancePriority}_${taskId}
|
||||||
|
//format ${processInstancePriority}_${processInstanceId}_${taskInstancePriority}_${taskId}
|
||||||
|
String formatTask = String.format("%s_%d_%s_%d", i, i + 1, j, j == 0 ? 0 : j + new Random().nextInt(100)); |
||||||
|
tasksQueue.add(Constants.SCHEDULER_TASKS_QUEUE, formatTask); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
String node1 = tasksQueue.poll(Constants.SCHEDULER_TASKS_QUEUE); |
||||||
|
assertEquals(node1,"0"); |
||||||
|
|
||||||
|
//clear all data
|
||||||
|
tasksQueue.delete(); |
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
} |
@ -0,0 +1,76 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.shell; |
||||||
|
|
||||||
|
import cn.escheduler.common.thread.ThreadPoolExecutors; |
||||||
|
import org.junit.Test; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
import java.io.IOException; |
||||||
|
import java.util.concurrent.CountDownLatch; |
||||||
|
|
||||||
|
public class ShellExecutorTest { |
||||||
|
private static final Logger logger = LoggerFactory.getLogger(ShellExecutorTest.class); |
||||||
|
|
||||||
|
@Test |
||||||
|
public void execCommand() throws InterruptedException { |
||||||
|
|
||||||
|
ThreadPoolExecutors executors = ThreadPoolExecutors.getInstance(); |
||||||
|
CountDownLatch latch = new CountDownLatch(200); |
||||||
|
|
||||||
|
executors.execute(new Runnable() { |
||||||
|
@Override |
||||||
|
public void run() { |
||||||
|
|
||||||
|
try { |
||||||
|
int i =0; |
||||||
|
while(i++ <= 100){ |
||||||
|
String res = ShellExecutor.execCommand("groups"); |
||||||
|
logger.info("time:" + i + ",thread id:" + Thread.currentThread().getId() + ", result:" + res.substring(0,5)); |
||||||
|
Thread.sleep(100l); |
||||||
|
latch.countDown(); |
||||||
|
} |
||||||
|
|
||||||
|
} catch (IOException | InterruptedException e) { |
||||||
|
e.printStackTrace(); |
||||||
|
} |
||||||
|
} |
||||||
|
}); |
||||||
|
|
||||||
|
executors.execute(new Runnable() { |
||||||
|
@Override |
||||||
|
public void run() { |
||||||
|
|
||||||
|
try { |
||||||
|
int i =0; |
||||||
|
while(i++ <= 100){ |
||||||
|
String res = ShellExecutor.execCommand("whoami"); |
||||||
|
logger.info("time:" + i + ",thread id:" + Thread.currentThread().getId() + ", result2:" + res); |
||||||
|
Thread.sleep(100l); |
||||||
|
latch.countDown(); |
||||||
|
} |
||||||
|
|
||||||
|
} catch (IOException | InterruptedException e) { |
||||||
|
e.printStackTrace(); |
||||||
|
} |
||||||
|
} |
||||||
|
}); |
||||||
|
|
||||||
|
latch.await(); |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,53 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.threadutils; |
||||||
|
|
||||||
|
import cn.escheduler.common.thread.ThreadPoolExecutors; |
||||||
|
import org.junit.Test; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
public class ThreadPoolExecutorsTest { |
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(ThreadPoolExecutors.class); |
||||||
|
|
||||||
|
|
||||||
|
@Test |
||||||
|
public void testThreadPoolExecutors() throws InterruptedException { |
||||||
|
|
||||||
|
Thread2[] threadArr = new Thread2[10]; |
||||||
|
for (int i = 0; i < threadArr.length; i++) { |
||||||
|
|
||||||
|
threadArr[i] = new Thread2(); |
||||||
|
threadArr[i].setDaemon(false); |
||||||
|
threadArr[i].start(); |
||||||
|
} |
||||||
|
|
||||||
|
Thread.currentThread().join(40000l); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
//test thread
|
||||||
|
class Thread2 extends Thread { |
||||||
|
@Override |
||||||
|
public void run() { |
||||||
|
logger.info(String.format("ThreadPoolExecutors instance's hashcode is: %s ",ThreadPoolExecutors.getInstance("a",2).hashCode())); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
} |
@ -0,0 +1,63 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.utils; |
||||||
|
|
||||||
|
import org.junit.Assert; |
||||||
|
import org.junit.Test; |
||||||
|
|
||||||
|
import java.util.ArrayList; |
||||||
|
import java.util.HashSet; |
||||||
|
import java.util.List; |
||||||
|
import java.util.Set; |
||||||
|
|
||||||
|
|
||||||
|
public class CollectionUtilsTest { |
||||||
|
|
||||||
|
@Test |
||||||
|
public void equalLists() { |
||||||
|
List<Integer> a = new ArrayList<Integer>(); |
||||||
|
|
||||||
|
a.add(1); |
||||||
|
a.add(2); |
||||||
|
a.add(3); |
||||||
|
|
||||||
|
List<Integer> b = new ArrayList<Integer>(); |
||||||
|
b.add(3); |
||||||
|
b.add(2); |
||||||
|
b.add(1); |
||||||
|
|
||||||
|
Assert.assertTrue(CollectionUtils.equalLists(a,b)); |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void subtract() { |
||||||
|
Set<Integer> a = new HashSet<Integer>(); |
||||||
|
|
||||||
|
a.add(1); |
||||||
|
a.add(2); |
||||||
|
a.add(3); |
||||||
|
|
||||||
|
Set<Integer> b = new HashSet<Integer>(); |
||||||
|
b.add(0); |
||||||
|
b.add(2); |
||||||
|
b.add(4); |
||||||
|
|
||||||
|
|
||||||
|
Assert.assertArrayEquals(new Integer[]{1,3},CollectionUtils.subtract(a,b).toArray()); |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,61 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.utils; |
||||||
|
|
||||||
|
import org.junit.Test; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
import java.net.InetAddress; |
||||||
|
import java.net.UnknownHostException; |
||||||
|
|
||||||
|
/** |
||||||
|
* configuration test |
||||||
|
*/ |
||||||
|
public class CommonUtilsTest { |
||||||
|
private static final Logger logger = LoggerFactory.getLogger(CommonUtilsTest.class); |
||||||
|
@Test |
||||||
|
public void getHdfsDataBasePath() { |
||||||
|
logger.info(HadoopUtils.getHdfsDataBasePath()); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void getDownloadFilename() { |
||||||
|
logger.info(FileUtils.getDownloadFilename("a.txt")); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void getUploadFilename() { |
||||||
|
logger.info(FileUtils.getUploadFilename("1234", "a.txt")); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void getHdfsDir() { |
||||||
|
logger.info(HadoopUtils.getHdfsDir("1234")); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void test(){ |
||||||
|
InetAddress IP = null; |
||||||
|
try { |
||||||
|
IP = InetAddress.getLocalHost(); |
||||||
|
logger.info(IP.getHostAddress()); |
||||||
|
} catch (UnknownHostException e) { |
||||||
|
e.printStackTrace(); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,57 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
package cn.escheduler.common.utils; |
||||||
|
|
||||||
|
import org.junit.Assert; |
||||||
|
import org.junit.Test; |
||||||
|
|
||||||
|
import java.text.ParseException; |
||||||
|
import java.text.SimpleDateFormat; |
||||||
|
import java.util.Date; |
||||||
|
|
||||||
|
public class DateUtilsTest { |
||||||
|
|
||||||
|
@Test |
||||||
|
public void format2Readable() throws ParseException { |
||||||
|
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); |
||||||
|
|
||||||
|
String start = "2015-12-21 18:00:36"; |
||||||
|
Date startDate = sdf.parse(start); |
||||||
|
|
||||||
|
String end = "2015-12-23 03:23:44"; |
||||||
|
Date endDate = sdf.parse(end); |
||||||
|
|
||||||
|
String readableDate = DateUtils.format2Readable(endDate.getTime() - startDate.getTime()); |
||||||
|
|
||||||
|
Assert.assertEquals("01 09:23:08", readableDate); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
@Test |
||||||
|
public void testWeek(){ |
||||||
|
|
||||||
|
Date curr = DateUtils.stringToDate("2019-02-01 00:00:00"); |
||||||
|
Date monday1 = DateUtils.stringToDate("2019-01-28 00:00:00"); |
||||||
|
Date sunday1 = DateUtils.stringToDate("2019-02-03 00:00:00"); |
||||||
|
Date monday = DateUtils.getMonday(curr); |
||||||
|
Date sunday = DateUtils.getSunday(monday); |
||||||
|
|
||||||
|
Assert.assertEquals(monday, monday1); |
||||||
|
Assert.assertEquals(sunday, sunday1); |
||||||
|
|
||||||
|
} |
||||||
|
} |
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue