Browse Source

add embedded database h2 and update embedded zookeeper (#2082)

add embedded database h2 and update embedded zookeeper
pull/2/head
dailidong 5 years ago committed by GitHub
parent
commit
4230d5a737
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 19
      dolphinscheduler-dao/pom.xml
  2. 4
      dolphinscheduler-dao/src/main/resources/application.properties
  3. 100
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/zk/StandaloneZKServerForTest.java
  4. 5
      dolphinscheduler-service/pom.xml
  5. 87
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZKServer.java
  6. 4
      dolphinscheduler-service/src/main/resources/quartz.properties
  7. 41
      dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/ZKServerTest.java
  8. 1
      dolphinscheduler-service/src/test/java/queue/BaseTaskQueueTest.java
  9. 14
      pom.xml
  10. BIN
      sql/h2.mv.db

19
dolphinscheduler-dao/pom.xml

@ -44,6 +44,12 @@
<groupId>com.baomidou</groupId> <groupId>com.baomidou</groupId>
<artifactId>mybatis-plus-boot-starter</artifactId> <artifactId>mybatis-plus-boot-starter</artifactId>
<version>${mybatis-plus.version}</version> <version>${mybatis-plus.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-to-slf4j</artifactId>
</exclusion>
</exclusions>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.postgresql</groupId> <groupId>org.postgresql</groupId>
@ -71,6 +77,14 @@
<artifactId>log4j-api</artifactId> <artifactId>log4j-api</artifactId>
<groupId>org.apache.logging.log4j</groupId> <groupId>org.apache.logging.log4j</groupId>
</exclusion> </exclusion>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-tomcat</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-to-slf4j</artifactId>
</exclusion>
</exclusions> </exclusions>
</dependency> </dependency>
@ -78,7 +92,10 @@
<groupId>mysql</groupId> <groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId> <artifactId>mysql-connector-java</artifactId>
</dependency> </dependency>
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
</dependency>
<dependency> <dependency>
<groupId>com.alibaba</groupId> <groupId>com.alibaba</groupId>
<artifactId>druid</artifactId> <artifactId>druid</artifactId>

4
dolphinscheduler-dao/src/main/resources/application.properties

@ -23,6 +23,10 @@ spring.datasource.url=jdbc:postgresql://localhost:5432/dolphinscheduler
# mysql # mysql
#spring.datasource.driver-class-name=com.mysql.jdbc.Driver #spring.datasource.driver-class-name=com.mysql.jdbc.Driver
#spring.datasource.url=jdbc:mysql://192.168.xx.xx:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 #spring.datasource.url=jdbc:mysql://192.168.xx.xx:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8
# h2
#spring.datasource.driver-class-name=org.h2.Driver
#spring.datasource.url=jdbc:h2:file:../sql/h2;AUTO_SERVER=TRUE
spring.datasource.username=test spring.datasource.username=test
spring.datasource.password=test spring.datasource.password=test

100
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/zk/StandaloneZKServerForTest.java

@ -1,100 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.zk;
import org.apache.dolphinscheduler.common.thread.ThreadPoolExecutors;
import org.apache.zookeeper.server.ServerConfig;
import org.apache.zookeeper.server.ZooKeeperServerMain;
import org.apache.zookeeper.server.quorum.QuorumPeerConfig;
import org.junit.Before;
import org.junit.Ignore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.util.Properties;
/**
* just for test
*/
@Ignore
public class StandaloneZKServerForTest {
private static final Logger logger = LoggerFactory.getLogger(StandaloneZKServerForTest.class);
private static volatile ZooKeeperServerMain zkServer = null;
@Before
public void before() {
logger.info("standalone zookeeper server for test service start ");
ThreadPoolExecutors.getInstance().execute(new Runnable() {
@Override
public void run() {
//delete zk data dir ?
File zkFile = new File(System.getProperty("java.io.tmpdir"), "zookeeper");
startStandaloneServer("2000", zkFile.getAbsolutePath(), "2181", "10", "5");
}
});
}
/**
* start zk server
* @param tickTime zookeeper ticktime
* @param dataDir zookeeper data dir
* @param clientPort zookeeper client port
* @param initLimit zookeeper init limit
* @param syncLimit zookeeper sync limit
*/
private void startStandaloneServer(String tickTime, String dataDir, String clientPort, String initLimit, String syncLimit) {
Properties props = new Properties();
props.setProperty("tickTime", tickTime);
props.setProperty("dataDir", dataDir);
props.setProperty("clientPort", clientPort);
props.setProperty("initLimit", initLimit);
props.setProperty("syncLimit", syncLimit);
QuorumPeerConfig quorumConfig = new QuorumPeerConfig();
try {
quorumConfig.parseProperties(props);
if(zkServer == null ){
synchronized (StandaloneZKServerForTest.class){
if(zkServer == null ){
zkServer = new ZooKeeperServerMain();
final ServerConfig config = new ServerConfig();
config.readFrom(quorumConfig);
zkServer.runFromConfig(config);
}
}
}
} catch (Exception e) {
logger.error("start standalone server fail!", e);
}
}
}

5
dolphinscheduler-service/pom.xml

@ -48,6 +48,11 @@
</exclusions> </exclusions>
</dependency> </dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
</dependency>
<dependency> <dependency>
<groupId>org.quartz-scheduler</groupId> <groupId>org.quartz-scheduler</groupId>
<artifactId>quartz-jobs</artifactId> <artifactId>quartz-jobs</artifactId>

87
dolphinscheduler-service/src/test/java/queue/ZKServer.java → dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZKServer.java

@ -14,11 +14,9 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package queue; package org.apache.dolphinscheduler.service.zk;
import org.apache.curator.framework.CuratorFramework; import org.apache.zookeeper.server.ZooKeeperServer;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.retry.ExponentialBackoffRetry;
import org.apache.zookeeper.server.ZooKeeperServerMain; import org.apache.zookeeper.server.ZooKeeperServerMain;
import org.apache.zookeeper.server.quorum.QuorumPeerConfig; import org.apache.zookeeper.server.quorum.QuorumPeerConfig;
import org.slf4j.Logger; import org.slf4j.Logger;
@ -26,27 +24,45 @@ import org.slf4j.LoggerFactory;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicBoolean;
/** /**
* just for test * just speed experience version
* embedded zookeeper service
*/ */
public class ZKServer { public class ZKServer {
private static final Logger logger = LoggerFactory.getLogger(ZKServer.class); private static final Logger logger = LoggerFactory.getLogger(ZKServer.class);
private static volatile PublicZooKeeperServerMain zkServer = null; private static volatile PublicZooKeeperServerMain zkServer = null;
public static final int DEFAULT_ZK_TEST_PORT = 2181; public static final int DEFAULT_ZK_TEST_PORT = 2181;
public static final String DEFAULT_ZK_STR = "localhost:" + DEFAULT_ZK_TEST_PORT;
private static String dataDir = null; private static String dataDir = null;
private static final AtomicBoolean isStarted = new AtomicBoolean(false); private static final AtomicBoolean isStarted = new AtomicBoolean(false);
public static void main(String[] args) {
if(!isStarted()){
ZKServer.start();
/**
* register hooks, which are called before the process exits
*/
Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
@Override
public void run() {
stop();
}
}));
}else{
logger.info("zk server aleady started");
}
}
/**
* start service
*/
public static void start() { public static void start() {
try { try {
startLocalZkServer(DEFAULT_ZK_TEST_PORT); startLocalZkServer(DEFAULT_ZK_TEST_PORT);
@ -79,7 +95,8 @@ public class ZKServer {
* @param port The port to listen on * @param port The port to listen on
*/ */
public static void startLocalZkServer(final int port) { public static void startLocalZkServer(final int port) {
startLocalZkServer(port, org.apache.commons.io.FileUtils.getTempDirectoryPath() + File.separator + "test-" + System.currentTimeMillis());
startLocalZkServer(port, System.getProperty("user.dir") +"/zookeeper_data", ZooKeeperServer.DEFAULT_TICK_TIME,"20");
} }
/** /**
@ -87,48 +104,28 @@ public class ZKServer {
* *
* @param port The port to listen on * @param port The port to listen on
* @param dataDirPath The path for the Zk data directory * @param dataDirPath The path for the Zk data directory
* @param tickTime zk tick time
* @param maxClientCnxns zk max client connections
*/ */
private static synchronized void startLocalZkServer(final int port, final String dataDirPath) { private static synchronized void startLocalZkServer(final int port, final String dataDirPath,final int tickTime,String maxClientCnxns) {
if (zkServer != null) { if (zkServer != null) {
throw new RuntimeException("Zookeeper server is already started!"); throw new RuntimeException("Zookeeper server is already started!");
} }
try { zkServer = new PublicZooKeeperServerMain();
zkServer = new PublicZooKeeperServerMain(); logger.info("Zookeeper data path : {} ", dataDirPath);
logger.info("Zookeeper data path : {} ", dataDirPath); dataDir = dataDirPath;
dataDir = dataDirPath; final String[] args = new String[]{Integer.toString(port), dataDirPath, Integer.toString(tickTime), maxClientCnxns};
final String[] args = new String[]{Integer.toString(port), dataDirPath};
Thread init = new Thread(new Runnable() {
@Override
public void run() {
try {
zkServer.initializeAndRun(args);
} catch (QuorumPeerConfig.ConfigException e) {
logger.warn("Caught exception while starting ZK", e);
} catch (IOException e) {
logger.warn("Caught exception while starting ZK", e);
}
}
}, "init-zk-thread");
init.start();
} catch (Exception e) {
logger.warn("Caught exception while starting ZK", e);
throw new RuntimeException(e);
}
CuratorFramework zkClient = CuratorFrameworkFactory.builder()
.connectString(DEFAULT_ZK_STR)
.retryPolicy(new ExponentialBackoffRetry(10,100))
.sessionTimeoutMs(1000 * 30)
.connectionTimeoutMs(1000 * 30)
.build();
try { try {
zkClient.blockUntilConnected(10, TimeUnit.SECONDS); logger.info("Zookeeper server started ");
zkClient.close(); isStarted.compareAndSet(false, true);
} catch (InterruptedException ignore) {
zkServer.initializeAndRun(args);
} catch (QuorumPeerConfig.ConfigException e) {
logger.warn("Caught exception while starting ZK", e);
} catch (IOException e) {
logger.warn("Caught exception while starting ZK", e);
} }
isStarted.compareAndSet(false, true);
logger.info("zk server started");
} }
/** /**

4
dolphinscheduler-service/src/main/resources/quartz.properties

@ -26,6 +26,10 @@ org.quartz.dataSource.myDs.URL = jdbc:postgresql://localhost:5432/dolphinschedul
# mysql # mysql
#org.quartz.dataSource.myDs.driver = com.mysql.jdbc.Driver #org.quartz.dataSource.myDs.driver = com.mysql.jdbc.Driver
#org.quartz.dataSource.myDs.URL = jdbc:mysql://localhost:3306/dolphinscheduler?characterEncoding=utf8 #org.quartz.dataSource.myDs.URL = jdbc:mysql://localhost:3306/dolphinscheduler?characterEncoding=utf8
#h2
#org.quartz.dataSource.myDs.driver=org.h2.Driver
#org.quartz.dataSource.myDs.URL=jdbc:h2:file:/Users/stone/work/myworkspace/incubator-dolphinscheduler/h2;AUTO_SERVER=TRUE
org.quartz.dataSource.myDs.user = test org.quartz.dataSource.myDs.user = test
org.quartz.dataSource.myDs.password = test org.quartz.dataSource.myDs.password = test

41
dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/ZKServerTest.java

@ -0,0 +1,41 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.service.zk;
import org.junit.Ignore;
import org.junit.Test;
import static org.junit.Assert.*;
@Ignore
public class ZKServerTest {
@Test
public void start() {
//ZKServer is a process, can't unit test
}
@Test
public void isStarted() {
}
@Test
public void stop() {
ZKServer.stop();
}
}

1
dolphinscheduler-service/src/test/java/queue/BaseTaskQueueTest.java

@ -18,6 +18,7 @@ package queue;
import org.apache.dolphinscheduler.service.queue.ITaskQueue; import org.apache.dolphinscheduler.service.queue.ITaskQueue;
import org.apache.dolphinscheduler.service.queue.TaskQueueFactory; import org.apache.dolphinscheduler.service.queue.TaskQueueFactory;
import org.apache.dolphinscheduler.service.zk.ZKServer;
import org.junit.*; import org.junit.*;
/** /**

14
pom.xml

@ -72,7 +72,7 @@
<cron.utils.version>5.0.5</cron.utils.version> <cron.utils.version>5.0.5</cron.utils.version>
<fastjson.version>1.2.61</fastjson.version> <fastjson.version>1.2.61</fastjson.version>
<druid.version>1.1.14</druid.version> <druid.version>1.1.14</druid.version>
<h2.version>1.3.163</h2.version> <h2.version>1.4.200</h2.version>
<commons.codec.version>1.6</commons.codec.version> <commons.codec.version>1.6</commons.codec.version>
<commons.logging.version>1.1.1</commons.logging.version> <commons.logging.version>1.1.1</commons.logging.version>
<httpclient.version>4.4.1</httpclient.version> <httpclient.version>4.4.1</httpclient.version>
@ -200,13 +200,6 @@
<version>${spring.version}</version> <version>${spring.version}</version>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
<version>${h2.version}</version>
<scope>test</scope>
</dependency>
<dependency> <dependency>
<groupId>org.apache.dolphinscheduler</groupId> <groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-server</artifactId> <artifactId>dolphinscheduler-server</artifactId>
@ -349,6 +342,11 @@
<version>${mysql.connector.version}</version> <version>${mysql.connector.version}</version>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
<version>${h2.version}</version>
</dependency>
<dependency> <dependency>
<groupId>org.slf4j</groupId> <groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId> <artifactId>slf4j-api</artifactId>

BIN
sql/h2.mv.db

Binary file not shown.
Loading…
Cancel
Save