Browse Source
* cluster test * fix init db failed * fix init db failed * fix init db failed * fix init db failed * fix init db failed * add github actions * add github actions * add github actions * add github actions * add github actions * add github actions * add sudo * add github actions * add github actions * add github actions * add github actions * add github actions * add github actions * add github actions * add github actions * add github actions * add github actions * add github actions * add github actions * add github actions * add github actions * add github actions * remove cluster-test workflows * add github actions * add github actions * refactor test to docker mode * refactor test to docker mode * refactor test to docker mode * refactor test to docker mode * remove create schema logic * remove create schema logic * remove create schema logic * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * add github actions * add github actions * add github actions * add cluster test timeout * add cluster start test loop check * add cluster start test loop check * optimize cluster start test loop check3.1.0-release
xiangzihao
3 years ago
committed by
GitHub
16 changed files with 412 additions and 83 deletions
@ -0,0 +1,47 @@
|
||||
# |
||||
# Licensed to the Apache Software Foundation (ASF) under one or more |
||||
# contributor license agreements. See the NOTICE file distributed with |
||||
# this work for additional information regarding copyright ownership. |
||||
# The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
# (the "License"); you may not use this file except in compliance with |
||||
# the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
# |
||||
|
||||
FROM openjdk:8-jre-slim-buster |
||||
|
||||
RUN apt update ; \ |
||||
apt install -y curl wget default-mysql-client sudo openssh-server netcat-traditional ; |
||||
|
||||
#COPY ./dolphinscheduler-dist/target/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz /root/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz |
||||
COPY ./apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz /root/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz |
||||
RUN tar -zxvf /root/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz -C ~ |
||||
|
||||
ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-dev-SNAPSHOT-bin |
||||
|
||||
#Setting install.sh |
||||
COPY .github/workflows/cluster-test/mysql/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh |
||||
|
||||
#Setting dolphinscheduler_env.sh |
||||
COPY .github/workflows/cluster-test/mysql/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh |
||||
|
||||
#Download mysql jar |
||||
ENV MYSQL_URL "https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar" |
||||
ENV MYSQL_DRIVER "mysql-connector-java-8.0.16.jar" |
||||
RUN wget -O $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $MYSQL_URL ; \ |
||||
cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/api-server/libs/$MYSQL_DRIVER ; \ |
||||
cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/master-server/libs/$MYSQL_DRIVER ; \ |
||||
cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/worker-server/libs/$MYSQL_DRIVER ; \ |
||||
cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/tools/libs/$MYSQL_DRIVER |
||||
|
||||
#Deploy |
||||
COPY .github/workflows/cluster-test/mysql/deploy.sh /root/deploy.sh |
||||
|
||||
CMD [ "/bin/bash", "/root/deploy.sh" ] |
||||
@ -0,0 +1,44 @@
|
||||
#!/bin/bash |
||||
# |
||||
# Licensed to the Apache Software Foundation (ASF) under one or more |
||||
# contributor license agreements. See the NOTICE file distributed with |
||||
# this work for additional information regarding copyright ownership. |
||||
# The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
# (the "License"); you may not use this file except in compliance with |
||||
# the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
# |
||||
set -euox pipefail |
||||
|
||||
|
||||
USER=root |
||||
DOLPHINSCHEDULER_HOME=/root/apache-dolphinscheduler-dev-SNAPSHOT-bin |
||||
|
||||
#Create database |
||||
mysql -hmysql -P3306 -uroot -p123456 -e "CREATE DATABASE IF NOT EXISTS dolphinscheduler DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;" |
||||
|
||||
#Sudo |
||||
sed -i '$a'$USER' ALL=(ALL) NOPASSWD: NOPASSWD: ALL' /etc/sudoers |
||||
sed -i 's/Defaults requirett/#Defaults requirett/g' /etc/sudoers |
||||
|
||||
#SSH |
||||
ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa |
||||
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys |
||||
chmod 600 ~/.ssh/authorized_keys |
||||
service ssh start |
||||
|
||||
#Init schema |
||||
/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/upgrade-schema.sh |
||||
|
||||
#Start Cluster |
||||
/bin/bash $DOLPHINSCHEDULER_HOME/bin/start-all.sh |
||||
|
||||
#Keep running |
||||
tail -f /dev/null |
||||
@ -0,0 +1,65 @@
|
||||
# |
||||
# Licensed to the Apache Software Foundation (ASF) under one or more |
||||
# contributor license agreements. See the NOTICE file distributed with |
||||
# this work for additional information regarding copyright ownership. |
||||
# The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
# (the "License"); you may not use this file except in compliance with |
||||
# the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
# |
||||
|
||||
version: "3" |
||||
|
||||
services: |
||||
mysql: |
||||
container_name: mysql |
||||
image: mysql:5.7.36 |
||||
command: --default-authentication-plugin=mysql_native_password |
||||
restart: always |
||||
environment: |
||||
MYSQL_ROOT_PASSWORD: 123456 |
||||
ports: |
||||
- "3306:3306" |
||||
healthcheck: |
||||
test: mysqladmin ping -h 127.0.0.1 -u root --password=$$MYSQL_ROOT_PASSWORD |
||||
interval: 5s |
||||
timeout: 60s |
||||
retries: 120 |
||||
|
||||
zoo1: |
||||
image: zookeeper:3.8.0 |
||||
restart: always |
||||
hostname: zoo1 |
||||
ports: |
||||
- "2181:2181" |
||||
environment: |
||||
ZOO_MY_ID: 1 |
||||
ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181 |
||||
|
||||
zoo2: |
||||
image: zookeeper:3.8.0 |
||||
restart: always |
||||
hostname: zoo2 |
||||
ports: |
||||
- "2182:2181" |
||||
environment: |
||||
ZOO_MY_ID: 2 |
||||
ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181 |
||||
|
||||
zoo3: |
||||
image: zookeeper:3.8.0 |
||||
restart: always |
||||
hostname: zoo3 |
||||
ports: |
||||
- "2183:2181" |
||||
environment: |
||||
ZOO_MY_ID: 3 |
||||
ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181 |
||||
|
||||
@ -0,0 +1,48 @@
|
||||
# |
||||
# Licensed to the Apache Software Foundation (ASF) under one or more |
||||
# contributor license agreements. See the NOTICE file distributed with |
||||
# this work for additional information regarding copyright ownership. |
||||
# The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
# (the "License"); you may not use this file except in compliance with |
||||
# the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
# |
||||
|
||||
# JAVA_HOME, will use it to start DolphinScheduler server |
||||
export JAVA_HOME=${JAVA_HOME:-/usr/local/openjdk-8} |
||||
|
||||
# Database related configuration, set database type, username and password |
||||
export DATABASE=${DATABASE:-mysql} |
||||
export SPRING_PROFILES_ACTIVE=${DATABASE} |
||||
export SPRING_DATASOURCE_DRIVER_CLASS_NAME=com.mysql.cj.jdbc.Driver |
||||
export SPRING_DATASOURCE_URL="jdbc:mysql://mysql:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8&useSSL=false" |
||||
export SPRING_DATASOURCE_USERNAME=root |
||||
export SPRING_DATASOURCE_PASSWORD=123456 |
||||
|
||||
# DolphinScheduler server related configuration |
||||
export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none} |
||||
export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC} |
||||
export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10} |
||||
|
||||
# Registry center configuration, determines the type and link of the registry center |
||||
export REGISTRY_TYPE=${REGISTRY_TYPE:-zookeeper} |
||||
export REGISTRY_ZOOKEEPER_CONNECT_STRING=${REGISTRY_ZOOKEEPER_CONNECT_STRING:-zoo1:2181,zoo2:2182,zoo3:2183} |
||||
|
||||
# Tasks related configurations, need to change the configuration if you use the related tasks. |
||||
export HADOOP_HOME=${HADOOP_HOME:-/opt/soft/hadoop} |
||||
export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/opt/soft/hadoop/etc/hadoop} |
||||
export SPARK_HOME1=${SPARK_HOME1:-/opt/soft/spark1} |
||||
export SPARK_HOME2=${SPARK_HOME2:-/opt/soft/spark2} |
||||
export PYTHON_HOME=${PYTHON_HOME:-/opt/soft/python} |
||||
export HIVE_HOME=${HIVE_HOME:-/opt/soft/hive} |
||||
export FLINK_HOME=${FLINK_HOME:-/opt/soft/flink} |
||||
export DATAX_HOME=${DATAX_HOME:-/opt/soft/datax} |
||||
|
||||
export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH |
||||
@ -0,0 +1,61 @@
|
||||
# |
||||
# Licensed to the Apache Software Foundation (ASF) under one or more |
||||
# contributor license agreements. See the NOTICE file distributed with |
||||
# this work for additional information regarding copyright ownership. |
||||
# The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
# (the "License"); you may not use this file except in compliance with |
||||
# the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
# |
||||
|
||||
# --------------------------------------------------------- |
||||
# INSTALL MACHINE |
||||
# --------------------------------------------------------- |
||||
# A comma separated list of machine hostname or IP would be installed DolphinScheduler, |
||||
# including master, worker, api, alert. If you want to deploy in pseudo-distributed |
||||
# mode, just write a pseudo-distributed hostname |
||||
# Example for hostnames: ips="ds1,ds2,ds3,ds4,ds5", Example for IPs: ips="192.168.8.1,192.168.8.2,192.168.8.3,192.168.8.4,192.168.8.5" |
||||
ips=${ips:-"localhost"} |
||||
|
||||
# Port of SSH protocol, default value is 22. For now we only support same port in all `ips` machine |
||||
# modify it if you use different ssh port |
||||
sshPort=${sshPort:-"22"} |
||||
|
||||
# A comma separated list of machine hostname or IP would be installed Master server, it |
||||
# must be a subset of configuration `ips`. |
||||
# Example for hostnames: masters="ds1,ds2", Example for IPs: masters="192.168.8.1,192.168.8.2" |
||||
masters=${masters:-"localhost"} |
||||
|
||||
# A comma separated list of machine <hostname>:<workerGroup> or <IP>:<workerGroup>.All hostname or IP must be a |
||||
# subset of configuration `ips`, And workerGroup have default value as `default`, but we recommend you declare behind the hosts |
||||
# Example for hostnames: workers="ds1:default,ds2:default,ds3:default", Example for IPs: workers="192.168.8.1:default,192.168.8.2:default,192.168.8.3:default" |
||||
workers=${workers:-"localhost:default"} |
||||
|
||||
# A comma separated list of machine hostname or IP would be installed Alert server, it |
||||
# must be a subset of configuration `ips`. |
||||
# Example for hostname: alertServer="ds3", Example for IP: alertServer="192.168.8.3" |
||||
alertServer=${alertServer:-"localhost"} |
||||
|
||||
# A comma separated list of machine hostname or IP would be installed API server, it |
||||
# must be a subset of configuration `ips`. |
||||
# Example for hostname: apiServers="ds1", Example for IP: apiServers="192.168.8.1" |
||||
apiServers=${apiServers:-"localhost"} |
||||
|
||||
# The directory to install DolphinScheduler for all machine we config above. It will automatically be created by `install.sh` script if not exists. |
||||
# Do not set this configuration same as the current path (pwd) |
||||
installPath=${installPath:-"/root/apache-dolphinscheduler-dev-SNAPSHOT-bin"} |
||||
|
||||
# The user to deploy DolphinScheduler for all machine we config above. For now user must create by yourself before running `install.sh` |
||||
# script. The user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled than the root directory needs |
||||
# to be created by this user |
||||
deployUser=${deployUser:-"dolphinscheduler"} |
||||
|
||||
# The root of zookeeper, for now DolphinScheduler default registry server is zookeeper. |
||||
zkRoot=${zkRoot:-"/dolphinscheduler"} |
||||
@ -0,0 +1,81 @@
|
||||
#!/bin/bash |
||||
# |
||||
# Licensed to the Apache Software Foundation (ASF) under one or more |
||||
# contributor license agreements. See the NOTICE file distributed with |
||||
# this work for additional information regarding copyright ownership. |
||||
# The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
# (the "License"); you may not use this file except in compliance with |
||||
# the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
# |
||||
set -x |
||||
|
||||
|
||||
API_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:12345/dolphinscheduler/actuator/health" |
||||
MASTER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:5679/actuator/health" |
||||
WORKER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:1235/actuator/health" |
||||
|
||||
#Cluster start health check |
||||
TIMEOUT=120 |
||||
START_HEALTHCHECK_EXITCODE=0 |
||||
|
||||
for ((i=1; i<=TIMEOUT; i++)) |
||||
do |
||||
MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND") |
||||
WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND") |
||||
API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND") |
||||
if [[ $MASTER_HTTP_STATUS -eq 200 && $WORKER_HTTP_STATUS -eq 200 && $API_HTTP_STATUS -eq 200 ]];then |
||||
START_HEALTHCHECK_EXITCODE=0 |
||||
else |
||||
START_HEALTHCHECK_EXITCODE=2 |
||||
fi |
||||
|
||||
if [[ $START_HEALTHCHECK_EXITCODE -eq 0 ]];then |
||||
echo "cluster start health check success" |
||||
break |
||||
fi |
||||
|
||||
if [[ $i -eq $TIMEOUT ]];then |
||||
docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-dev-SNAPSHOT-bin/master-server/logs/dolphinscheduler-master.log" |
||||
echo "cluster start health check failed" |
||||
exit $START_HEALTHCHECK_EXITCODE |
||||
fi |
||||
|
||||
sleep 1 |
||||
done |
||||
|
||||
#Stop Cluster |
||||
docker exec -u root ds bash -c "/root/apache-dolphinscheduler-dev-SNAPSHOT-bin/bin/stop-all.sh" |
||||
|
||||
#Cluster stop health check |
||||
sleep 5 |
||||
MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND") |
||||
if [[ $MASTER_HTTP_STATUS -ne 200 ]];then |
||||
echo "master stop health check success" |
||||
else |
||||
echo "master stop health check failed" |
||||
exit 3 |
||||
fi |
||||
|
||||
WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND") |
||||
if [[ $WORKER_HTTP_STATUS -ne 200 ]];then |
||||
echo "worker stop health check success" |
||||
else |
||||
echo "worker stop health check failed" |
||||
exit 3 |
||||
fi |
||||
|
||||
API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND") |
||||
if [[ $API_HTTP_STATUS -ne 200 ]];then |
||||
echo "api stop health check success" |
||||
else |
||||
echo "api stop health check failed" |
||||
exit 3 |
||||
fi |
||||
@ -1,55 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.tools.datasource; |
||||
|
||||
import org.slf4j.Logger; |
||||
import org.slf4j.LoggerFactory; |
||||
import org.springframework.boot.CommandLineRunner; |
||||
import org.springframework.boot.SpringApplication; |
||||
import org.springframework.boot.autoconfigure.SpringBootApplication; |
||||
import org.springframework.stereotype.Component; |
||||
|
||||
@SpringBootApplication |
||||
public class CreateDolphinScheduler { |
||||
public static void main(String[] args) { |
||||
SpringApplication.run(CreateDolphinScheduler.class, args); |
||||
} |
||||
|
||||
@Component |
||||
static class CreateRunner implements CommandLineRunner { |
||||
private static final Logger logger = LoggerFactory.getLogger(CreateRunner.class); |
||||
|
||||
private final DolphinSchedulerManager dolphinSchedulerManager; |
||||
|
||||
CreateRunner(DolphinSchedulerManager dolphinSchedulerManager) { |
||||
this.dolphinSchedulerManager = dolphinSchedulerManager; |
||||
} |
||||
|
||||
@Override |
||||
public void run(String... args) throws Exception { |
||||
if (dolphinSchedulerManager.schemaIsInitialized()) { |
||||
dolphinSchedulerManager.upgradeDolphinScheduler(); |
||||
logger.info("upgrade DolphinScheduler finished"); |
||||
} else { |
||||
dolphinSchedulerManager.initDolphinScheduler(); |
||||
logger.info("init DolphinScheduler finished"); |
||||
} |
||||
logger.info("create DolphinScheduler success"); |
||||
} |
||||
} |
||||
} |
||||
Loading…
Reference in new issue