From baf654cb47d60a753dd6372698ebb9230fca4859 Mon Sep 17 00:00:00 2001 From: xiangzihao <460888207@qq.com> Date: Sat, 14 May 2022 12:30:57 +0800 Subject: [PATCH] [Feature-9474] [CI] Add cluster test script verify on shell script (#9997) * cluster test * fix init db failed * fix init db failed * fix init db failed * fix init db failed * fix init db failed * add github actions * add github actions * add github actions * add github actions * add github actions * add github actions * add sudo * add github actions * add github actions * add github actions * add github actions * add github actions * add github actions * add github actions * add github actions * add github actions * add github actions * add github actions * add github actions * add github actions * add github actions * add github actions * remove cluster-test workflows * add github actions * add github actions * refactor test to docker mode * refactor test to docker mode * refactor test to docker mode * refactor test to docker mode * remove create schema logic * remove create schema logic * remove create schema logic * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * fix runnint cluster test error * add github actions * add github actions * add github actions * add cluster test timeout * add cluster start test loop check * add cluster start test loop check * optimize cluster start test loop check --- .github/workflows/backend.yml | 31 ++++++- .../workflows/cluster-test/mysql/Dockerfile | 47 +++++++++++ .../workflows/cluster-test/mysql/deploy.sh | 44 ++++++++++ .../mysql/docker-compose-base.yaml | 65 +++++++++++++++ .../mysql/docker-compose-cluster.yaml | 21 ++--- .../mysql/dolphinscheduler_env.sh | 48 +++++++++++ .../cluster-test/mysql/install_env.sh | 61 ++++++++++++++ .../cluster-test/mysql/running_test.sh | 81 +++++++++++++++++++ .../en/guide/installation/pseudo-cluster.md | 6 +- .../zh/guide/installation/pseudo-cluster.md | 6 +- .../src/main/bin/upgrade-schema.sh | 1 + .../datasource/CreateDolphinScheduler.java | 55 ------------- .../datasource/InitDolphinScheduler.java | 2 + .../datasource/UpgradeDolphinScheduler.java | 11 ++- script/start-all.sh | 8 +- script/stop-all.sh | 8 +- 16 files changed, 412 insertions(+), 83 deletions(-) create mode 100644 .github/workflows/cluster-test/mysql/Dockerfile create mode 100644 .github/workflows/cluster-test/mysql/deploy.sh create mode 100644 .github/workflows/cluster-test/mysql/docker-compose-base.yaml rename dolphinscheduler-tools/src/main/bin/create-schema.sh => .github/workflows/cluster-test/mysql/docker-compose-cluster.yaml (57%) mode change 100755 => 100644 create mode 100755 .github/workflows/cluster-test/mysql/dolphinscheduler_env.sh create mode 100644 .github/workflows/cluster-test/mysql/install_env.sh create mode 100644 .github/workflows/cluster-test/mysql/running_test.sh delete mode 100644 dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/CreateDolphinScheduler.java diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index fa7495aa40..502ae4c62f 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -78,11 +78,38 @@ jobs: -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 - name: Check dependency license run: tools/dependencies/check-LICENSE.sh + - uses: actions/upload-artifact@v2 + name: Upload Binary Package + with: + name: binary-package + path: ./dolphinscheduler-dist/target/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz + retention-days: 1 + cluster-test: + name: Cluster-Test + needs: build + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - uses: actions/download-artifact@v2 + name: Download Binary Package + with: + name: binary-package + path: ./ + - name: Running cluster test(Mysql) + run: | + docker-compose -f .github/workflows/cluster-test/mysql/docker-compose-base.yaml up -d + docker build -t jdk8:ds_cluster -f .github/workflows/cluster-test/mysql/Dockerfile . + docker-compose -f .github/workflows/cluster-test/mysql/docker-compose-cluster.yaml up -d + /bin/bash .github/workflows/cluster-test/mysql/running_test.sh + docker rm -f $(docker ps -aq) result: name: Build runs-on: ubuntu-latest timeout-minutes: 30 - needs: [ build, paths-filter ] + needs: [ build, paths-filter, cluster-test ] if: always() steps: - name: Status @@ -91,7 +118,7 @@ jobs: echo "Skip Build!" exit 0 fi - if [[ ${{ needs.build.result }} != 'success' ]]; then + if [[ ${{ needs.build.result }} != 'success' || ${{ needs.cluster-test.result }} != 'success' ]]; then echo "Build Failed!" exit -1 fi diff --git a/.github/workflows/cluster-test/mysql/Dockerfile b/.github/workflows/cluster-test/mysql/Dockerfile new file mode 100644 index 0000000000..e117b86dc6 --- /dev/null +++ b/.github/workflows/cluster-test/mysql/Dockerfile @@ -0,0 +1,47 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM openjdk:8-jre-slim-buster + +RUN apt update ; \ + apt install -y curl wget default-mysql-client sudo openssh-server netcat-traditional ; + +#COPY ./dolphinscheduler-dist/target/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz /root/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz +COPY ./apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz /root/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz +RUN tar -zxvf /root/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz -C ~ + +ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-dev-SNAPSHOT-bin + +#Setting install.sh +COPY .github/workflows/cluster-test/mysql/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh + +#Setting dolphinscheduler_env.sh +COPY .github/workflows/cluster-test/mysql/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh + +#Download mysql jar +ENV MYSQL_URL "https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar" +ENV MYSQL_DRIVER "mysql-connector-java-8.0.16.jar" +RUN wget -O $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $MYSQL_URL ; \ +cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/api-server/libs/$MYSQL_DRIVER ; \ +cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/master-server/libs/$MYSQL_DRIVER ; \ +cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/worker-server/libs/$MYSQL_DRIVER ; \ +cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/tools/libs/$MYSQL_DRIVER + +#Deploy +COPY .github/workflows/cluster-test/mysql/deploy.sh /root/deploy.sh + +CMD [ "/bin/bash", "/root/deploy.sh" ] diff --git a/.github/workflows/cluster-test/mysql/deploy.sh b/.github/workflows/cluster-test/mysql/deploy.sh new file mode 100644 index 0000000000..75b23d08a2 --- /dev/null +++ b/.github/workflows/cluster-test/mysql/deploy.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -euox pipefail + + +USER=root +DOLPHINSCHEDULER_HOME=/root/apache-dolphinscheduler-dev-SNAPSHOT-bin + +#Create database +mysql -hmysql -P3306 -uroot -p123456 -e "CREATE DATABASE IF NOT EXISTS dolphinscheduler DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;" + +#Sudo +sed -i '$a'$USER' ALL=(ALL) NOPASSWD: NOPASSWD: ALL' /etc/sudoers +sed -i 's/Defaults requirett/#Defaults requirett/g' /etc/sudoers + +#SSH +ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa +cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys +chmod 600 ~/.ssh/authorized_keys +service ssh start + +#Init schema +/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/upgrade-schema.sh + +#Start Cluster +/bin/bash $DOLPHINSCHEDULER_HOME/bin/start-all.sh + +#Keep running +tail -f /dev/null diff --git a/.github/workflows/cluster-test/mysql/docker-compose-base.yaml b/.github/workflows/cluster-test/mysql/docker-compose-base.yaml new file mode 100644 index 0000000000..251b72672d --- /dev/null +++ b/.github/workflows/cluster-test/mysql/docker-compose-base.yaml @@ -0,0 +1,65 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: "3" + +services: + mysql: + container_name: mysql + image: mysql:5.7.36 + command: --default-authentication-plugin=mysql_native_password + restart: always + environment: + MYSQL_ROOT_PASSWORD: 123456 + ports: + - "3306:3306" + healthcheck: + test: mysqladmin ping -h 127.0.0.1 -u root --password=$$MYSQL_ROOT_PASSWORD + interval: 5s + timeout: 60s + retries: 120 + + zoo1: + image: zookeeper:3.8.0 + restart: always + hostname: zoo1 + ports: + - "2181:2181" + environment: + ZOO_MY_ID: 1 + ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181 + + zoo2: + image: zookeeper:3.8.0 + restart: always + hostname: zoo2 + ports: + - "2182:2181" + environment: + ZOO_MY_ID: 2 + ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181 + + zoo3: + image: zookeeper:3.8.0 + restart: always + hostname: zoo3 + ports: + - "2183:2181" + environment: + ZOO_MY_ID: 3 + ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181 + diff --git a/dolphinscheduler-tools/src/main/bin/create-schema.sh b/.github/workflows/cluster-test/mysql/docker-compose-cluster.yaml old mode 100755 new mode 100644 similarity index 57% rename from dolphinscheduler-tools/src/main/bin/create-schema.sh rename to .github/workflows/cluster-test/mysql/docker-compose-cluster.yaml index 29cc5d7711..1b2d9e91ab --- a/dolphinscheduler-tools/src/main/bin/create-schema.sh +++ b/.github/workflows/cluster-test/mysql/docker-compose-cluster.yaml @@ -1,4 +1,3 @@ -#!/bin/bash # # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with @@ -16,13 +15,15 @@ # limitations under the License. # -BIN_DIR=$(dirname $0) -DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/../..; pwd)} +version: "3" -source "$DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh" - -JAVA_OPTS=${JAVA_OPTS:-"-server -Duser.timezone=${SPRING_JACKSON_TIME_ZONE} -Xms1g -Xmx1g -Xmn512m -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"} - -java $JAVA_OPTS \ - -cp "$DOLPHINSCHEDULER_HOME/tools/conf":"$DOLPHINSCHEDULER_HOME/tools/libs/*":"$DOLPHINSCHEDULER_HOME/tools/sql" \ - org.apache.dolphinscheduler.tools.datasource.CreateDolphinScheduler +services: + ds: + container_name: ds + image: jdk8:ds_cluster + restart: always + ports: + - "12345:12345" + - "5679:5679" + - "1235:1235" + - "50053:50053" diff --git a/.github/workflows/cluster-test/mysql/dolphinscheduler_env.sh b/.github/workflows/cluster-test/mysql/dolphinscheduler_env.sh new file mode 100755 index 0000000000..07d5a8dbba --- /dev/null +++ b/.github/workflows/cluster-test/mysql/dolphinscheduler_env.sh @@ -0,0 +1,48 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# JAVA_HOME, will use it to start DolphinScheduler server +export JAVA_HOME=${JAVA_HOME:-/usr/local/openjdk-8} + +# Database related configuration, set database type, username and password +export DATABASE=${DATABASE:-mysql} +export SPRING_PROFILES_ACTIVE=${DATABASE} +export SPRING_DATASOURCE_DRIVER_CLASS_NAME=com.mysql.cj.jdbc.Driver +export SPRING_DATASOURCE_URL="jdbc:mysql://mysql:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8&useSSL=false" +export SPRING_DATASOURCE_USERNAME=root +export SPRING_DATASOURCE_PASSWORD=123456 + +# DolphinScheduler server related configuration +export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none} +export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC} +export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10} + +# Registry center configuration, determines the type and link of the registry center +export REGISTRY_TYPE=${REGISTRY_TYPE:-zookeeper} +export REGISTRY_ZOOKEEPER_CONNECT_STRING=${REGISTRY_ZOOKEEPER_CONNECT_STRING:-zoo1:2181,zoo2:2182,zoo3:2183} + +# Tasks related configurations, need to change the configuration if you use the related tasks. +export HADOOP_HOME=${HADOOP_HOME:-/opt/soft/hadoop} +export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/opt/soft/hadoop/etc/hadoop} +export SPARK_HOME1=${SPARK_HOME1:-/opt/soft/spark1} +export SPARK_HOME2=${SPARK_HOME2:-/opt/soft/spark2} +export PYTHON_HOME=${PYTHON_HOME:-/opt/soft/python} +export HIVE_HOME=${HIVE_HOME:-/opt/soft/hive} +export FLINK_HOME=${FLINK_HOME:-/opt/soft/flink} +export DATAX_HOME=${DATAX_HOME:-/opt/soft/datax} + +export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH diff --git a/.github/workflows/cluster-test/mysql/install_env.sh b/.github/workflows/cluster-test/mysql/install_env.sh new file mode 100644 index 0000000000..b7c9797e20 --- /dev/null +++ b/.github/workflows/cluster-test/mysql/install_env.sh @@ -0,0 +1,61 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# --------------------------------------------------------- +# INSTALL MACHINE +# --------------------------------------------------------- +# A comma separated list of machine hostname or IP would be installed DolphinScheduler, +# including master, worker, api, alert. If you want to deploy in pseudo-distributed +# mode, just write a pseudo-distributed hostname +# Example for hostnames: ips="ds1,ds2,ds3,ds4,ds5", Example for IPs: ips="192.168.8.1,192.168.8.2,192.168.8.3,192.168.8.4,192.168.8.5" +ips=${ips:-"localhost"} + +# Port of SSH protocol, default value is 22. For now we only support same port in all `ips` machine +# modify it if you use different ssh port +sshPort=${sshPort:-"22"} + +# A comma separated list of machine hostname or IP would be installed Master server, it +# must be a subset of configuration `ips`. +# Example for hostnames: masters="ds1,ds2", Example for IPs: masters="192.168.8.1,192.168.8.2" +masters=${masters:-"localhost"} + +# A comma separated list of machine : or :.All hostname or IP must be a +# subset of configuration `ips`, And workerGroup have default value as `default`, but we recommend you declare behind the hosts +# Example for hostnames: workers="ds1:default,ds2:default,ds3:default", Example for IPs: workers="192.168.8.1:default,192.168.8.2:default,192.168.8.3:default" +workers=${workers:-"localhost:default"} + +# A comma separated list of machine hostname or IP would be installed Alert server, it +# must be a subset of configuration `ips`. +# Example for hostname: alertServer="ds3", Example for IP: alertServer="192.168.8.3" +alertServer=${alertServer:-"localhost"} + +# A comma separated list of machine hostname or IP would be installed API server, it +# must be a subset of configuration `ips`. +# Example for hostname: apiServers="ds1", Example for IP: apiServers="192.168.8.1" +apiServers=${apiServers:-"localhost"} + +# The directory to install DolphinScheduler for all machine we config above. It will automatically be created by `install.sh` script if not exists. +# Do not set this configuration same as the current path (pwd) +installPath=${installPath:-"/root/apache-dolphinscheduler-dev-SNAPSHOT-bin"} + +# The user to deploy DolphinScheduler for all machine we config above. For now user must create by yourself before running `install.sh` +# script. The user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled than the root directory needs +# to be created by this user +deployUser=${deployUser:-"dolphinscheduler"} + +# The root of zookeeper, for now DolphinScheduler default registry server is zookeeper. +zkRoot=${zkRoot:-"/dolphinscheduler"} diff --git a/.github/workflows/cluster-test/mysql/running_test.sh b/.github/workflows/cluster-test/mysql/running_test.sh new file mode 100644 index 0000000000..c812d05402 --- /dev/null +++ b/.github/workflows/cluster-test/mysql/running_test.sh @@ -0,0 +1,81 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -x + + +API_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:12345/dolphinscheduler/actuator/health" +MASTER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:5679/actuator/health" +WORKER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:1235/actuator/health" + +#Cluster start health check +TIMEOUT=120 +START_HEALTHCHECK_EXITCODE=0 + +for ((i=1; i<=TIMEOUT; i++)) +do + MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND") + WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND") + API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND") + if [[ $MASTER_HTTP_STATUS -eq 200 && $WORKER_HTTP_STATUS -eq 200 && $API_HTTP_STATUS -eq 200 ]];then + START_HEALTHCHECK_EXITCODE=0 + else + START_HEALTHCHECK_EXITCODE=2 + fi + + if [[ $START_HEALTHCHECK_EXITCODE -eq 0 ]];then + echo "cluster start health check success" + break + fi + + if [[ $i -eq $TIMEOUT ]];then + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-dev-SNAPSHOT-bin/master-server/logs/dolphinscheduler-master.log" + echo "cluster start health check failed" + exit $START_HEALTHCHECK_EXITCODE + fi + + sleep 1 +done + +#Stop Cluster +docker exec -u root ds bash -c "/root/apache-dolphinscheduler-dev-SNAPSHOT-bin/bin/stop-all.sh" + +#Cluster stop health check +sleep 5 +MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND") +if [[ $MASTER_HTTP_STATUS -ne 200 ]];then + echo "master stop health check success" +else + echo "master stop health check failed" + exit 3 +fi + +WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND") +if [[ $WORKER_HTTP_STATUS -ne 200 ]];then + echo "worker stop health check success" +else + echo "worker stop health check failed" + exit 3 +fi + +API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND") +if [[ $API_HTTP_STATUS -ne 200 ]];then + echo "api stop health check success" +else + echo "api stop health check failed" + exit 3 +fi diff --git a/docs/docs/en/guide/installation/pseudo-cluster.md b/docs/docs/en/guide/installation/pseudo-cluster.md index b96679a63b..d6ff451175 100644 --- a/docs/docs/en/guide/installation/pseudo-cluster.md +++ b/docs/docs/en/guide/installation/pseudo-cluster.md @@ -115,7 +115,7 @@ export JAVA_HOME=${JAVA_HOME:-/opt/soft/java} export DATABASE=${DATABASE:-postgresql} export SPRING_PROFILES_ACTIVE=${DATABASE} export SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.postgresql.Driver -export SPRING_DATASOURCE_URL=jdbc:postgresql://127.0.0.1:5432/dolphinscheduler +export SPRING_DATASOURCE_URL="jdbc:postgresql://127.0.0.1:5432/dolphinscheduler" export SPRING_DATASOURCE_USERNAME={user} export SPRING_DATASOURCE_PASSWORD={password} @@ -180,7 +180,7 @@ Then, modify `./bin/env/dolphinscheduler_env.sh` to use mysql, change {user} and export DATABASE=${DATABASE:-mysql} export SPRING_PROFILES_ACTIVE=${DATABASE} export SPRING_DATASOURCE_DRIVER_CLASS_NAME=com.mysql.cj.jdbc.Driver -export SPRING_DATASOURCE_URL=jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 +export SPRING_DATASOURCE_URL="jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8&useSSL=false" export SPRING_DATASOURCE_USERNAME={user} export SPRING_DATASOURCE_PASSWORD={password} ``` @@ -188,7 +188,7 @@ export SPRING_DATASOURCE_PASSWORD={password} After the above steps done you would create a new database for DolphinScheduler, then run the Shell script to init database: ```shell -sh tools/bin/create-schema.sh +sh tools/bin/upgrade-schema.sh ``` ## Start DolphinScheduler diff --git a/docs/docs/zh/guide/installation/pseudo-cluster.md b/docs/docs/zh/guide/installation/pseudo-cluster.md index 2e0570cce2..107d27b2f1 100644 --- a/docs/docs/zh/guide/installation/pseudo-cluster.md +++ b/docs/docs/zh/guide/installation/pseudo-cluster.md @@ -112,7 +112,7 @@ export JAVA_HOME=${JAVA_HOME:-/opt/soft/java} export DATABASE=${DATABASE:-postgresql} export SPRING_PROFILES_ACTIVE=${DATABASE} export SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.postgresql.Driver -export SPRING_DATASOURCE_URL=jdbc:postgresql://127.0.0.1:5432/dolphinscheduler +export SPRING_DATASOURCE_URL="jdbc:postgresql://127.0.0.1:5432/dolphinscheduler" export SPRING_DATASOURCE_USERNAME={user} export SPRING_DATASOURCE_PASSWORD={password} @@ -177,7 +177,7 @@ mysql> FLUSH PRIVILEGES; export DATABASE=${DATABASE:-mysql} export SPRING_PROFILES_ACTIVE=${DATABASE} export SPRING_DATASOURCE_DRIVER_CLASS_NAME=com.mysql.cj.jdbc.Driver -export SPRING_DATASOURCE_URL=jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 +export SPRING_DATASOURCE_URL="jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8&useSSL=false" export SPRING_DATASOURCE_USERNAME={user} export SPRING_DATASOURCE_PASSWORD={password} ``` @@ -185,7 +185,7 @@ export SPRING_DATASOURCE_PASSWORD={password} 完成上述步骤后,您已经为 DolphinScheduler 创建一个新数据库,现在你可以通过快速的 Shell 脚本来初始化数据库 ```shell -sh tools/bin/create-schema.sh +sh tools/bin/upgrade-schema.sh ``` ## 启动 DolphinScheduler diff --git a/dolphinscheduler-tools/src/main/bin/upgrade-schema.sh b/dolphinscheduler-tools/src/main/bin/upgrade-schema.sh index d11dc6a2c1..b3df345871 100755 --- a/dolphinscheduler-tools/src/main/bin/upgrade-schema.sh +++ b/dolphinscheduler-tools/src/main/bin/upgrade-schema.sh @@ -25,4 +25,5 @@ JAVA_OPTS=${JAVA_OPTS:-"-server -Duser.timezone=${SPRING_JACKSON_TIME_ZONE} -Xms java $JAVA_OPTS \ -cp "$DOLPHINSCHEDULER_HOME/tools/conf":"$DOLPHINSCHEDULER_HOME/tools/libs/*":"$DOLPHINSCHEDULER_HOME/tools/sql" \ + -Dspring.profiles.active=upgrade \ org.apache.dolphinscheduler.tools.datasource.UpgradeDolphinScheduler diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/CreateDolphinScheduler.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/CreateDolphinScheduler.java deleted file mode 100644 index 204dc5c2cc..0000000000 --- a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/CreateDolphinScheduler.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.tools.datasource; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.boot.CommandLineRunner; -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.stereotype.Component; - -@SpringBootApplication -public class CreateDolphinScheduler { - public static void main(String[] args) { - SpringApplication.run(CreateDolphinScheduler.class, args); - } - - @Component - static class CreateRunner implements CommandLineRunner { - private static final Logger logger = LoggerFactory.getLogger(CreateRunner.class); - - private final DolphinSchedulerManager dolphinSchedulerManager; - - CreateRunner(DolphinSchedulerManager dolphinSchedulerManager) { - this.dolphinSchedulerManager = dolphinSchedulerManager; - } - - @Override - public void run(String... args) throws Exception { - if (dolphinSchedulerManager.schemaIsInitialized()) { - dolphinSchedulerManager.upgradeDolphinScheduler(); - logger.info("upgrade DolphinScheduler finished"); - } else { - dolphinSchedulerManager.initDolphinScheduler(); - logger.info("init DolphinScheduler finished"); - } - logger.info("create DolphinScheduler success"); - } - } -} diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/InitDolphinScheduler.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/InitDolphinScheduler.java index 18eb310654..6ccd73decd 100644 --- a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/InitDolphinScheduler.java +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/InitDolphinScheduler.java @@ -22,6 +22,7 @@ import org.slf4j.LoggerFactory; import org.springframework.boot.CommandLineRunner; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Component; @SpringBootApplication @@ -31,6 +32,7 @@ public class InitDolphinScheduler { } @Component + @Profile("init") static class InitRunner implements CommandLineRunner { private static final Logger logger = LoggerFactory.getLogger(InitRunner.class); diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/UpgradeDolphinScheduler.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/UpgradeDolphinScheduler.java index d09acc6692..22e1338431 100644 --- a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/UpgradeDolphinScheduler.java +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/UpgradeDolphinScheduler.java @@ -22,6 +22,7 @@ import org.slf4j.LoggerFactory; import org.springframework.boot.CommandLineRunner; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Component; @SpringBootApplication @@ -31,6 +32,7 @@ public class UpgradeDolphinScheduler { } @Component + @Profile("upgrade") static class UpgradeRunner implements CommandLineRunner { private static final Logger logger = LoggerFactory.getLogger(UpgradeRunner.class); @@ -42,8 +44,13 @@ public class UpgradeDolphinScheduler { @Override public void run(String... args) throws Exception { - dolphinSchedulerManager.upgradeDolphinScheduler(); - logger.info("upgrade DolphinScheduler success"); + if (dolphinSchedulerManager.schemaIsInitialized()) { + dolphinSchedulerManager.upgradeDolphinScheduler(); + logger.info("upgrade DolphinScheduler finished"); + } else { + dolphinSchedulerManager.initDolphinScheduler(); + logger.info("init DolphinScheduler finished"); + } } } } diff --git a/script/start-all.sh b/script/start-all.sh index 35936eea0a..0126a1381b 100755 --- a/script/start-all.sh +++ b/script/start-all.sh @@ -33,7 +33,7 @@ mastersHost=(${masters//,/ }) for master in ${mastersHost[@]} do echo "$master master server is starting" - ssh -p $sshPort $master "cd $installPath/; bash bin/dolphinscheduler-daemon.sh start master-server;" + ssh -o StrictHostKeyChecking=no -p $sshPort $master "cd $installPath/; bash bin/dolphinscheduler-daemon.sh start master-server;" done @@ -41,16 +41,16 @@ for worker in ${workerNames[@]} do echo "$worker worker server is starting" - ssh -p $sshPort $worker "cd $installPath/; bash bin/dolphinscheduler-daemon.sh start worker-server;" + ssh -o StrictHostKeyChecking=no -p $sshPort $worker "cd $installPath/; bash bin/dolphinscheduler-daemon.sh start worker-server;" done -ssh -p $sshPort $alertServer "cd $installPath/; bash bin/dolphinscheduler-daemon.sh start alert-server;" +ssh -o StrictHostKeyChecking=no -p $sshPort $alertServer "cd $installPath/; bash bin/dolphinscheduler-daemon.sh start alert-server;" apiServersHost=(${apiServers//,/ }) for apiServer in ${apiServersHost[@]} do echo "$apiServer api server is starting" - ssh -p $sshPort $apiServer "cd $installPath/; bash bin/dolphinscheduler-daemon.sh start api-server;" + ssh -o StrictHostKeyChecking=no -p $sshPort $apiServer "cd $installPath/; bash bin/dolphinscheduler-daemon.sh start api-server;" done # query server status diff --git a/script/stop-all.sh b/script/stop-all.sh index ee3a09cd41..e8eb6da5d1 100755 --- a/script/stop-all.sh +++ b/script/stop-all.sh @@ -33,21 +33,21 @@ mastersHost=(${masters//,/ }) for master in ${mastersHost[@]} do echo "$master master server is stopping" - ssh -p $sshPort $master "cd $installPath/; bash bin/dolphinscheduler-daemon.sh stop master-server;" + ssh -o StrictHostKeyChecking=no -p $sshPort $master "cd $installPath/; bash bin/dolphinscheduler-daemon.sh stop master-server;" done for worker in ${workerNames[@]} do echo "$worker worker server is stopping" - ssh -p $sshPort $worker "cd $installPath/; bash bin/dolphinscheduler-daemon.sh stop worker-server;" + ssh -o StrictHostKeyChecking=no -p $sshPort $worker "cd $installPath/; bash bin/dolphinscheduler-daemon.sh stop worker-server;" done -ssh -p $sshPort $alertServer "cd $installPath/; bash bin/dolphinscheduler-daemon.sh stop alert-server;" +ssh -o StrictHostKeyChecking=no -p $sshPort $alertServer "cd $installPath/; bash bin/dolphinscheduler-daemon.sh stop alert-server;" apiServersHost=(${apiServers//,/ }) for apiServer in ${apiServersHost[@]} do echo "$apiServer api server is stopping" - ssh -p $sshPort $apiServer "cd $installPath/; bash bin/dolphinscheduler-daemon.sh stop api-server;" + ssh -o StrictHostKeyChecking=no -p $sshPort $apiServer "cd $installPath/; bash bin/dolphinscheduler-daemon.sh stop api-server;" done