From 7a11d39f109008376d5f31678cf1f1a86cc05686 Mon Sep 17 00:00:00 2001 From: "xiaochun.liu" Date: Mon, 24 Feb 2020 20:11:27 +0800 Subject: [PATCH 1/4] docker for e2e --- .github/workflows/ci_e2e.yml | 53 ++++++++++++++++++++++++++++++++++++ dockerfile/README.md | 2 +- dockerfile/hooks/build | 15 ++++++++-- dockerfile/hooks/check | 26 ++++++++++++++++++ dockerfile/startup.sh | 4 ++- 5 files changed, 96 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/ci_e2e.yml create mode 100644 dockerfile/hooks/check diff --git a/.github/workflows/ci_e2e.yml b/.github/workflows/ci_e2e.yml new file mode 100644 index 0000000000..0f867b60b3 --- /dev/null +++ b/.github/workflows/ci_e2e.yml @@ -0,0 +1,53 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +on: ["push", "pull_request"] +env: + DOCKER_DIR: ./docker + LOG_DIR: /tmp/dolphinscheduler + +name: e2e Test + +jobs: + + build: + name: Test + runs-on: ubuntu-latest + steps: + + - uses: actions/checkout@v1 + with: + submodules: true + - uses: actions/cache@v1 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + - name: Build Image + run: export VERSION=latest && sh ./dockerfile/hooks/build + - name: Docker Run + run: docker run -dit -e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test -p 8888:8888 dolphinscheduler all + - name: Check Server Status + run: sh ./dockerfile/hooks/check + - name: Run e2e Test + run: echo "Run e2e Test" + - name: Collect logs + run: | + mkdir -p ${LOG_DIR} + docker logs dolphinscheduler > ${LOG_DIR}/dolphinscheduler.txt + continue-on-error: true diff --git a/dockerfile/README.md b/dockerfile/README.md index fb1500b495..8259ce9a6f 100644 --- a/dockerfile/README.md +++ b/dockerfile/README.md @@ -12,7 +12,7 @@ Official Website: https://dolphinscheduler.apache.org #### You can start a dolphinscheduler instance ``` -$ docker run -d --name dolphinscheduler \ +$ docker run -dit --name dolphinscheduler \ -e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test \ -p 8888:8888 \ dolphinscheduler all diff --git a/dockerfile/hooks/build b/dockerfile/hooks/build index 2394a8cd9c..32d2ee4e38 100644 --- a/dockerfile/hooks/build +++ b/dockerfile/hooks/build @@ -18,11 +18,22 @@ echo "------ dolphinscheduler start - build -------" printenv +if [[ "${VERSION}x" = "x" ]]; then + VERSION=`cat $(pwd)/pom.xml| grep "SNAPSHOT" | awk -F "-SNAPSHOT" '{print $1}' | awk -F ">" '{print $2}'` +fi + +if [[ "${DOCKER_REPO}x" = "x" ]]; then + DOCKER_REPO='dolphinscheduler' +fi + +echo "Version: $VERSION" +echo "Repo: $DOCKER_REPO" + echo -e "Current Directory is $(pwd)\n" # maven package(Project Directory) -echo -e "mvn clean compile package -Prelease" -mvn clean compile package -Prelease +echo -e "mvn -B clean compile package -Prelease -Dmaven.test.skip=true" +mvn -B clean compile package -Prelease -Dmaven.test.skip=true # mv dolphinscheduler-bin.tar.gz file to dockerfile directory echo -e "mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${VERSION}-SNAPSHOT-dolphinscheduler-bin.tar.gz $(pwd)/dockerfile/\n" diff --git a/dockerfile/hooks/check b/dockerfile/hooks/check new file mode 100644 index 0000000000..1a6052cc79 --- /dev/null +++ b/dockerfile/hooks/check @@ -0,0 +1,26 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +echo "------ dolphinscheduler check - server - status -------" +server_num=$(docker top `docker container list | grep startup | awk '{print $1}'`| grep java | grep "dolphinscheduler" | awk -F 'classpath ' '{print $2}' | awk '{print $2}' | sort | uniq -c | wc -l) +if [ $server_num -eq 5 ] +then + echo "Server all start successfully" +else + echo "Server start failed "$server_num + exit 1 +fi diff --git a/dockerfile/startup.sh b/dockerfile/startup.sh index e52384d6b9..610a86f5d3 100644 --- a/dockerfile/startup.sh +++ b/dockerfile/startup.sh @@ -187,4 +187,6 @@ case "$1" in ;; esac -exec tee ${LOGFILE} \ No newline at end of file +echo "tee begin" +exec tee ${LOGFILE} + From 5aebdc8b26ae329216d70f6b5a4c188014eb0ce6 Mon Sep 17 00:00:00 2001 From: "xiaochun.liu" Date: Mon, 24 Feb 2020 20:43:31 +0800 Subject: [PATCH 2/4] docker github action --- .github/workflows/ci_e2e.yml | 8 ++++++-- dockerfile/hooks/build | 4 ---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci_e2e.yml b/.github/workflows/ci_e2e.yml index 0f867b60b3..616f62d049 100644 --- a/.github/workflows/ci_e2e.yml +++ b/.github/workflows/ci_e2e.yml @@ -39,9 +39,13 @@ jobs: restore-keys: | ${{ runner.os }}-maven- - name: Build Image - run: export VERSION=latest && sh ./dockerfile/hooks/build + run: | + export VERSION=`cat $(pwd)/pom.xml| grep "SNAPSHOT" | awk -F "-SNAPSHOT" '{print $1}' | awk -F ">" '{print $2}'` + sh ./dockerfile/hooks/build - name: Docker Run - run: docker run -dit -e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test -p 8888:8888 dolphinscheduler all + run: | + VERSION=`cat $(pwd)/pom.xml| grep "SNAPSHOT" | awk -F "-SNAPSHOT" '{print $1}' | awk -F ">" '{print $2}'` + docker run -dit -e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test -p 8888:8888 dolphinscheduler:$VERSION all - name: Check Server Status run: sh ./dockerfile/hooks/check - name: Run e2e Test diff --git a/dockerfile/hooks/build b/dockerfile/hooks/build index 32d2ee4e38..ee173591af 100644 --- a/dockerfile/hooks/build +++ b/dockerfile/hooks/build @@ -18,10 +18,6 @@ echo "------ dolphinscheduler start - build -------" printenv -if [[ "${VERSION}x" = "x" ]]; then - VERSION=`cat $(pwd)/pom.xml| grep "SNAPSHOT" | awk -F "-SNAPSHOT" '{print $1}' | awk -F ">" '{print $2}'` -fi - if [[ "${DOCKER_REPO}x" = "x" ]]; then DOCKER_REPO='dolphinscheduler' fi From 8a362b5464b3950f5bf222f9db358d9f3956d9f2 Mon Sep 17 00:00:00 2001 From: "xiaochun.liu" Date: Mon, 24 Feb 2020 20:44:44 +0800 Subject: [PATCH 3/4] docker github action --- dockerfile/hooks/check | 1 + 1 file changed, 1 insertion(+) diff --git a/dockerfile/hooks/check b/dockerfile/hooks/check index 1a6052cc79..fbc3f8bd44 100644 --- a/dockerfile/hooks/check +++ b/dockerfile/hooks/check @@ -16,6 +16,7 @@ # limitations under the License. # echo "------ dolphinscheduler check - server - status -------" +sleep 20 server_num=$(docker top `docker container list | grep startup | awk '{print $1}'`| grep java | grep "dolphinscheduler" | awk -F 'classpath ' '{print $2}' | awk '{print $2}' | sort | uniq -c | wc -l) if [ $server_num -eq 5 ] then From e2cc538c6ec022cb13076a1992df6e992a22c1a9 Mon Sep 17 00:00:00 2001 From: "xiaochun.liu" Date: Mon, 24 Feb 2020 20:59:19 +0800 Subject: [PATCH 4/4] docker github action --- dockerfile/hooks/build | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dockerfile/hooks/build b/dockerfile/hooks/build index ee173591af..e4bb5a92b3 100644 --- a/dockerfile/hooks/build +++ b/dockerfile/hooks/build @@ -18,7 +18,8 @@ echo "------ dolphinscheduler start - build -------" printenv -if [[ "${DOCKER_REPO}x" = "x" ]]; then +if [ "${DOCKER_REPO}x" = "x" ] +then DOCKER_REPO='dolphinscheduler' fi