github
/
DolphinScheduler
mirror of https://gitee.com/dolphinscheduler/DolphinScheduler.git
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

fix zookeeper health check and merge volumes into logs volume (#2674) 

* fix zookeeper health check and merge volumes into logs volume

1. Fix zookeeper health check
2. Merge volumes into logs volume
3. Modify `dockerfile/hooks/build` script

* Modify ci_e2e and docker image registry

* Modify ci_e2e

* Modify ci_e2e

* Modify ci_e2e and move ./dockerfile/hooks/check to ./docker/docker-swarm/
pull/3/MERGE
liwenhe1993 5 years ago committed by GitHub
parent
83ede1037e
commit
3c014c5a57
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 48 additions and 53 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 11
      .github/workflows/ci_e2e.yml
  2. 2
      docker/docker-swarm/check
  3. 41
      docker/docker-swarm/docker-compose.yml
  4. 27
      docker/docker-swarm/docker-stack.yml
  5. 6
      docker/kubernetes/dolphinscheduler/values.yaml
  6. 4
      dockerfile/Dockerfile
  7. 10
      dockerfile/hooks/build

11
.github/workflows/ci_e2e.yml
Unescape View File

@ -44,15 +44,14 @@ jobs:
${{ runner.os }}-maven- ${{ runner.os }}-maven-
- name: Build Image - name: Build Image
run: | run: |
export VERSION=`cat $(pwd)/pom.xml| grep "SNAPSHOT</version>" | awk -F "-SNAPSHOT" '{print $1}' | awk -F ">" '{print $2}'`
sh ./dockerfile/hooks/build sh ./dockerfile/hooks/build
- name: Docker Run - name: Docker Run
run: | run: |
VERSION=`cat $(pwd)/pom.xml| grep "SNAPSHOT</version>" | awk -F "-SNAPSHOT" '{print $1}' | awk -F ">" '{print $2}'` export VERSION=$(cat $(pwd)/pom.xml | grep '<version>' -m 1 | awk '{print $1}' | sed 's/<version>//' | sed 's/<\/version>//')
mkdir -p /tmp/logs sed -i "s/apache\/dolphinscheduler:latest/apache\/dolphinscheduler:${VERSION}/g" $(pwd)/docker/docker-swarm/docker-compose.yml
docker run -dit -e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test -v /tmp/logs:/opt/dolphinscheduler/logs -p 8888:8888 dolphinscheduler:$VERSION all docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml up -d
- name: Check Server Status - name: Check Server Status
run: sh ./dockerfile/hooks/check run: sh $(pwd)/docker/docker-swarm/check
- name: Prepare e2e env - name: Prepare e2e env
run: | run: |
sudo apt-get install -y libxss1 libappindicator1 libindicator7 xvfb unzip libgbm1 sudo apt-get install -y libxss1 libappindicator1 libindicator7 xvfb unzip libgbm1
@ -70,6 +69,6 @@ jobs:
uses: actions/upload-artifact@v1 uses: actions/upload-artifact@v1
with: with:
name: dslogs name: dslogs
path: /tmp/logs path: /var/lib/docker/volumes/docker-swarm_dolphinscheduler-logs/_data

2
dockerfile/hooks/check → docker/docker-swarm/check
Unescape View File

@ -17,7 +17,7 @@
# #
echo "------ dolphinscheduler check - server - status -------" echo "------ dolphinscheduler check - server - status -------"
sleep 60 sleep 60
server_num=$(docker top `docker container list | grep '/sbin/tini' | awk '{print $1}'`| grep java | grep "dolphinscheduler" | awk -F 'classpath ' '{print $2}' | awk '{print $2}' | sort | uniq -c | wc -l) server_num=$(docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml top | grep java | grep "dolphinscheduler" | awk -F 'classpath ' '{print $2}' | awk '{print $2}' | sort | uniq -c | wc -l)
if [ $server_num -eq 5 ] if [ $server_num -eq 5 ]
then then
echo "Server all start successfully" echo "Server all start successfully"

41
docker/docker-swarm/docker-compose.yml
Unescape View File

@ -41,13 +41,14 @@ services:
environment: environment:
TZ: Asia/Shanghai TZ: Asia/Shanghai
ALLOW_ANONYMOUS_LOGIN: "yes" ALLOW_ANONYMOUS_LOGIN: "yes"
ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons
volumes: volumes:
- dolphinscheduler-zookeeper:/bitnami/zookeeper - dolphinscheduler-zookeeper:/bitnami/zookeeper
networks: networks:
- dolphinscheduler - dolphinscheduler
dolphinscheduler-api: dolphinscheduler-api:
image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev image: apache/dolphinscheduler:latest
container_name: dolphinscheduler-api container_name: dolphinscheduler-api
command: ["api-server"] command: ["api-server"]
ports: ports:
@ -70,12 +71,12 @@ services:
- dolphinscheduler-postgresql - dolphinscheduler-postgresql
- dolphinscheduler-zookeeper - dolphinscheduler-zookeeper
volumes: volumes:
- dolphinscheduler-api:/opt/dolphinscheduler/logs - dolphinscheduler-logs:/opt/dolphinscheduler/logs
networks: networks:
- dolphinscheduler - dolphinscheduler
dolphinscheduler-frontend: dolphinscheduler-frontend:
image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev image: apache/dolphinscheduler:latest
container_name: dolphinscheduler-frontend container_name: dolphinscheduler-frontend
command: ["frontend"] command: ["frontend"]
ports: ports:
@ -93,12 +94,12 @@ services:
depends_on: depends_on:
- dolphinscheduler-api - dolphinscheduler-api
volumes: volumes:
- dolphinscheduler-frontend:/var/log/nginx - dolphinscheduler-logs:/var/log/nginx
networks: networks:
- dolphinscheduler - dolphinscheduler
dolphinscheduler-alert: dolphinscheduler-alert:
image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev image: apache/dolphinscheduler:latest
container_name: dolphinscheduler-alert container_name: dolphinscheduler-alert
command: ["alert-server"] command: ["alert-server"]
environment: environment:
@ -130,18 +131,18 @@ services:
start_period: 30s start_period: 30s
depends_on: depends_on:
- dolphinscheduler-postgresql - dolphinscheduler-postgresql
volumes: volumes:
- dolphinscheduler-alert:/opt/dolphinscheduler/logs - dolphinscheduler-logs:/opt/dolphinscheduler/logs
networks: networks:
- dolphinscheduler - dolphinscheduler
dolphinscheduler-master: dolphinscheduler-master:
image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev image: apache/dolphinscheduler:latest
container_name: dolphinscheduler-master container_name: dolphinscheduler-master
command: ["master-server"] command: ["master-server"]
ports: ports:
- 5678:5678 - 5678:5678
environment: environment:
TZ: Asia/Shanghai TZ: Asia/Shanghai
MASTER_EXEC_THREADS: "100" MASTER_EXEC_THREADS: "100"
MASTER_EXEC_TASK_NUM: "20" MASTER_EXEC_TASK_NUM: "20"
@ -162,16 +163,16 @@ services:
timeout: 5s timeout: 5s
retries: 3 retries: 3
start_period: 30s start_period: 30s
depends_on: depends_on:
- dolphinscheduler-postgresql - dolphinscheduler-postgresql
- dolphinscheduler-zookeeper - dolphinscheduler-zookeeper
volumes: volumes:
- dolphinscheduler-master:/opt/dolphinscheduler/logs - dolphinscheduler-logs:/opt/dolphinscheduler/logs
networks: networks:
- dolphinscheduler - dolphinscheduler
dolphinscheduler-worker: dolphinscheduler-worker:
image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev image: apache/dolphinscheduler:latest
container_name: dolphinscheduler-worker container_name: dolphinscheduler-worker
command: ["worker-server"] command: ["worker-server"]
ports: ports:
@ -209,7 +210,7 @@ services:
source: dolphinscheduler-worker-data source: dolphinscheduler-worker-data
target: /tmp/dolphinscheduler target: /tmp/dolphinscheduler
- type: volume - type: volume
source: dolphinscheduler-worker-logs source: dolphinscheduler-logs
target: /opt/dolphinscheduler/logs target: /opt/dolphinscheduler/logs
networks: networks:
- dolphinscheduler - dolphinscheduler
@ -221,12 +222,8 @@ networks:
volumes: volumes:
dolphinscheduler-postgresql: dolphinscheduler-postgresql:
dolphinscheduler-zookeeper: dolphinscheduler-zookeeper:
dolphinscheduler-api:
dolphinscheduler-frontend:
dolphinscheduler-alert:
dolphinscheduler-master:
dolphinscheduler-worker-data: dolphinscheduler-worker-data:
dolphinscheduler-worker-logs: dolphinscheduler-logs:
configs: configs:
dolphinscheduler-worker-task-env: dolphinscheduler-worker-task-env:

27
docker/docker-swarm/docker-stack.yml
Unescape View File

@ -42,6 +42,7 @@ services:
environment: environment:
TZ: Asia/Shanghai TZ: Asia/Shanghai
ALLOW_ANONYMOUS_LOGIN: "yes" ALLOW_ANONYMOUS_LOGIN: "yes"
ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons
volumes: volumes:
- dolphinscheduler-zookeeper:/bitnami/zookeeper - dolphinscheduler-zookeeper:/bitnami/zookeeper
networks: networks:
@ -51,7 +52,7 @@ services:
replicas: 1 replicas: 1
dolphinscheduler-api: dolphinscheduler-api:
image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev image: apache/dolphinscheduler:latest
command: ["api-server"] command: ["api-server"]
ports: ports:
- 12345:12345 - 12345:12345
@ -70,7 +71,7 @@ services:
retries: 3 retries: 3
start_period: 30s start_period: 30s
volumes: volumes:
- dolphinscheduler-api:/opt/dolphinscheduler/logs - dolphinscheduler-logs:/opt/dolphinscheduler/logs
networks: networks:
- dolphinscheduler - dolphinscheduler
deploy: deploy:
@ -78,7 +79,7 @@ services:
replicas: 1 replicas: 1
dolphinscheduler-frontend: dolphinscheduler-frontend:
image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev image: apache/dolphinscheduler:latest
command: ["frontend"] command: ["frontend"]
ports: ports:
- 8888:8888 - 8888:8888
@ -93,7 +94,7 @@ services:
retries: 3 retries: 3
start_period: 30s start_period: 30s
volumes: volumes:
- dolphinscheduler-frontend:/var/log/nginx - dolphinscheduler-logs:/var/log/nginx
networks: networks:
- dolphinscheduler - dolphinscheduler
deploy: deploy:
@ -101,7 +102,7 @@ services:
replicas: 1 replicas: 1
dolphinscheduler-alert: dolphinscheduler-alert:
image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev image: apache/dolphinscheduler:latest
command: ["alert-server"] command: ["alert-server"]
environment: environment:
TZ: Asia/Shanghai TZ: Asia/Shanghai
@ -131,7 +132,7 @@ services:
retries: 3 retries: 3
start_period: 30s start_period: 30s
volumes: volumes:
- dolphinscheduler-alert:/opt/dolphinscheduler/logs - dolphinscheduler-logs:/opt/dolphinscheduler/logs
networks: networks:
- dolphinscheduler - dolphinscheduler
deploy: deploy:
@ -139,7 +140,7 @@ services:
replicas: 1 replicas: 1
dolphinscheduler-master: dolphinscheduler-master:
image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev image: apache/dolphinscheduler:latest
command: ["master-server"] command: ["master-server"]
ports: ports:
- 5678:5678 - 5678:5678
@ -165,7 +166,7 @@ services:
retries: 3 retries: 3
start_period: 30s start_period: 30s
volumes: volumes:
- dolphinscheduler-master:/opt/dolphinscheduler/logs - dolphinscheduler-logs:/opt/dolphinscheduler/logs
networks: networks:
- dolphinscheduler - dolphinscheduler
deploy: deploy:
@ -173,7 +174,7 @@ services:
replicas: 1 replicas: 1
dolphinscheduler-worker: dolphinscheduler-worker:
image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev image: apache/dolphinscheduler:latest
command: ["worker-server"] command: ["worker-server"]
ports: ports:
- 1234:1234 - 1234:1234
@ -201,7 +202,7 @@ services:
start_period: 30s start_period: 30s
volumes: volumes:
- dolphinscheduler-worker-data:/tmp/dolphinscheduler - dolphinscheduler-worker-data:/tmp/dolphinscheduler
- dolphinscheduler-worker-logs:/opt/dolphinscheduler/logs - dolphinscheduler-logs:/opt/dolphinscheduler/logs
configs: configs:
- source: dolphinscheduler-worker-task-env - source: dolphinscheduler-worker-task-env
target: /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh target: /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh
@ -218,12 +219,8 @@ networks:
volumes: volumes:
dolphinscheduler-postgresql: dolphinscheduler-postgresql:
dolphinscheduler-zookeeper: dolphinscheduler-zookeeper:
dolphinscheduler-api:
dolphinscheduler-frontend:
dolphinscheduler-alert:
dolphinscheduler-master:
dolphinscheduler-worker-data: dolphinscheduler-worker-data:
dolphinscheduler-worker-logs: dolphinscheduler-logs:
configs: configs:
dolphinscheduler-worker-task-env: dolphinscheduler-worker-task-env:

6
docker/kubernetes/dolphinscheduler/values.yaml
Unescape View File

@ -25,9 +25,9 @@ fullnameOverride: ""
timezone: "Asia/Shanghai" timezone: "Asia/Shanghai"
image: image:
registry: "docker.io" registry: "apache"
repository: "dolphinscheduler" repository: "dolphinscheduler"
tag: "1.3.0" tag: "latest"
pullPolicy: "IfNotPresent" pullPolicy: "IfNotPresent"
imagePullSecrets: [] imagePullSecrets: []
@ -56,6 +56,8 @@ externalDatabase:
zookeeper: zookeeper:
enabled: true enabled: true
taskQueue: "zookeeper" taskQueue: "zookeeper"
config:
ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons
service: service:
port: "2181" port: "2181"
persistence: persistence:

4
dockerfile/Dockerfile
Unescape View File

@ -53,8 +53,8 @@ ENV PATH $ZK_HOME/bin:$PATH
RUN apk add postgresql postgresql-contrib RUN apk add postgresql postgresql-contrib
#5. add dolphinscheduler #5. add dolphinscheduler
ADD ./apache-dolphinscheduler-incubating-${VERSION}-SNAPSHOT-dolphinscheduler-bin.tar.gz /opt/ ADD ./apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz /opt/
RUN mv /opt/apache-dolphinscheduler-incubating-${VERSION}-SNAPSHOT-dolphinscheduler-bin/ /opt/dolphinscheduler/ RUN mv /opt/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin/ /opt/dolphinscheduler/
ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler
#6. modify nginx #6. modify nginx

10
dockerfile/hooks/build
Unescape View File

@ -24,13 +24,13 @@ printenv
if [ -z "${VERSION}" ] if [ -z "${VERSION}" ]
then then
echo "set default environment variable [VERSION]" echo "set default environment variable [VERSION]"
VERSION=$(cat $(pwd)/sql/soft_version) export VERSION=$(cat $(pwd)/pom.xml | grep '<version>' -m 1 | awk '{print $1}' | sed 's/<version>//' | sed 's/<\/version>//')
fi fi
if [ "${DOCKER_REPO}x" = "x" ] if [ "${DOCKER_REPO}x" = "x" ]
then then
echo "set default environment variable [DOCKER_REPO]" echo "set default environment variable [DOCKER_REPO]"
DOCKER_REPO='dolphinscheduler' export DOCKER_REPO='apache/dolphinscheduler'
fi fi
echo "Version: $VERSION" echo "Version: $VERSION"
@ -43,11 +43,11 @@ echo -e "mvn -B clean compile package -Prelease -Dmaven.test.skip=true"
mvn -B clean compile package -Prelease -Dmaven.test.skip=true mvn -B clean compile package -Prelease -Dmaven.test.skip=true
# mv dolphinscheduler-bin.tar.gz file to dockerfile directory # mv dolphinscheduler-bin.tar.gz file to dockerfile directory
echo -e "mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${VERSION}-SNAPSHOT-dolphinscheduler-bin.tar.gz $(pwd)/dockerfile/\n" echo -e "mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz $(pwd)/dockerfile/\n"
mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${VERSION}-SNAPSHOT-dolphinscheduler-bin.tar.gz $(pwd)/dockerfile/ mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz $(pwd)/dockerfile/
# docker build # docker build
echo -e "docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/dockerfile/\n" echo -e "docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/dockerfile/\n"
docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/dockerfile/ sudo docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/dockerfile/
echo "------ dolphinscheduler end - build -------" echo "------ dolphinscheduler end - build -------"

Write Preview
Loading…
Cancel
Save