From cbada7c16de4c5b853aada6765bd352ff5262416 Mon Sep 17 00:00:00 2001 From: Shiwen Cheng Date: Sun, 14 Feb 2021 17:42:04 +0800 Subject: [PATCH] [Improvement][Docker] Reduce docker image size, remove frontend/nginx, remove stdout log, improve compose/swarm deployment and fix random PSQLException (#4643) * [Improvement][Docker] Reduce docker image size, remove frontend/nginx, remove stdout log and improve compose/swarm deployment * [Improvement][Docker] Fix random PSQLException --- .gitignore | 4 + docker/build/Dockerfile | 57 ++++--------- docker/build/README.md | 36 ++------ docker/build/README_zh_CN.md | 35 ++------ docker/build/checkpoint.sh | 0 .../env/dolphinscheduler_env.sh | 12 ++- .../logback/logback-alert.xml | 9 -- .../dolphinscheduler/logback/logback-api.xml | 9 -- .../logback/logback-master.xml | 9 -- .../logback/logback-worker.xml | 9 -- docker/build/conf/nginx/dolphinscheduler.conf | 51 ----------- docker/build/conf/zookeeper/zoo.cfg | 47 ----------- docker/build/hooks/build | 9 +- docker/build/hooks/push | 0 docker/build/startup-init-conf.sh | 18 +--- docker/build/startup.sh | 55 +++++------- docker/docker-swarm/check | 2 +- docker/docker-swarm/docker-compose.yml | 80 ++++++++---------- docker/docker-swarm/docker-stack.yml | 84 +++++++++---------- docker/docker-swarm/dolphinscheduler_env.sh | 12 ++- .../install-dolphinscheduler-ui.sh | 0 .../test/resources/config/config.properties | 2 +- script/create-dolphinscheduler.sh | 0 script/dolphinscheduler-daemon.sh | 6 ++ script/env/dolphinscheduler_env.sh | 0 script/monitor-server.sh | 0 script/scp-hosts.sh | 0 script/start-all.sh | 0 script/status-all.sh | 0 script/stop-all.sh | 0 script/upgrade-dolphinscheduler.sh | 0 31 files changed, 170 insertions(+), 376 deletions(-) mode change 100644 => 100755 docker/build/checkpoint.sh mode change 100644 => 100755 docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh delete mode 100644 docker/build/conf/nginx/dolphinscheduler.conf delete mode 100644 docker/build/conf/zookeeper/zoo.cfg mode change 100644 => 100755 docker/build/hooks/build mode change 100644 => 100755 docker/build/hooks/push mode change 100644 => 100755 docker/build/startup-init-conf.sh mode change 100644 => 100755 docker/build/startup.sh mode change 100644 => 100755 docker/docker-swarm/check mode change 100644 => 100755 docker/docker-swarm/dolphinscheduler_env.sh mode change 100644 => 100755 dolphinscheduler-ui/install-dolphinscheduler-ui.sh mode change 100644 => 100755 script/create-dolphinscheduler.sh mode change 100644 => 100755 script/dolphinscheduler-daemon.sh mode change 100644 => 100755 script/env/dolphinscheduler_env.sh mode change 100644 => 100755 script/monitor-server.sh mode change 100644 => 100755 script/scp-hosts.sh mode change 100644 => 100755 script/start-all.sh mode change 100644 => 100755 script/status-all.sh mode change 100644 => 100755 script/stop-all.sh mode change 100644 => 100755 script/upgrade-dolphinscheduler.sh diff --git a/.gitignore b/.gitignore index 6717162b37..9011db1479 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,10 @@ .target .idea/ target/ +dist/ +all-dependencies.txt +self-modules.txt +third-party-dependencies.txt .settings .nbproject .classpath diff --git a/docker/build/Dockerfile b/docker/build/Dockerfile index 7b0f09c539..a33cf490b1 100644 --- a/docker/build/Dockerfile +++ b/docker/build/Dockerfile @@ -15,55 +15,37 @@ # limitations under the License. # -FROM nginx:alpine +FROM openjdk:8-jdk-alpine ARG VERSION ENV TZ Asia/Shanghai ENV LANG C.UTF-8 -ENV DEBIAN_FRONTEND noninteractive +ENV DOCKER true -#1. install dos2unix shadow bash openrc python sudo vim wget iputils net-tools ssh pip tini kazoo. -#If install slowly, you can replcae alpine's mirror with aliyun's mirror, Example: -#RUN sed -i "s/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g" /etc/apk/repositories +# 1. install command/library/software +# If install slowly, you can replcae alpine's mirror with aliyun's mirror, Example: +# RUN sed -i "s/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g" /etc/apk/repositories +# RUN sed -i 's/dl-cdn.alpinelinux.org/mirror.tuna.tsinghua.edu.cn/g' /etc/apk/repositories RUN apk update && \ - apk add --update --no-cache dos2unix shadow bash openrc python2 python3 sudo vim wget iputils net-tools openssh-server py-pip tini && \ - apk add --update --no-cache procps && \ - openrc boot && \ - pip install kazoo + apk add --no-cache tzdata dos2unix bash python2 python3 procps sudo shadow tini postgresql-client && \ + cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \ + apk del tzdata && \ + rm -rf /var/cache/apk/* -#2. install jdk -RUN apk add --update --no-cache openjdk8 -ENV JAVA_HOME /usr/lib/jvm/java-1.8-openjdk -ENV PATH $JAVA_HOME/bin:$PATH - -#3. add dolphinscheduler +# 2. add dolphinscheduler ADD ./apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz /opt/ -RUN mv /opt/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin/ /opt/dolphinscheduler/ +RUN ln -s /opt/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin /opt/dolphinscheduler ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler -#4. install database, if use mysql as your backend database, the `mysql-client` package should be installed -RUN apk add --update --no-cache postgresql postgresql-contrib - -#5. modify nginx -RUN echo "daemon off;" >> /etc/nginx/nginx.conf && \ - rm -rf /etc/nginx/conf.d/* -COPY ./conf/nginx/dolphinscheduler.conf /etc/nginx/conf.d - -#6. add configuration and modify permissions and set soft links +# 3. add configuration and modify permissions and set soft links COPY ./checkpoint.sh /root/checkpoint.sh COPY ./startup-init-conf.sh /root/startup-init-conf.sh COPY ./startup.sh /root/startup.sh COPY ./conf/dolphinscheduler/*.tpl /opt/dolphinscheduler/conf/ COPY ./conf/dolphinscheduler/logback/* /opt/dolphinscheduler/conf/ -COPY conf/dolphinscheduler/env/dolphinscheduler_env.sh /opt/dolphinscheduler/conf/env/ -RUN chmod +x /root/checkpoint.sh && \ - chmod +x /root/startup-init-conf.sh && \ - chmod +x /root/startup.sh && \ - chmod +x /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh && \ - chmod +x /opt/dolphinscheduler/script/*.sh && \ - chmod +x /opt/dolphinscheduler/bin/*.sh && \ - dos2unix /root/checkpoint.sh && \ +COPY ./conf/dolphinscheduler/env/dolphinscheduler_env.sh /opt/dolphinscheduler/conf/env/ +RUN dos2unix /root/checkpoint.sh && \ dos2unix /root/startup-init-conf.sh && \ dos2unix /root/startup.sh && \ dos2unix /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh && \ @@ -71,13 +53,10 @@ RUN chmod +x /root/checkpoint.sh && \ dos2unix /opt/dolphinscheduler/bin/*.sh && \ rm -rf /bin/sh && \ ln -s /bin/bash /bin/sh && \ - mkdir -p /tmp/xls && \ - #7. remove apk index cache and disable coredup for sudo - rm -rf /var/cache/apk/* && \ + mkdir -p /var/mail /tmp/xls && \ echo "Set disable_coredump false" >> /etc/sudo.conf - -#8. expose port -EXPOSE 2181 2888 3888 5432 5678 1234 12345 50051 8888 +# 4. expose port +EXPOSE 5678 1234 12345 50051 ENTRYPOINT ["/sbin/tini", "--", "/root/startup.sh"] diff --git a/docker/build/README.md b/docker/build/README.md index 951f2d6b51..3a2a1666bb 100644 --- a/docker/build/README.md +++ b/docker/build/README.md @@ -15,9 +15,9 @@ Official Website: https://dolphinscheduler.apache.org #### You can start a dolphinscheduler instance ``` -$ docker run -dit --name dolphinscheduler \ +$ docker run -dit --name dolphinscheduler \ -e DATABASE_USERNAME=test -e DATABASE_PASSWORD=test -e DATABASE_DATABASE=dolphinscheduler \ --p 8888:8888 \ +-p 12345:12345 \ dolphinscheduler all ``` @@ -33,7 +33,7 @@ You can specify **existing postgres service**. Example: $ docker run -dit --name dolphinscheduler \ -e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ -e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ --p 8888:8888 \ +-p 12345:12345 \ dolphinscheduler all ``` @@ -43,7 +43,7 @@ You can specify **existing zookeeper service**. Example: $ docker run -dit --name dolphinscheduler \ -e ZOOKEEPER_QUORUM="l92.168.x.x:2181" -e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" -e DATABASE_DATABASE="dolphinscheduler" \ --p 8888:8888 \ +-p 12345:12345 \ dolphinscheduler all ``` @@ -90,14 +90,6 @@ $ docker run -dit --name dolphinscheduler \ dolphinscheduler alert-server ``` -* Start a **frontend**, For example: - -``` -$ docker run -dit --name dolphinscheduler \ --e FRONTEND_API_SERVER_HOST="192.168.x.x" -e FRONTEND_API_SERVER_PORT="12345" \ --p 8888:8888 \ -dolphinscheduler frontend -``` **Note**: You must be specify `DATABASE_HOST` `DATABASE_PORT` `DATABASE_DATABASE` `DATABASE_USERNAME` `DATABASE_PASSWORD` `ZOOKEEPER_QUORUM` when start a standalone dolphinscheduler server. @@ -146,7 +138,7 @@ This environment variable sets the host for database. The default value is `127. This environment variable sets the port for database. The default value is `5432`. -**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. +**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. **`DATABASE_USERNAME`** @@ -306,18 +298,6 @@ This environment variable sets enterprise wechat agent id for `alert-server`. Th This environment variable sets enterprise wechat users for `alert-server`. The default value is empty. -**`FRONTEND_API_SERVER_HOST`** - -This environment variable sets api server host for `frontend`. The default value is `127.0.0.1`. - -**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `api-server`. - -**`FRONTEND_API_SERVER_PORT`** - -This environment variable sets api server port for `frontend`. The default value is `123451`. - -**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `api-server`. - ## Initialization scripts If you would like to do additional initialization in an image derived from this one, add one or more environment variable under `/root/start-init-conf.sh`, and modify template files in `/opt/dolphinscheduler/conf/*.tpl`. @@ -326,7 +306,7 @@ For example, to add an environment variable `API_SERVER_PORT` in `/root/start-in ``` export API_SERVER_PORT=5555 -``` +``` and to modify `/opt/dolphinscheduler/conf/application-api.properties.tpl` template file, add server port: ``` @@ -343,8 +323,4 @@ $(cat ${DOLPHINSCHEDULER_HOME}/conf/${line}) EOF " > ${DOLPHINSCHEDULER_HOME}/conf/${line%.*} done - -echo "generate nginx config" -sed -i "s/FRONTEND_API_SERVER_HOST/${FRONTEND_API_SERVER_HOST}/g" /etc/nginx/conf.d/dolphinscheduler.conf -sed -i "s/FRONTEND_API_SERVER_PORT/${FRONTEND_API_SERVER_PORT}/g" /etc/nginx/conf.d/dolphinscheduler.conf ``` diff --git a/docker/build/README_zh_CN.md b/docker/build/README_zh_CN.md index c4339a945c..ffa0020dcd 100644 --- a/docker/build/README_zh_CN.md +++ b/docker/build/README_zh_CN.md @@ -15,9 +15,9 @@ Official Website: https://dolphinscheduler.apache.org #### 你可以运行一个dolphinscheduler实例 ``` -$ docker run -dit --name dolphinscheduler \ +$ docker run -dit --name dolphinscheduler \ -e DATABASE_USERNAME=test -e DATABASE_PASSWORD=test -e DATABASE_DATABASE=dolphinscheduler \ --p 8888:8888 \ +-p 12345:12345 \ dolphinscheduler all ``` @@ -33,7 +33,7 @@ dolphinscheduler all $ docker run -dit --name dolphinscheduler \ -e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ -e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ --p 8888:8888 \ +-p 12345:12345 \ dolphinscheduler all ``` @@ -43,7 +43,7 @@ dolphinscheduler all $ docker run -dit --name dolphinscheduler \ -e ZOOKEEPER_QUORUM="l92.168.x.x:2181" -e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" -e DATABASE_DATABASE="dolphinscheduler" \ --p 8888:8888 \ +-p 12345:12345 \ dolphinscheduler all ``` @@ -90,15 +90,6 @@ $ docker run -dit --name dolphinscheduler \ dolphinscheduler alert-server ``` -* 启动一个 **frontend**, 如下: - -``` -$ docker run -dit --name dolphinscheduler \ --e FRONTEND_API_SERVER_HOST="192.168.x.x" -e FRONTEND_API_SERVER_PORT="12345" \ --p 8888:8888 \ -dolphinscheduler frontend -``` - **注意**: 当你运行dolphinscheduler中的部分服务时，你必须指定这些环境变量 `DATABASE_HOST` `DATABASE_PORT` `DATABASE_DATABASE` `DATABASE_USERNAME` `DATABASE_PASSWORD` `ZOOKEEPER_QUORUM`。 ## 如何构建一个docker镜像 @@ -306,18 +297,6 @@ Dolphin Scheduler映像使用了几个容易遗漏的环境变量。虽然这些 配置`alert-server`的邮件服务企业微信`USERS`，默认值 `空`。 -**`FRONTEND_API_SERVER_HOST`** - -配置`frontend`的连接`api-server`的地址，默认值 `127.0.0.1`。 - -**Note**: 当单独运行`api-server`时，你应该指定`api-server`这个值。 - -**`FRONTEND_API_SERVER_PORT`** - -配置`frontend`的连接`api-server`的端口，默认值 `12345`。 - -**Note**: 当单独运行`api-server`时，你应该指定`api-server`这个值。 - ## 初始化脚本 如果你想在编译的时候或者运行的时候附加一些其它的操作及新增一些环境变量，你可以在`/root/start-init-conf.sh`文件中进行修改，同时如果涉及到配置文件的修改，请在`/opt/dolphinscheduler/conf/*.tpl`中修改相应的配置文件 @@ -326,7 +305,7 @@ Dolphin Scheduler映像使用了几个容易遗漏的环境变量。虽然这些 ``` export API_SERVER_PORT=5555 -``` +``` 当添加以上环境变量后，你应该在相应的模板文件`/opt/dolphinscheduler/conf/application-api.properties.tpl`中添加这个环境变量配置: ``` @@ -343,8 +322,4 @@ $(cat ${DOLPHINSCHEDULER_HOME}/conf/${line}) EOF " > ${DOLPHINSCHEDULER_HOME}/conf/${line%.*} done - -echo "generate nginx config" -sed -i "s/FRONTEND_API_SERVER_HOST/${FRONTEND_API_SERVER_HOST}/g" /etc/nginx/conf.d/dolphinscheduler.conf -sed -i "s/FRONTEND_API_SERVER_PORT/${FRONTEND_API_SERVER_PORT}/g" /etc/nginx/conf.d/dolphinscheduler.conf ``` diff --git a/docker/build/checkpoint.sh b/docker/build/checkpoint.sh old mode 100644 new mode 100755 diff --git a/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh b/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh old mode 100644 new mode 100755 index 78c8f98bc1..7fd39335ae --- a/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh +++ b/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh @@ -15,6 +15,14 @@ # limitations under the License. # -export PYTHON_HOME=/usr/bin/python2 +export HADOOP_HOME=/opt/soft/hadoop +export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop +export SPARK_HOME1=/opt/soft/spark1 +export SPARK_HOME2=/opt/soft/spark2 +export PYTHON_HOME=/usr/bin/python export JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk -export PATH=$PYTHON_HOME/bin:$JAVA_HOME/bin:$PATH +export HIVE_HOME=/opt/soft/hive +export FLINK_HOME=/opt/soft/flink +export DATAX_HOME=/opt/soft/datax/bin/datax.py + +export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH:$FLINK_HOME/bin:$DATAX_HOME:$PATH diff --git a/docker/build/conf/dolphinscheduler/logback/logback-alert.xml b/docker/build/conf/dolphinscheduler/logback/logback-alert.xml index 1718947dd1..eec78385db 100644 --- a/docker/build/conf/dolphinscheduler/logback/logback-alert.xml +++ b/docker/build/conf/dolphinscheduler/logback/logback-alert.xml @@ -20,14 +20,6 @@ - - - - [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n - - UTF-8 - - ${log.base}/dolphinscheduler-alert.log @@ -45,7 +37,6 @@ - diff --git a/docker/build/conf/dolphinscheduler/logback/logback-api.xml b/docker/build/conf/dolphinscheduler/logback/logback-api.xml index 2df90d8392..6d29f8af5f 100644 --- a/docker/build/conf/dolphinscheduler/logback/logback-api.xml +++ b/docker/build/conf/dolphinscheduler/logback/logback-api.xml @@ -20,14 +20,6 @@ - - - - [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n - - UTF-8 - - @@ -55,7 +47,6 @@ - diff --git a/docker/build/conf/dolphinscheduler/logback/logback-master.xml b/docker/build/conf/dolphinscheduler/logback/logback-master.xml index 7410c01f05..d1bfb67aa1 100644 --- a/docker/build/conf/dolphinscheduler/logback/logback-master.xml +++ b/docker/build/conf/dolphinscheduler/logback/logback-master.xml @@ -20,14 +20,6 @@ - - - - [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n - - UTF-8 - - @@ -74,7 +66,6 @@ - diff --git a/docker/build/conf/dolphinscheduler/logback/logback-worker.xml b/docker/build/conf/dolphinscheduler/logback/logback-worker.xml index be1d0acde5..b7e08dd846 100644 --- a/docker/build/conf/dolphinscheduler/logback/logback-worker.xml +++ b/docker/build/conf/dolphinscheduler/logback/logback-worker.xml @@ -20,14 +20,6 @@ - - - - [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n - - UTF-8 - - - diff --git a/docker/build/conf/nginx/dolphinscheduler.conf b/docker/build/conf/nginx/dolphinscheduler.conf deleted file mode 100644 index a594385a0e..0000000000 --- a/docker/build/conf/nginx/dolphinscheduler.conf +++ /dev/null @@ -1,51 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -server { - listen 8888; - server_name localhost; - #charset koi8-r; - #access_log /var/log/nginx/host.access.log main; - location / { - root /opt/dolphinscheduler/ui; - index index.html index.html; - } - location /dolphinscheduler/ui{ - alias /opt/dolphinscheduler/ui; - } - location /dolphinscheduler { - proxy_pass http://FRONTEND_API_SERVER_HOST:FRONTEND_API_SERVER_PORT; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header x_real_ipP $remote_addr; - proxy_set_header remote_addr $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_http_version 1.1; - proxy_connect_timeout 300s; - proxy_read_timeout 300s; - proxy_send_timeout 300s; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - } - #error_page 404 /404.html; - # redirect server error pages to the static page /50x.html - # - error_page 500 502 503 504 /50x.html; - location = /50x.html { - root /usr/share/nginx/html; - } -} diff --git a/docker/build/conf/zookeeper/zoo.cfg b/docker/build/conf/zookeeper/zoo.cfg deleted file mode 100644 index 94f92d0620..0000000000 --- a/docker/build/conf/zookeeper/zoo.cfg +++ /dev/null @@ -1,47 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# The number of milliseconds of each tick -tickTime=2000 -# The number of ticks that the initial -# synchronization phase can take -initLimit=10 -# The number of ticks that can pass between -# sending a request and getting an acknowledgement -syncLimit=5 -# the directory where the snapshot is stored. -# do not use /tmp for storage, /tmp here is just -# example sakes. -dataDir=/tmp/zookeeper -# the port at which the clients will connect -clientPort=2181 -# the maximum number of client connections. -# increase this if you need to handle more clients -#maxClientCnxns=60 -# -# Be sure to read the maintenance section of the -# administrator guide before turning on autopurge. -# -# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance -# -# The number of snapshots to retain in dataDir -#autopurge.snapRetainCount=3 -# Purge task interval in hours -# Set to "0" to disable auto purge feature -#autopurge.purgeInterval=1 -#Four Letter Words commands:stat,ruok,conf,isro -4lw.commands.whitelist=* diff --git a/docker/build/hooks/build b/docker/build/hooks/build old mode 100644 new mode 100755 index 6362fdd299..ce7362bf55 --- a/docker/build/hooks/build +++ b/docker/build/hooks/build @@ -48,7 +48,12 @@ echo -e "mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-incubati mv "$(pwd)"/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-"${VERSION}"-dolphinscheduler-bin.tar.gz $(pwd)/docker/build/ # docker build -echo -e "docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/docker/build/\n" -sudo docker build --build-arg VERSION="${VERSION}" -t $DOCKER_REPO:"${VERSION}" "$(pwd)/docker/build/" +BUILD_COMMAND="docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/docker/build/" +echo -e "$BUILD_COMMAND\n" +if (docker info 2> /dev/null | grep -i "ERROR"); then + sudo $BUILD_COMMAND +else + $BUILD_COMMAND +fi echo "------ dolphinscheduler end - build -------" diff --git a/docker/build/hooks/push b/docker/build/hooks/push old mode 100644 new mode 100755 diff --git a/docker/build/startup-init-conf.sh b/docker/build/startup-init-conf.sh old mode 100644 new mode 100755 index d5cd86f1a4..c3aadcd673 --- a/docker/build/startup-init-conf.sh +++ b/docker/build/startup-init-conf.sh @@ -39,9 +39,9 @@ export DATABASE_PARAMS=${DATABASE_PARAMS:-"characterEncoding=utf8"} export DOLPHINSCHEDULER_ENV_PATH=${DOLPHINSCHEDULER_ENV_PATH:-"/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh"} export DOLPHINSCHEDULER_DATA_BASEDIR_PATH=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH:-"/tmp/dolphinscheduler"} export DOLPHINSCHEDULER_OPTS=${DOLPHINSCHEDULER_OPTS:-""} -export RESOURCE_STORAGE_TYPE=${RESOURCE_STORAGE_TYPE:-"NONE"} -export RESOURCE_UPLOAD_PATH=${RESOURCE_UPLOAD_PATH:-"/ds"} -export FS_DEFAULT_FS=${FS_DEFAULT_FS:-"s3a://xxxx"} +export RESOURCE_STORAGE_TYPE=${RESOURCE_STORAGE_TYPE:-"HDFS"} +export RESOURCE_UPLOAD_PATH=${RESOURCE_UPLOAD_PATH:-"/dolphinscheduler"} +export FS_DEFAULT_FS=${FS_DEFAULT_FS:-"file:///"} export FS_S3A_ENDPOINT=${FS_S3A_ENDPOINT:-"s3.xxx.amazonaws.com"} export FS_S3A_ACCESS_KEY=${FS_S3A_ACCESS_KEY:-"xxxxxxx"} export FS_S3A_SECRET_KEY=${FS_S3A_SECRET_KEY:-"xxxxxxx"} @@ -81,7 +81,7 @@ export WORKER_WEIGHT=${WORKER_WEIGHT:-"100"} #============================================================================ # alert plugin dir export ALERT_PLUGIN_DIR=${ALERT_PLUGIN_DIR:-"/opt/dolphinscheduler"} -# XLS FILE +# xls file export XLS_FILE_PATH=${XLS_FILE_PATH:-"/tmp/xls"} # mail export MAIL_SERVER_HOST=${MAIL_SERVER_HOST:-""} @@ -99,12 +99,6 @@ export ENTERPRISE_WECHAT_SECRET=${ENTERPRISE_WECHAT_SECRET:-""} export ENTERPRISE_WECHAT_AGENT_ID=${ENTERPRISE_WECHAT_AGENT_ID:-""} export ENTERPRISE_WECHAT_USERS=${ENTERPRISE_WECHAT_USERS:-""} -#============================================================================ -# Frontend -#============================================================================ -export FRONTEND_API_SERVER_HOST=${FRONTEND_API_SERVER_HOST:-"127.0.0.1"} -export FRONTEND_API_SERVER_PORT=${FRONTEND_API_SERVER_PORT:-"12345"} - echo "generate app config" ls ${DOLPHINSCHEDULER_HOME}/conf/ | grep ".tpl" | while read line; do eval "cat << EOF @@ -112,7 +106,3 @@ $(cat ${DOLPHINSCHEDULER_HOME}/conf/${line}) EOF " > ${DOLPHINSCHEDULER_HOME}/conf/${line%.*} done - -echo "generate nginx config" -sed -i "s/FRONTEND_API_SERVER_HOST/${FRONTEND_API_SERVER_HOST}/g" /etc/nginx/conf.d/dolphinscheduler.conf -sed -i "s/FRONTEND_API_SERVER_PORT/${FRONTEND_API_SERVER_PORT}/g" /etc/nginx/conf.d/dolphinscheduler.conf \ No newline at end of file diff --git a/docker/build/startup.sh b/docker/build/startup.sh old mode 100644 new mode 100755 index 0511788d48..8bd1895cc0 --- a/docker/build/startup.sh +++ b/docker/build/startup.sh @@ -22,8 +22,8 @@ DOLPHINSCHEDULER_BIN=${DOLPHINSCHEDULER_HOME}/bin DOLPHINSCHEDULER_SCRIPT=${DOLPHINSCHEDULER_HOME}/script DOLPHINSCHEDULER_LOGS=${DOLPHINSCHEDULER_HOME}/logs -# start database -initDatabase() { +# wait database +waitDatabase() { echo "test ${DATABASE_TYPE} service" while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do counter=$((counter+1)) @@ -43,19 +43,22 @@ initDatabase() { exit 1 fi else - v=$(sudo -u postgres PGPASSWORD=${DATABASE_PASSWORD} psql -h ${DATABASE_HOST} -p ${DATABASE_PORT} -U ${DATABASE_USERNAME} -d ${DATABASE_DATABASE} -tAc "select 1") + v=$(PGPASSWORD=${DATABASE_PASSWORD} psql -h ${DATABASE_HOST} -p ${DATABASE_PORT} -U ${DATABASE_USERNAME} -d ${DATABASE_DATABASE} -tAc "select 1") if [ "$(echo ${v} | grep 'FATAL' | wc -l)" -eq 1 ]; then echo "Error: Can't connect to database...${v}" exit 1 fi fi +} +# init database +initDatabase() { echo "import sql data" ${DOLPHINSCHEDULER_SCRIPT}/create-dolphinscheduler.sh } -# start zk -initZK() { +# wait zk +waitZK() { echo "connect remote zookeeper" echo "${ZOOKEEPER_QUORUM}" | awk -F ',' 'BEGIN{ i=1 }{ while( i <= NF ){ print $i; i++ } }' | while read line; do while ! nc -z ${line%:*} ${line#*:}; do @@ -70,12 +73,6 @@ initZK() { done } -# start nginx -initNginx() { - echo "start nginx" - nginx & -} - # start master-server initMasterServer() { echo "start master-server" @@ -115,59 +112,54 @@ initAlertServer() { printUsage() { echo -e "Dolphin Scheduler is a distributed and easy-to-expand visual DAG workflow scheduling system," echo -e "dedicated to solving the complex dependencies in data processing, making the scheduling system out of the box for data processing.\n" - echo -e "Usage: [ all | master-server | worker-server | api-server | alert-server | frontend ]\n" - printf "%-13s: %s\n" "all" "Run master-server, worker-server, api-server, alert-server and frontend." + echo -e "Usage: [ all | master-server | worker-server | api-server | alert-server ]\n" + printf "%-13s: %s\n" "all" "Run master-server, worker-server, api-server and alert-server" printf "%-13s: %s\n" "master-server" "MasterServer is mainly responsible for DAG task split, task submission monitoring." - printf "%-13s: %s\n" "worker-server" "WorkerServer is mainly responsible for task execution and providing log services.." - printf "%-13s: %s\n" "api-server" "ApiServer is mainly responsible for processing requests from the front-end UI layer." + printf "%-13s: %s\n" "worker-server" "WorkerServer is mainly responsible for task execution and providing log services." + printf "%-13s: %s\n" "api-server" "ApiServer is mainly responsible for processing requests and providing the front-end UI layer." printf "%-13s: %s\n" "alert-server" "AlertServer mainly include Alarms." - printf "%-13s: %s\n" "frontend" "Frontend mainly provides various visual operation interfaces of the system." } # init config file source /root/startup-init-conf.sh -LOGFILE=/var/log/nginx/access.log case "$1" in (all) - initZK + waitZK + waitDatabase initDatabase initMasterServer initWorkerServer initApiServer initAlertServer initLoggerServer - initNginx - LOGFILE=/var/log/nginx/access.log + LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-api-server.log ;; (master-server) - initZK - initDatabase + waitZK + waitDatabase initMasterServer LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-master.log ;; (worker-server) - initZK - initDatabase + waitZK + waitDatabase initWorkerServer initLoggerServer LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-worker.log ;; (api-server) - initZK + waitZK + waitDatabase initDatabase initApiServer LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-api-server.log ;; (alert-server) - initDatabase + waitDatabase initAlertServer LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-alert.log ;; - (frontend) - initNginx - LOGFILE=/var/log/nginx/access.log - ;; (help) printUsage exit 1 @@ -179,8 +171,7 @@ case "$1" in esac # init directories and log files -mkdir -p ${DOLPHINSCHEDULER_LOGS} && mkdir -p /var/log/nginx/ && cat /dev/null >> ${LOGFILE} +mkdir -p ${DOLPHINSCHEDULER_LOGS} && cat /dev/null >> ${LOGFILE} echo "tail begin" exec bash -c "tail -n 1 -f ${LOGFILE}" - diff --git a/docker/docker-swarm/check b/docker/docker-swarm/check old mode 100644 new mode 100755 index 59203c5b3e..f4093664da --- a/docker/docker-swarm/check +++ b/docker/docker-swarm/check @@ -25,7 +25,7 @@ else echo "Server start failed "$server_num exit 1 fi -ready=`curl http://127.0.0.1:8888/dolphinscheduler/login -d 'userName=admin&userPassword=dolphinscheduler123' -v | grep "login success" | wc -l` +ready=`curl http://127.0.0.1:12345/dolphinscheduler/login -d 'userName=admin&userPassword=dolphinscheduler123' -v | grep "login success" | wc -l` if [ $ready -eq 1 ] then echo "Servers is ready" diff --git a/docker/docker-swarm/docker-compose.yml b/docker/docker-swarm/docker-compose.yml index 349b3ad790..04a9bc556d 100644 --- a/docker/docker-swarm/docker-compose.yml +++ b/docker/docker-swarm/docker-compose.yml @@ -31,6 +31,7 @@ services: volumes: - dolphinscheduler-postgresql:/bitnami/postgresql - dolphinscheduler-postgresql-initdb:/docker-entrypoint-initdb.d + restart: unless-stopped networks: - dolphinscheduler @@ -45,13 +46,14 @@ services: ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons volumes: - dolphinscheduler-zookeeper:/bitnami/zookeeper + restart: unless-stopped networks: - dolphinscheduler dolphinscheduler-api: image: apache/dolphinscheduler:latest container_name: dolphinscheduler-api - command: ["api-server"] + command: api-server ports: - 12345:12345 environment: @@ -62,6 +64,9 @@ services: DATABASE_PASSWORD: root DATABASE_DATABASE: dolphinscheduler ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 + RESOURCE_STORAGE_TYPE: HDFS + RESOURCE_UPLOAD_PATH: /dolphinscheduler + FS_DEFAULT_FS: file:/// healthcheck: test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"] interval: 30s @@ -72,37 +77,16 @@ services: - dolphinscheduler-postgresql - dolphinscheduler-zookeeper volumes: - - ./dolphinscheduler-logs:/opt/dolphinscheduler/logs - networks: - - dolphinscheduler - - dolphinscheduler-frontend: - image: apache/dolphinscheduler:latest - container_name: dolphinscheduler-frontend - command: ["frontend"] - ports: - - 8888:8888 - environment: - TZ: Asia/Shanghai - FRONTEND_API_SERVER_HOST: dolphinscheduler-api - FRONTEND_API_SERVER_PORT: 12345 - healthcheck: - test: ["CMD", "nc", "-z", "localhost", "8888"] - interval: 30s - timeout: 5s - retries: 3 - start_period: 30s - depends_on: - - dolphinscheduler-api - volumes: - - ./dolphinscheduler-logs:/var/log/nginx + - dolphinscheduler-logs:/opt/dolphinscheduler/logs + - dolphinscheduler-resource-local:/dolphinscheduler + restart: unless-stopped networks: - dolphinscheduler dolphinscheduler-alert: image: apache/dolphinscheduler:latest container_name: dolphinscheduler-alert - command: ["alert-server"] + command: alert-server environment: TZ: Asia/Shanghai XLS_FILE_PATH: "/tmp/xls" @@ -133,14 +117,15 @@ services: depends_on: - dolphinscheduler-postgresql volumes: - - ./dolphinscheduler-logs:/opt/dolphinscheduler/logs + - dolphinscheduler-logs:/opt/dolphinscheduler/logs + restart: unless-stopped networks: - dolphinscheduler dolphinscheduler-master: image: apache/dolphinscheduler:latest container_name: dolphinscheduler-master - command: ["master-server"] + command: master-server ports: - 5678:5678 environment: @@ -168,14 +153,15 @@ services: - dolphinscheduler-postgresql - dolphinscheduler-zookeeper volumes: - - ./dolphinscheduler-logs:/opt/dolphinscheduler/logs + - dolphinscheduler-logs:/opt/dolphinscheduler/logs + restart: unless-stopped networks: - dolphinscheduler dolphinscheduler-worker: image: apache/dolphinscheduler:latest container_name: dolphinscheduler-worker - command: ["worker-server"] + command: worker-server ports: - 1234:1234 - 50051:50051 @@ -188,30 +174,40 @@ services: WORKER_RESERVED_MEMORY: "0.1" WORKER_GROUP: "default" WORKER_WEIGHT: "100" - DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler" + DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler + XLS_FILE_PATH: "/tmp/xls" + MAIL_SERVER_HOST: "" + MAIL_SERVER_PORT: "" + MAIL_SENDER: "" + MAIL_USER: "" + MAIL_PASSWD: "" + MAIL_SMTP_STARTTLS_ENABLE: "false" + MAIL_SMTP_SSL_ENABLE: "false" + MAIL_SMTP_SSL_TRUST: "" DATABASE_HOST: dolphinscheduler-postgresql DATABASE_PORT: 5432 DATABASE_USERNAME: root DATABASE_PASSWORD: root DATABASE_DATABASE: dolphinscheduler ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 + RESOURCE_STORAGE_TYPE: HDFS + RESOURCE_UPLOAD_PATH: /dolphinscheduler + FS_DEFAULT_FS: file:/// healthcheck: test: ["CMD", "/root/checkpoint.sh", "WorkerServer"] interval: 30s timeout: 5s retries: 3 start_period: 30s - depends_on: + depends_on: - dolphinscheduler-postgresql - dolphinscheduler-zookeeper volumes: - - type: bind - source: ./dolphinscheduler_env.sh - target: /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh - - type: volume - source: dolphinscheduler-worker-data - target: /tmp/dolphinscheduler - - ./dolphinscheduler-logs:/opt/dolphinscheduler/logs + - ./dolphinscheduler_env.sh:/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh + - dolphinscheduler-worker-data:/tmp/dolphinscheduler + - dolphinscheduler-logs:/opt/dolphinscheduler/logs + - dolphinscheduler-resource-local:/dolphinscheduler + restart: unless-stopped networks: - dolphinscheduler @@ -224,7 +220,5 @@ volumes: dolphinscheduler-postgresql-initdb: dolphinscheduler-zookeeper: dolphinscheduler-worker-data: - -configs: - dolphinscheduler-worker-task-env: - file: ./dolphinscheduler_env.sh \ No newline at end of file + dolphinscheduler-logs: + dolphinscheduler-resource-local: \ No newline at end of file diff --git a/docker/docker-swarm/docker-stack.yml b/docker/docker-swarm/docker-stack.yml index dff4a47b2c..39c36347bb 100644 --- a/docker/docker-swarm/docker-stack.yml +++ b/docker/docker-swarm/docker-stack.yml @@ -20,13 +20,13 @@ services: dolphinscheduler-postgresql: image: bitnami/postgresql:latest + ports: + - 5432:5432 environment: TZ: Asia/Shanghai POSTGRESQL_USERNAME: root POSTGRESQL_PASSWORD: root POSTGRESQL_DATABASE: dolphinscheduler - ports: - - 5432:5432 volumes: - dolphinscheduler-postgresql:/bitnami/postgresql networks: @@ -37,12 +37,12 @@ services: dolphinscheduler-zookeeper: image: bitnami/zookeeper:latest + ports: + - 2181:2181 environment: TZ: Asia/Shanghai ALLOW_ANONYMOUS_LOGIN: "yes" ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons - ports: - - 2181:2181 volumes: - dolphinscheduler-zookeeper:/bitnami/zookeeper networks: @@ -53,7 +53,9 @@ services: dolphinscheduler-api: image: apache/dolphinscheduler:latest - command: ["api-server"] + command: api-server + ports: + - 12345:12345 environment: TZ: Asia/Shanghai DATABASE_HOST: dolphinscheduler-postgresql @@ -62,39 +64,17 @@ services: DATABASE_PASSWORD: root DATABASE_DATABASE: dolphinscheduler ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 - ports: - - 12345:12345 + RESOURCE_STORAGE_TYPE: HDFS + RESOURCE_UPLOAD_PATH: /dolphinscheduler + FS_DEFAULT_FS: file:/// healthcheck: test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"] - interval: 30 + interval: 30s timeout: 5s retries: 3 start_period: 30s volumes: - dolphinscheduler-logs:/opt/dolphinscheduler/logs - networks: - - dolphinscheduler - deploy: - mode: replicated - replicas: 1 - - dolphinscheduler-frontend: - image: apache/dolphinscheduler:latest - command: ["frontend"] - ports: - - 8888:8888 - environment: - TZ: Asia/Shanghai - FRONTEND_API_SERVER_HOST: dolphinscheduler-api - FRONTEND_API_SERVER_PORT: 12345 - healthcheck: - test: ["CMD", "nc", "-z", "localhost", "8888"] - interval: 30 - timeout: 5s - retries: 3 - start_period: 30s - volumes: - - dolphinscheduler-logs:/var/log/nginx networks: - dolphinscheduler deploy: @@ -103,7 +83,7 @@ services: dolphinscheduler-alert: image: apache/dolphinscheduler:latest - command: ["alert-server"] + command: alert-server environment: TZ: Asia/Shanghai XLS_FILE_PATH: "/tmp/xls" @@ -127,13 +107,13 @@ services: DATABASE_DATABASE: dolphinscheduler healthcheck: test: ["CMD", "/root/checkpoint.sh", "AlertServer"] - interval: 30 + interval: 30s timeout: 5s retries: 3 start_period: 30s volumes: - dolphinscheduler-logs:/opt/dolphinscheduler/logs - networks: + networks: - dolphinscheduler deploy: mode: replicated @@ -141,10 +121,10 @@ services: dolphinscheduler-master: image: apache/dolphinscheduler:latest - command: ["master-server"] - ports: + command: master-server + ports: - 5678:5678 - environment: + environment: TZ: Asia/Shanghai MASTER_EXEC_THREADS: "100" MASTER_EXEC_TASK_NUM: "20" @@ -161,7 +141,7 @@ services: ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 healthcheck: test: ["CMD", "/root/checkpoint.sh", "MasterServer"] - interval: 30 + interval: 30s timeout: 5s retries: 3 start_period: 30s @@ -175,11 +155,11 @@ services: dolphinscheduler-worker: image: apache/dolphinscheduler:latest - command: ["worker-server"] - ports: + command: worker-server + ports: - 1234:1234 - 50051:50051 - environment: + environment: TZ: Asia/Shanghai WORKER_EXEC_THREADS: "100" WORKER_HEARTBEAT_INTERVAL: "10" @@ -188,25 +168,37 @@ services: WORKER_RESERVED_MEMORY: "0.1" WORKER_GROUP: "default" WORKER_WEIGHT: "100" - DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler" + DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler + XLS_FILE_PATH: "/tmp/xls" + MAIL_SERVER_HOST: "" + MAIL_SERVER_PORT: "" + MAIL_SENDER: "" + MAIL_USER: "" + MAIL_PASSWD: "" + MAIL_SMTP_STARTTLS_ENABLE: "false" + MAIL_SMTP_SSL_ENABLE: "false" + MAIL_SMTP_SSL_TRUST: "" DATABASE_HOST: dolphinscheduler-postgresql DATABASE_PORT: 5432 DATABASE_USERNAME: root DATABASE_PASSWORD: root DATABASE_DATABASE: dolphinscheduler ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 + RESOURCE_STORAGE_TYPE: HDFS + RESOURCE_UPLOAD_PATH: /dolphinscheduler + FS_DEFAULT_FS: file:/// healthcheck: test: ["CMD", "/root/checkpoint.sh", "WorkerServer"] - interval: 30 + interval: 30s timeout: 5s retries: 3 start_period: 30s - volumes: - - dolphinscheduler-worker-data:/tmp/dolphinscheduler - - dolphinscheduler-logs:/opt/dolphinscheduler/logs configs: - source: dolphinscheduler-worker-task-env target: /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh + volumes: + - dolphinscheduler-worker-data:/tmp/dolphinscheduler + - dolphinscheduler-logs:/opt/dolphinscheduler/logs networks: - dolphinscheduler deploy: diff --git a/docker/docker-swarm/dolphinscheduler_env.sh b/docker/docker-swarm/dolphinscheduler_env.sh old mode 100644 new mode 100755 index 654318cb41..7fd39335ae --- a/docker/docker-swarm/dolphinscheduler_env.sh +++ b/docker/docker-swarm/dolphinscheduler_env.sh @@ -15,6 +15,14 @@ # limitations under the License. # -export PYTHON_HOME=/usr/bin/python2 +export HADOOP_HOME=/opt/soft/hadoop +export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop +export SPARK_HOME1=/opt/soft/spark1 +export SPARK_HOME2=/opt/soft/spark2 +export PYTHON_HOME=/usr/bin/python export JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk -export PATH=$PYTHON_HOME:$JAVA_HOME/bin:$PATH \ No newline at end of file +export HIVE_HOME=/opt/soft/hive +export FLINK_HOME=/opt/soft/flink +export DATAX_HOME=/opt/soft/datax/bin/datax.py + +export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH:$FLINK_HOME/bin:$DATAX_HOME:$PATH diff --git a/dolphinscheduler-ui/install-dolphinscheduler-ui.sh b/dolphinscheduler-ui/install-dolphinscheduler-ui.sh old mode 100644 new mode 100755 diff --git a/e2e/src/test/resources/config/config.properties b/e2e/src/test/resources/config/config.properties index 352d9375be..135af0baa0 100644 --- a/e2e/src/test/resources/config/config.properties +++ b/e2e/src/test/resources/config/config.properties @@ -17,7 +17,7 @@ ############### project ############## # login url -LOGIN_URL=http://127.0.0.1:8888/dolphinscheduler/ +LOGIN_URL=http://127.0.0.1:12345/dolphinscheduler/ #login username USER_NAME=admin #login password diff --git a/script/create-dolphinscheduler.sh b/script/create-dolphinscheduler.sh old mode 100644 new mode 100755 diff --git a/script/dolphinscheduler-daemon.sh b/script/dolphinscheduler-daemon.sh old mode 100644 new mode 100755 index 5e7652199e..a87e62e187 --- a/script/dolphinscheduler-daemon.sh +++ b/script/dolphinscheduler-daemon.sh @@ -54,6 +54,12 @@ fi log=$DOLPHINSCHEDULER_LOG_DIR/dolphinscheduler-$command-$HOSTNAME.out pid=$DOLPHINSCHEDULER_PID_DIR/dolphinscheduler-$command.pid +# print logs to /dev/null in docker +if [ "$DOCKER" = "true" ]; then + echo "start in docker" + log=/dev/null +fi + cd $DOLPHINSCHEDULER_HOME if [ "$command" = "api-server" ]; then diff --git a/script/env/dolphinscheduler_env.sh b/script/env/dolphinscheduler_env.sh old mode 100644 new mode 100755 diff --git a/script/monitor-server.sh b/script/monitor-server.sh old mode 100644 new mode 100755 diff --git a/script/scp-hosts.sh b/script/scp-hosts.sh old mode 100644 new mode 100755 diff --git a/script/start-all.sh b/script/start-all.sh old mode 100644 new mode 100755 diff --git a/script/status-all.sh b/script/status-all.sh old mode 100644 new mode 100755 diff --git a/script/stop-all.sh b/script/stop-all.sh old mode 100644 new mode 100755 diff --git a/script/upgrade-dolphinscheduler.sh b/script/upgrade-dolphinscheduler.sh old mode 100644 new mode 100755