From c1efbf3080c616af3eaaa6619547cdc02a78ac4d Mon Sep 17 00:00:00 2001 From: yankexi Date: Fri, 12 Feb 2021 04:40:30 -0600 Subject: [PATCH 1/8] [Fix-4760][api][Repair dolphinscheduler-postgre.sql error sql] (#4763) * Repair error sql about dolphinscheduler-postgre.sql * Repair error sql about dolphinscheduler-postgre.sql * Repair error sql about dolphinscheduler-postgre.sql (cherry picked from commit 739a633) * [Fix-4760][api][Repair dolphinscheduler-postgre.sql error sql] --- sql/dolphinscheduler-postgre.sql | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sql/dolphinscheduler-postgre.sql b/sql/dolphinscheduler-postgre.sql index 8ca6cd8338..b8f638ff99 100644 --- a/sql/dolphinscheduler-postgre.sql +++ b/sql/dolphinscheduler-postgre.sql @@ -769,8 +769,7 @@ VALUES ('admin', '7ad2410b2f4c074479a8937a28a22b8f', '0', 'xxx@qq.com', '', '0', -- Records of t_ds_alertgroup，dolphinscheduler warning group INSERT INTO t_ds_alertgroup(id,alert_instance_ids, create_user_id, group_name, description, create_time, update_time) -VALUES (1, '1,2','dolphinscheduler warning group', 'dolphinscheduler warning group', '2018-11-29 10:20:39', - '2018-11-29 10:20:39'); +VALUES (1,'1,2', 1, 'default admin warning group', 'default admin warning group', '2018-11-29 10:20:39', '2018-11-29 10:20:39'); -- Records of t_ds_queue,default queue name : default INSERT INTO t_ds_queue(queue_name, queue, create_time, update_time) From 7a6b905597d974d1c3a3154aa21ec9cbf61e519f Mon Sep 17 00:00:00 2001 From: guohaozhang Date: Fri, 12 Feb 2021 18:50:38 +0800 Subject: [PATCH 2/8] [Fix-#4716][mater,worker] The task execution path should be calculated by the worker, not the master (#4717) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [fix][api,mater,worker] 修复master 计算实际执行路径，worker上报实际执行路径 * remove unused import --- .../builder/TaskExecutionContextBuilder.java | 1 - .../master/consumer/TaskPriorityQueueConsumer.java | 14 -------------- .../worker/processor/TaskExecuteProcessor.java | 1 + 3 files changed, 1 insertion(+), 15 deletions(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/builder/TaskExecutionContextBuilder.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/builder/TaskExecutionContextBuilder.java index 74b0635145..7bfd9a0507 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/builder/TaskExecutionContextBuilder.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/builder/TaskExecutionContextBuilder.java @@ -44,7 +44,6 @@ public class TaskExecutionContextBuilder { taskExecutionContext.setStartTime(taskInstance.getStartTime()); taskExecutionContext.setTaskType(taskInstance.getTaskType()); taskExecutionContext.setLogPath(taskInstance.getLogPath()); - taskExecutionContext.setExecutePath(taskInstance.getExecutePath()); taskExecutionContext.setTaskJson(taskInstance.getTaskJson()); taskExecutionContext.setWorkerGroup(taskInstance.getWorkerGroup()); taskExecutionContext.setHost(taskInstance.getHost()); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java index 23255084e0..d7de840dfa 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java @@ -35,7 +35,6 @@ import org.apache.dolphinscheduler.common.task.sqoop.targets.TargetMysqlParamete import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.EnumUtils; -import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.common.utils.TaskParametersUtils; @@ -222,7 +221,6 @@ public class TaskPriorityQueueConsumer extends Thread { String userQueue = processService.queryUserQueueByProcessInstanceId(taskInstance.getProcessInstanceId()); taskInstance.getProcessInstance().setQueue(StringUtils.isEmpty(userQueue) ? tenant.getQueue() : userQueue); taskInstance.getProcessInstance().setTenantCode(tenant.getTenantCode()); - taskInstance.setExecutePath(getExecLocalPath(taskInstance)); taskInstance.setResources(getResourceFullNames(taskNode)); SQLTaskExecutionContext sqlTaskExecutionContext = new SQLTaskExecutionContext(); @@ -363,18 +361,6 @@ public class TaskPriorityQueueConsumer extends Thread { } } - /** - * get execute local path - * - * @return execute local path - */ - private String getExecLocalPath(TaskInstance taskInstance) { - return FileUtils.getProcessExecDir(taskInstance.getProcessDefine().getProjectId(), - taskInstance.getProcessDefine().getId(), - taskInstance.getProcessInstance().getId(), - taskInstance.getId()); - } - /** * whehter tenant is null * diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java index d8a6e2bfd8..18971dd363 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java @@ -147,6 +147,7 @@ public class TaskExecuteProcessor implements NettyRequestProcessor { // local execute path String execLocalPath = getExecLocalPath(taskExecutionContext); logger.info("task instance local execute path : {} ", execLocalPath); + taskExecutionContext.setExecutePath(execLocalPath); FileUtils.taskLoggerThreadLocal.set(taskLogger); try { From cbada7c16de4c5b853aada6765bd352ff5262416 Mon Sep 17 00:00:00 2001 From: Shiwen Cheng Date: Sun, 14 Feb 2021 17:42:04 +0800 Subject: [PATCH 3/8] [Improvement][Docker] Reduce docker image size, remove frontend/nginx, remove stdout log, improve compose/swarm deployment and fix random PSQLException (#4643) * [Improvement][Docker] Reduce docker image size, remove frontend/nginx, remove stdout log and improve compose/swarm deployment * [Improvement][Docker] Fix random PSQLException --- .gitignore | 4 + docker/build/Dockerfile | 57 ++++--------- docker/build/README.md | 36 ++------ docker/build/README_zh_CN.md | 35 ++------ docker/build/checkpoint.sh | 0 .../env/dolphinscheduler_env.sh | 12 ++- .../logback/logback-alert.xml | 9 -- .../dolphinscheduler/logback/logback-api.xml | 9 -- .../logback/logback-master.xml | 9 -- .../logback/logback-worker.xml | 9 -- docker/build/conf/nginx/dolphinscheduler.conf | 51 ----------- docker/build/conf/zookeeper/zoo.cfg | 47 ----------- docker/build/hooks/build | 9 +- docker/build/hooks/push | 0 docker/build/startup-init-conf.sh | 18 +--- docker/build/startup.sh | 55 +++++------- docker/docker-swarm/check | 2 +- docker/docker-swarm/docker-compose.yml | 80 ++++++++---------- docker/docker-swarm/docker-stack.yml | 84 +++++++++---------- docker/docker-swarm/dolphinscheduler_env.sh | 12 ++- .../install-dolphinscheduler-ui.sh | 0 .../test/resources/config/config.properties | 2 +- script/create-dolphinscheduler.sh | 0 script/dolphinscheduler-daemon.sh | 6 ++ script/env/dolphinscheduler_env.sh | 0 script/monitor-server.sh | 0 script/scp-hosts.sh | 0 script/start-all.sh | 0 script/status-all.sh | 0 script/stop-all.sh | 0 script/upgrade-dolphinscheduler.sh | 0 31 files changed, 170 insertions(+), 376 deletions(-) mode change 100644 => 100755 docker/build/checkpoint.sh mode change 100644 => 100755 docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh delete mode 100644 docker/build/conf/nginx/dolphinscheduler.conf delete mode 100644 docker/build/conf/zookeeper/zoo.cfg mode change 100644 => 100755 docker/build/hooks/build mode change 100644 => 100755 docker/build/hooks/push mode change 100644 => 100755 docker/build/startup-init-conf.sh mode change 100644 => 100755 docker/build/startup.sh mode change 100644 => 100755 docker/docker-swarm/check mode change 100644 => 100755 docker/docker-swarm/dolphinscheduler_env.sh mode change 100644 => 100755 dolphinscheduler-ui/install-dolphinscheduler-ui.sh mode change 100644 => 100755 script/create-dolphinscheduler.sh mode change 100644 => 100755 script/dolphinscheduler-daemon.sh mode change 100644 => 100755 script/env/dolphinscheduler_env.sh mode change 100644 => 100755 script/monitor-server.sh mode change 100644 => 100755 script/scp-hosts.sh mode change 100644 => 100755 script/start-all.sh mode change 100644 => 100755 script/status-all.sh mode change 100644 => 100755 script/stop-all.sh mode change 100644 => 100755 script/upgrade-dolphinscheduler.sh diff --git a/.gitignore b/.gitignore index 6717162b37..9011db1479 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,10 @@ .target .idea/ target/ +dist/ +all-dependencies.txt +self-modules.txt +third-party-dependencies.txt .settings .nbproject .classpath diff --git a/docker/build/Dockerfile b/docker/build/Dockerfile index 7b0f09c539..a33cf490b1 100644 --- a/docker/build/Dockerfile +++ b/docker/build/Dockerfile @@ -15,55 +15,37 @@ # limitations under the License. # -FROM nginx:alpine +FROM openjdk:8-jdk-alpine ARG VERSION ENV TZ Asia/Shanghai ENV LANG C.UTF-8 -ENV DEBIAN_FRONTEND noninteractive +ENV DOCKER true -#1. install dos2unix shadow bash openrc python sudo vim wget iputils net-tools ssh pip tini kazoo. -#If install slowly, you can replcae alpine's mirror with aliyun's mirror, Example: -#RUN sed -i "s/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g" /etc/apk/repositories +# 1. install command/library/software +# If install slowly, you can replcae alpine's mirror with aliyun's mirror, Example: +# RUN sed -i "s/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g" /etc/apk/repositories +# RUN sed -i 's/dl-cdn.alpinelinux.org/mirror.tuna.tsinghua.edu.cn/g' /etc/apk/repositories RUN apk update && \ - apk add --update --no-cache dos2unix shadow bash openrc python2 python3 sudo vim wget iputils net-tools openssh-server py-pip tini && \ - apk add --update --no-cache procps && \ - openrc boot && \ - pip install kazoo + apk add --no-cache tzdata dos2unix bash python2 python3 procps sudo shadow tini postgresql-client && \ + cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \ + apk del tzdata && \ + rm -rf /var/cache/apk/* -#2. install jdk -RUN apk add --update --no-cache openjdk8 -ENV JAVA_HOME /usr/lib/jvm/java-1.8-openjdk -ENV PATH $JAVA_HOME/bin:$PATH - -#3. add dolphinscheduler +# 2. add dolphinscheduler ADD ./apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz /opt/ -RUN mv /opt/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin/ /opt/dolphinscheduler/ +RUN ln -s /opt/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin /opt/dolphinscheduler ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler -#4. install database, if use mysql as your backend database, the `mysql-client` package should be installed -RUN apk add --update --no-cache postgresql postgresql-contrib - -#5. modify nginx -RUN echo "daemon off;" >> /etc/nginx/nginx.conf && \ - rm -rf /etc/nginx/conf.d/* -COPY ./conf/nginx/dolphinscheduler.conf /etc/nginx/conf.d - -#6. add configuration and modify permissions and set soft links +# 3. add configuration and modify permissions and set soft links COPY ./checkpoint.sh /root/checkpoint.sh COPY ./startup-init-conf.sh /root/startup-init-conf.sh COPY ./startup.sh /root/startup.sh COPY ./conf/dolphinscheduler/*.tpl /opt/dolphinscheduler/conf/ COPY ./conf/dolphinscheduler/logback/* /opt/dolphinscheduler/conf/ -COPY conf/dolphinscheduler/env/dolphinscheduler_env.sh /opt/dolphinscheduler/conf/env/ -RUN chmod +x /root/checkpoint.sh && \ - chmod +x /root/startup-init-conf.sh && \ - chmod +x /root/startup.sh && \ - chmod +x /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh && \ - chmod +x /opt/dolphinscheduler/script/*.sh && \ - chmod +x /opt/dolphinscheduler/bin/*.sh && \ - dos2unix /root/checkpoint.sh && \ +COPY ./conf/dolphinscheduler/env/dolphinscheduler_env.sh /opt/dolphinscheduler/conf/env/ +RUN dos2unix /root/checkpoint.sh && \ dos2unix /root/startup-init-conf.sh && \ dos2unix /root/startup.sh && \ dos2unix /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh && \ @@ -71,13 +53,10 @@ RUN chmod +x /root/checkpoint.sh && \ dos2unix /opt/dolphinscheduler/bin/*.sh && \ rm -rf /bin/sh && \ ln -s /bin/bash /bin/sh && \ - mkdir -p /tmp/xls && \ - #7. remove apk index cache and disable coredup for sudo - rm -rf /var/cache/apk/* && \ + mkdir -p /var/mail /tmp/xls && \ echo "Set disable_coredump false" >> /etc/sudo.conf - -#8. expose port -EXPOSE 2181 2888 3888 5432 5678 1234 12345 50051 8888 +# 4. expose port +EXPOSE 5678 1234 12345 50051 ENTRYPOINT ["/sbin/tini", "--", "/root/startup.sh"] diff --git a/docker/build/README.md b/docker/build/README.md index 951f2d6b51..3a2a1666bb 100644 --- a/docker/build/README.md +++ b/docker/build/README.md @@ -15,9 +15,9 @@ Official Website: https://dolphinscheduler.apache.org #### You can start a dolphinscheduler instance ``` -$ docker run -dit --name dolphinscheduler \ +$ docker run -dit --name dolphinscheduler \ -e DATABASE_USERNAME=test -e DATABASE_PASSWORD=test -e DATABASE_DATABASE=dolphinscheduler \ --p 8888:8888 \ +-p 12345:12345 \ dolphinscheduler all ``` @@ -33,7 +33,7 @@ You can specify **existing postgres service**. Example: $ docker run -dit --name dolphinscheduler \ -e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ -e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ --p 8888:8888 \ +-p 12345:12345 \ dolphinscheduler all ``` @@ -43,7 +43,7 @@ You can specify **existing zookeeper service**. Example: $ docker run -dit --name dolphinscheduler \ -e ZOOKEEPER_QUORUM="l92.168.x.x:2181" -e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" -e DATABASE_DATABASE="dolphinscheduler" \ --p 8888:8888 \ +-p 12345:12345 \ dolphinscheduler all ``` @@ -90,14 +90,6 @@ $ docker run -dit --name dolphinscheduler \ dolphinscheduler alert-server ``` -* Start a **frontend**, For example: - -``` -$ docker run -dit --name dolphinscheduler \ --e FRONTEND_API_SERVER_HOST="192.168.x.x" -e FRONTEND_API_SERVER_PORT="12345" \ --p 8888:8888 \ -dolphinscheduler frontend -``` **Note**: You must be specify `DATABASE_HOST` `DATABASE_PORT` `DATABASE_DATABASE` `DATABASE_USERNAME` `DATABASE_PASSWORD` `ZOOKEEPER_QUORUM` when start a standalone dolphinscheduler server. @@ -146,7 +138,7 @@ This environment variable sets the host for database. The default value is `127. This environment variable sets the port for database. The default value is `5432`. -**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. +**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. **`DATABASE_USERNAME`** @@ -306,18 +298,6 @@ This environment variable sets enterprise wechat agent id for `alert-server`. Th This environment variable sets enterprise wechat users for `alert-server`. The default value is empty. -**`FRONTEND_API_SERVER_HOST`** - -This environment variable sets api server host for `frontend`. The default value is `127.0.0.1`. - -**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `api-server`. - -**`FRONTEND_API_SERVER_PORT`** - -This environment variable sets api server port for `frontend`. The default value is `123451`. - -**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `api-server`. - ## Initialization scripts If you would like to do additional initialization in an image derived from this one, add one or more environment variable under `/root/start-init-conf.sh`, and modify template files in `/opt/dolphinscheduler/conf/*.tpl`. @@ -326,7 +306,7 @@ For example, to add an environment variable `API_SERVER_PORT` in `/root/start-in ``` export API_SERVER_PORT=5555 -``` +``` and to modify `/opt/dolphinscheduler/conf/application-api.properties.tpl` template file, add server port: ``` @@ -343,8 +323,4 @@ $(cat ${DOLPHINSCHEDULER_HOME}/conf/${line}) EOF " > ${DOLPHINSCHEDULER_HOME}/conf/${line%.*} done - -echo "generate nginx config" -sed -i "s/FRONTEND_API_SERVER_HOST/${FRONTEND_API_SERVER_HOST}/g" /etc/nginx/conf.d/dolphinscheduler.conf -sed -i "s/FRONTEND_API_SERVER_PORT/${FRONTEND_API_SERVER_PORT}/g" /etc/nginx/conf.d/dolphinscheduler.conf ``` diff --git a/docker/build/README_zh_CN.md b/docker/build/README_zh_CN.md index c4339a945c..ffa0020dcd 100644 --- a/docker/build/README_zh_CN.md +++ b/docker/build/README_zh_CN.md @@ -15,9 +15,9 @@ Official Website: https://dolphinscheduler.apache.org #### 你可以运行一个dolphinscheduler实例 ``` -$ docker run -dit --name dolphinscheduler \ +$ docker run -dit --name dolphinscheduler \ -e DATABASE_USERNAME=test -e DATABASE_PASSWORD=test -e DATABASE_DATABASE=dolphinscheduler \ --p 8888:8888 \ +-p 12345:12345 \ dolphinscheduler all ``` @@ -33,7 +33,7 @@ dolphinscheduler all $ docker run -dit --name dolphinscheduler \ -e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ -e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ --p 8888:8888 \ +-p 12345:12345 \ dolphinscheduler all ``` @@ -43,7 +43,7 @@ dolphinscheduler all $ docker run -dit --name dolphinscheduler \ -e ZOOKEEPER_QUORUM="l92.168.x.x:2181" -e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" -e DATABASE_DATABASE="dolphinscheduler" \ --p 8888:8888 \ +-p 12345:12345 \ dolphinscheduler all ``` @@ -90,15 +90,6 @@ $ docker run -dit --name dolphinscheduler \ dolphinscheduler alert-server ``` -* 启动一个 **frontend**, 如下: - -``` -$ docker run -dit --name dolphinscheduler \ --e FRONTEND_API_SERVER_HOST="192.168.x.x" -e FRONTEND_API_SERVER_PORT="12345" \ --p 8888:8888 \ -dolphinscheduler frontend -``` - **注意**: 当你运行dolphinscheduler中的部分服务时，你必须指定这些环境变量 `DATABASE_HOST` `DATABASE_PORT` `DATABASE_DATABASE` `DATABASE_USERNAME` `DATABASE_PASSWORD` `ZOOKEEPER_QUORUM`。 ## 如何构建一个docker镜像 @@ -306,18 +297,6 @@ Dolphin Scheduler映像使用了几个容易遗漏的环境变量。虽然这些 配置`alert-server`的邮件服务企业微信`USERS`，默认值 `空`。 -**`FRONTEND_API_SERVER_HOST`** - -配置`frontend`的连接`api-server`的地址，默认值 `127.0.0.1`。 - -**Note**: 当单独运行`api-server`时，你应该指定`api-server`这个值。 - -**`FRONTEND_API_SERVER_PORT`** - -配置`frontend`的连接`api-server`的端口，默认值 `12345`。 - -**Note**: 当单独运行`api-server`时，你应该指定`api-server`这个值。 - ## 初始化脚本 如果你想在编译的时候或者运行的时候附加一些其它的操作及新增一些环境变量，你可以在`/root/start-init-conf.sh`文件中进行修改，同时如果涉及到配置文件的修改，请在`/opt/dolphinscheduler/conf/*.tpl`中修改相应的配置文件 @@ -326,7 +305,7 @@ Dolphin Scheduler映像使用了几个容易遗漏的环境变量。虽然这些 ``` export API_SERVER_PORT=5555 -``` +``` 当添加以上环境变量后，你应该在相应的模板文件`/opt/dolphinscheduler/conf/application-api.properties.tpl`中添加这个环境变量配置: ``` @@ -343,8 +322,4 @@ $(cat ${DOLPHINSCHEDULER_HOME}/conf/${line}) EOF " > ${DOLPHINSCHEDULER_HOME}/conf/${line%.*} done - -echo "generate nginx config" -sed -i "s/FRONTEND_API_SERVER_HOST/${FRONTEND_API_SERVER_HOST}/g" /etc/nginx/conf.d/dolphinscheduler.conf -sed -i "s/FRONTEND_API_SERVER_PORT/${FRONTEND_API_SERVER_PORT}/g" /etc/nginx/conf.d/dolphinscheduler.conf ``` diff --git a/docker/build/checkpoint.sh b/docker/build/checkpoint.sh old mode 100644 new mode 100755 diff --git a/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh b/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh old mode 100644 new mode 100755 index 78c8f98bc1..7fd39335ae --- a/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh +++ b/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh @@ -15,6 +15,14 @@ # limitations under the License. # -export PYTHON_HOME=/usr/bin/python2 +export HADOOP_HOME=/opt/soft/hadoop +export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop +export SPARK_HOME1=/opt/soft/spark1 +export SPARK_HOME2=/opt/soft/spark2 +export PYTHON_HOME=/usr/bin/python export JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk -export PATH=$PYTHON_HOME/bin:$JAVA_HOME/bin:$PATH +export HIVE_HOME=/opt/soft/hive +export FLINK_HOME=/opt/soft/flink +export DATAX_HOME=/opt/soft/datax/bin/datax.py + +export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH:$FLINK_HOME/bin:$DATAX_HOME:$PATH diff --git a/docker/build/conf/dolphinscheduler/logback/logback-alert.xml b/docker/build/conf/dolphinscheduler/logback/logback-alert.xml index 1718947dd1..eec78385db 100644 --- a/docker/build/conf/dolphinscheduler/logback/logback-alert.xml +++ b/docker/build/conf/dolphinscheduler/logback/logback-alert.xml @@ -20,14 +20,6 @@ - - - - [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n - - UTF-8 - - ${log.base}/dolphinscheduler-alert.log @@ -45,7 +37,6 @@ - diff --git a/docker/build/conf/dolphinscheduler/logback/logback-api.xml b/docker/build/conf/dolphinscheduler/logback/logback-api.xml index 2df90d8392..6d29f8af5f 100644 --- a/docker/build/conf/dolphinscheduler/logback/logback-api.xml +++ b/docker/build/conf/dolphinscheduler/logback/logback-api.xml @@ -20,14 +20,6 @@ - - - - [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n - - UTF-8 - - @@ -55,7 +47,6 @@ - diff --git a/docker/build/conf/dolphinscheduler/logback/logback-master.xml b/docker/build/conf/dolphinscheduler/logback/logback-master.xml index 7410c01f05..d1bfb67aa1 100644 --- a/docker/build/conf/dolphinscheduler/logback/logback-master.xml +++ b/docker/build/conf/dolphinscheduler/logback/logback-master.xml @@ -20,14 +20,6 @@ - - - - [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n - - UTF-8 - - @@ -74,7 +66,6 @@ - diff --git a/docker/build/conf/dolphinscheduler/logback/logback-worker.xml b/docker/build/conf/dolphinscheduler/logback/logback-worker.xml index be1d0acde5..b7e08dd846 100644 --- a/docker/build/conf/dolphinscheduler/logback/logback-worker.xml +++ b/docker/build/conf/dolphinscheduler/logback/logback-worker.xml @@ -20,14 +20,6 @@ - - - - [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n - - UTF-8 - - - diff --git a/docker/build/conf/nginx/dolphinscheduler.conf b/docker/build/conf/nginx/dolphinscheduler.conf deleted file mode 100644 index a594385a0e..0000000000 --- a/docker/build/conf/nginx/dolphinscheduler.conf +++ /dev/null @@ -1,51 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -server { - listen 8888; - server_name localhost; - #charset koi8-r; - #access_log /var/log/nginx/host.access.log main; - location / { - root /opt/dolphinscheduler/ui; - index index.html index.html; - } - location /dolphinscheduler/ui{ - alias /opt/dolphinscheduler/ui; - } - location /dolphinscheduler { - proxy_pass http://FRONTEND_API_SERVER_HOST:FRONTEND_API_SERVER_PORT; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header x_real_ipP $remote_addr; - proxy_set_header remote_addr $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_http_version 1.1; - proxy_connect_timeout 300s; - proxy_read_timeout 300s; - proxy_send_timeout 300s; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - } - #error_page 404 /404.html; - # redirect server error pages to the static page /50x.html - # - error_page 500 502 503 504 /50x.html; - location = /50x.html { - root /usr/share/nginx/html; - } -} diff --git a/docker/build/conf/zookeeper/zoo.cfg b/docker/build/conf/zookeeper/zoo.cfg deleted file mode 100644 index 94f92d0620..0000000000 --- a/docker/build/conf/zookeeper/zoo.cfg +++ /dev/null @@ -1,47 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# The number of milliseconds of each tick -tickTime=2000 -# The number of ticks that the initial -# synchronization phase can take -initLimit=10 -# The number of ticks that can pass between -# sending a request and getting an acknowledgement -syncLimit=5 -# the directory where the snapshot is stored. -# do not use /tmp for storage, /tmp here is just -# example sakes. -dataDir=/tmp/zookeeper -# the port at which the clients will connect -clientPort=2181 -# the maximum number of client connections. -# increase this if you need to handle more clients -#maxClientCnxns=60 -# -# Be sure to read the maintenance section of the -# administrator guide before turning on autopurge. -# -# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance -# -# The number of snapshots to retain in dataDir -#autopurge.snapRetainCount=3 -# Purge task interval in hours -# Set to "0" to disable auto purge feature -#autopurge.purgeInterval=1 -#Four Letter Words commands:stat,ruok,conf,isro -4lw.commands.whitelist=* diff --git a/docker/build/hooks/build b/docker/build/hooks/build old mode 100644 new mode 100755 index 6362fdd299..ce7362bf55 --- a/docker/build/hooks/build +++ b/docker/build/hooks/build @@ -48,7 +48,12 @@ echo -e "mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-incubati mv "$(pwd)"/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-"${VERSION}"-dolphinscheduler-bin.tar.gz $(pwd)/docker/build/ # docker build -echo -e "docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/docker/build/\n" -sudo docker build --build-arg VERSION="${VERSION}" -t $DOCKER_REPO:"${VERSION}" "$(pwd)/docker/build/" +BUILD_COMMAND="docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/docker/build/" +echo -e "$BUILD_COMMAND\n" +if (docker info 2> /dev/null | grep -i "ERROR"); then + sudo $BUILD_COMMAND +else + $BUILD_COMMAND +fi echo "------ dolphinscheduler end - build -------" diff --git a/docker/build/hooks/push b/docker/build/hooks/push old mode 100644 new mode 100755 diff --git a/docker/build/startup-init-conf.sh b/docker/build/startup-init-conf.sh old mode 100644 new mode 100755 index d5cd86f1a4..c3aadcd673 --- a/docker/build/startup-init-conf.sh +++ b/docker/build/startup-init-conf.sh @@ -39,9 +39,9 @@ export DATABASE_PARAMS=${DATABASE_PARAMS:-"characterEncoding=utf8"} export DOLPHINSCHEDULER_ENV_PATH=${DOLPHINSCHEDULER_ENV_PATH:-"/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh"} export DOLPHINSCHEDULER_DATA_BASEDIR_PATH=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH:-"/tmp/dolphinscheduler"} export DOLPHINSCHEDULER_OPTS=${DOLPHINSCHEDULER_OPTS:-""} -export RESOURCE_STORAGE_TYPE=${RESOURCE_STORAGE_TYPE:-"NONE"} -export RESOURCE_UPLOAD_PATH=${RESOURCE_UPLOAD_PATH:-"/ds"} -export FS_DEFAULT_FS=${FS_DEFAULT_FS:-"s3a://xxxx"} +export RESOURCE_STORAGE_TYPE=${RESOURCE_STORAGE_TYPE:-"HDFS"} +export RESOURCE_UPLOAD_PATH=${RESOURCE_UPLOAD_PATH:-"/dolphinscheduler"} +export FS_DEFAULT_FS=${FS_DEFAULT_FS:-"file:///"} export FS_S3A_ENDPOINT=${FS_S3A_ENDPOINT:-"s3.xxx.amazonaws.com"} export FS_S3A_ACCESS_KEY=${FS_S3A_ACCESS_KEY:-"xxxxxxx"} export FS_S3A_SECRET_KEY=${FS_S3A_SECRET_KEY:-"xxxxxxx"} @@ -81,7 +81,7 @@ export WORKER_WEIGHT=${WORKER_WEIGHT:-"100"} #============================================================================ # alert plugin dir export ALERT_PLUGIN_DIR=${ALERT_PLUGIN_DIR:-"/opt/dolphinscheduler"} -# XLS FILE +# xls file export XLS_FILE_PATH=${XLS_FILE_PATH:-"/tmp/xls"} # mail export MAIL_SERVER_HOST=${MAIL_SERVER_HOST:-""} @@ -99,12 +99,6 @@ export ENTERPRISE_WECHAT_SECRET=${ENTERPRISE_WECHAT_SECRET:-""} export ENTERPRISE_WECHAT_AGENT_ID=${ENTERPRISE_WECHAT_AGENT_ID:-""} export ENTERPRISE_WECHAT_USERS=${ENTERPRISE_WECHAT_USERS:-""} -#============================================================================ -# Frontend -#============================================================================ -export FRONTEND_API_SERVER_HOST=${FRONTEND_API_SERVER_HOST:-"127.0.0.1"} -export FRONTEND_API_SERVER_PORT=${FRONTEND_API_SERVER_PORT:-"12345"} - echo "generate app config" ls ${DOLPHINSCHEDULER_HOME}/conf/ | grep ".tpl" | while read line; do eval "cat << EOF @@ -112,7 +106,3 @@ $(cat ${DOLPHINSCHEDULER_HOME}/conf/${line}) EOF " > ${DOLPHINSCHEDULER_HOME}/conf/${line%.*} done - -echo "generate nginx config" -sed -i "s/FRONTEND_API_SERVER_HOST/${FRONTEND_API_SERVER_HOST}/g" /etc/nginx/conf.d/dolphinscheduler.conf -sed -i "s/FRONTEND_API_SERVER_PORT/${FRONTEND_API_SERVER_PORT}/g" /etc/nginx/conf.d/dolphinscheduler.conf \ No newline at end of file diff --git a/docker/build/startup.sh b/docker/build/startup.sh old mode 100644 new mode 100755 index 0511788d48..8bd1895cc0 --- a/docker/build/startup.sh +++ b/docker/build/startup.sh @@ -22,8 +22,8 @@ DOLPHINSCHEDULER_BIN=${DOLPHINSCHEDULER_HOME}/bin DOLPHINSCHEDULER_SCRIPT=${DOLPHINSCHEDULER_HOME}/script DOLPHINSCHEDULER_LOGS=${DOLPHINSCHEDULER_HOME}/logs -# start database -initDatabase() { +# wait database +waitDatabase() { echo "test ${DATABASE_TYPE} service" while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do counter=$((counter+1)) @@ -43,19 +43,22 @@ initDatabase() { exit 1 fi else - v=$(sudo -u postgres PGPASSWORD=${DATABASE_PASSWORD} psql -h ${DATABASE_HOST} -p ${DATABASE_PORT} -U ${DATABASE_USERNAME} -d ${DATABASE_DATABASE} -tAc "select 1") + v=$(PGPASSWORD=${DATABASE_PASSWORD} psql -h ${DATABASE_HOST} -p ${DATABASE_PORT} -U ${DATABASE_USERNAME} -d ${DATABASE_DATABASE} -tAc "select 1") if [ "$(echo ${v} | grep 'FATAL' | wc -l)" -eq 1 ]; then echo "Error: Can't connect to database...${v}" exit 1 fi fi +} +# init database +initDatabase() { echo "import sql data" ${DOLPHINSCHEDULER_SCRIPT}/create-dolphinscheduler.sh } -# start zk -initZK() { +# wait zk +waitZK() { echo "connect remote zookeeper" echo "${ZOOKEEPER_QUORUM}" | awk -F ',' 'BEGIN{ i=1 }{ while( i <= NF ){ print $i; i++ } }' | while read line; do while ! nc -z ${line%:*} ${line#*:}; do @@ -70,12 +73,6 @@ initZK() { done } -# start nginx -initNginx() { - echo "start nginx" - nginx & -} - # start master-server initMasterServer() { echo "start master-server" @@ -115,59 +112,54 @@ initAlertServer() { printUsage() { echo -e "Dolphin Scheduler is a distributed and easy-to-expand visual DAG workflow scheduling system," echo -e "dedicated to solving the complex dependencies in data processing, making the scheduling system out of the box for data processing.\n" - echo -e "Usage: [ all | master-server | worker-server | api-server | alert-server | frontend ]\n" - printf "%-13s: %s\n" "all" "Run master-server, worker-server, api-server, alert-server and frontend." + echo -e "Usage: [ all | master-server | worker-server | api-server | alert-server ]\n" + printf "%-13s: %s\n" "all" "Run master-server, worker-server, api-server and alert-server" printf "%-13s: %s\n" "master-server" "MasterServer is mainly responsible for DAG task split, task submission monitoring." - printf "%-13s: %s\n" "worker-server" "WorkerServer is mainly responsible for task execution and providing log services.." - printf "%-13s: %s\n" "api-server" "ApiServer is mainly responsible for processing requests from the front-end UI layer." + printf "%-13s: %s\n" "worker-server" "WorkerServer is mainly responsible for task execution and providing log services." + printf "%-13s: %s\n" "api-server" "ApiServer is mainly responsible for processing requests and providing the front-end UI layer." printf "%-13s: %s\n" "alert-server" "AlertServer mainly include Alarms." - printf "%-13s: %s\n" "frontend" "Frontend mainly provides various visual operation interfaces of the system." } # init config file source /root/startup-init-conf.sh -LOGFILE=/var/log/nginx/access.log case "$1" in (all) - initZK + waitZK + waitDatabase initDatabase initMasterServer initWorkerServer initApiServer initAlertServer initLoggerServer - initNginx - LOGFILE=/var/log/nginx/access.log + LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-api-server.log ;; (master-server) - initZK - initDatabase + waitZK + waitDatabase initMasterServer LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-master.log ;; (worker-server) - initZK - initDatabase + waitZK + waitDatabase initWorkerServer initLoggerServer LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-worker.log ;; (api-server) - initZK + waitZK + waitDatabase initDatabase initApiServer LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-api-server.log ;; (alert-server) - initDatabase + waitDatabase initAlertServer LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-alert.log ;; - (frontend) - initNginx - LOGFILE=/var/log/nginx/access.log - ;; (help) printUsage exit 1 @@ -179,8 +171,7 @@ case "$1" in esac # init directories and log files -mkdir -p ${DOLPHINSCHEDULER_LOGS} && mkdir -p /var/log/nginx/ && cat /dev/null >> ${LOGFILE} +mkdir -p ${DOLPHINSCHEDULER_LOGS} && cat /dev/null >> ${LOGFILE} echo "tail begin" exec bash -c "tail -n 1 -f ${LOGFILE}" - diff --git a/docker/docker-swarm/check b/docker/docker-swarm/check old mode 100644 new mode 100755 index 59203c5b3e..f4093664da --- a/docker/docker-swarm/check +++ b/docker/docker-swarm/check @@ -25,7 +25,7 @@ else echo "Server start failed "$server_num exit 1 fi -ready=`curl http://127.0.0.1:8888/dolphinscheduler/login -d 'userName=admin&userPassword=dolphinscheduler123' -v | grep "login success" | wc -l` +ready=`curl http://127.0.0.1:12345/dolphinscheduler/login -d 'userName=admin&userPassword=dolphinscheduler123' -v | grep "login success" | wc -l` if [ $ready -eq 1 ] then echo "Servers is ready" diff --git a/docker/docker-swarm/docker-compose.yml b/docker/docker-swarm/docker-compose.yml index 349b3ad790..04a9bc556d 100644 --- a/docker/docker-swarm/docker-compose.yml +++ b/docker/docker-swarm/docker-compose.yml @@ -31,6 +31,7 @@ services: volumes: - dolphinscheduler-postgresql:/bitnami/postgresql - dolphinscheduler-postgresql-initdb:/docker-entrypoint-initdb.d + restart: unless-stopped networks: - dolphinscheduler @@ -45,13 +46,14 @@ services: ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons volumes: - dolphinscheduler-zookeeper:/bitnami/zookeeper + restart: unless-stopped networks: - dolphinscheduler dolphinscheduler-api: image: apache/dolphinscheduler:latest container_name: dolphinscheduler-api - command: ["api-server"] + command: api-server ports: - 12345:12345 environment: @@ -62,6 +64,9 @@ services: DATABASE_PASSWORD: root DATABASE_DATABASE: dolphinscheduler ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 + RESOURCE_STORAGE_TYPE: HDFS + RESOURCE_UPLOAD_PATH: /dolphinscheduler + FS_DEFAULT_FS: file:/// healthcheck: test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"] interval: 30s @@ -72,37 +77,16 @@ services: - dolphinscheduler-postgresql - dolphinscheduler-zookeeper volumes: - - ./dolphinscheduler-logs:/opt/dolphinscheduler/logs - networks: - - dolphinscheduler - - dolphinscheduler-frontend: - image: apache/dolphinscheduler:latest - container_name: dolphinscheduler-frontend - command: ["frontend"] - ports: - - 8888:8888 - environment: - TZ: Asia/Shanghai - FRONTEND_API_SERVER_HOST: dolphinscheduler-api - FRONTEND_API_SERVER_PORT: 12345 - healthcheck: - test: ["CMD", "nc", "-z", "localhost", "8888"] - interval: 30s - timeout: 5s - retries: 3 - start_period: 30s - depends_on: - - dolphinscheduler-api - volumes: - - ./dolphinscheduler-logs:/var/log/nginx + - dolphinscheduler-logs:/opt/dolphinscheduler/logs + - dolphinscheduler-resource-local:/dolphinscheduler + restart: unless-stopped networks: - dolphinscheduler dolphinscheduler-alert: image: apache/dolphinscheduler:latest container_name: dolphinscheduler-alert - command: ["alert-server"] + command: alert-server environment: TZ: Asia/Shanghai XLS_FILE_PATH: "/tmp/xls" @@ -133,14 +117,15 @@ services: depends_on: - dolphinscheduler-postgresql volumes: - - ./dolphinscheduler-logs:/opt/dolphinscheduler/logs + - dolphinscheduler-logs:/opt/dolphinscheduler/logs + restart: unless-stopped networks: - dolphinscheduler dolphinscheduler-master: image: apache/dolphinscheduler:latest container_name: dolphinscheduler-master - command: ["master-server"] + command: master-server ports: - 5678:5678 environment: @@ -168,14 +153,15 @@ services: - dolphinscheduler-postgresql - dolphinscheduler-zookeeper volumes: - - ./dolphinscheduler-logs:/opt/dolphinscheduler/logs + - dolphinscheduler-logs:/opt/dolphinscheduler/logs + restart: unless-stopped networks: - dolphinscheduler dolphinscheduler-worker: image: apache/dolphinscheduler:latest container_name: dolphinscheduler-worker - command: ["worker-server"] + command: worker-server ports: - 1234:1234 - 50051:50051 @@ -188,30 +174,40 @@ services: WORKER_RESERVED_MEMORY: "0.1" WORKER_GROUP: "default" WORKER_WEIGHT: "100" - DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler" + DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler + XLS_FILE_PATH: "/tmp/xls" + MAIL_SERVER_HOST: "" + MAIL_SERVER_PORT: "" + MAIL_SENDER: "" + MAIL_USER: "" + MAIL_PASSWD: "" + MAIL_SMTP_STARTTLS_ENABLE: "false" + MAIL_SMTP_SSL_ENABLE: "false" + MAIL_SMTP_SSL_TRUST: "" DATABASE_HOST: dolphinscheduler-postgresql DATABASE_PORT: 5432 DATABASE_USERNAME: root DATABASE_PASSWORD: root DATABASE_DATABASE: dolphinscheduler ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 + RESOURCE_STORAGE_TYPE: HDFS + RESOURCE_UPLOAD_PATH: /dolphinscheduler + FS_DEFAULT_FS: file:/// healthcheck: test: ["CMD", "/root/checkpoint.sh", "WorkerServer"] interval: 30s timeout: 5s retries: 3 start_period: 30s - depends_on: + depends_on: - dolphinscheduler-postgresql - dolphinscheduler-zookeeper volumes: - - type: bind - source: ./dolphinscheduler_env.sh - target: /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh - - type: volume - source: dolphinscheduler-worker-data - target: /tmp/dolphinscheduler - - ./dolphinscheduler-logs:/opt/dolphinscheduler/logs + - ./dolphinscheduler_env.sh:/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh + - dolphinscheduler-worker-data:/tmp/dolphinscheduler + - dolphinscheduler-logs:/opt/dolphinscheduler/logs + - dolphinscheduler-resource-local:/dolphinscheduler + restart: unless-stopped networks: - dolphinscheduler @@ -224,7 +220,5 @@ volumes: dolphinscheduler-postgresql-initdb: dolphinscheduler-zookeeper: dolphinscheduler-worker-data: - -configs: - dolphinscheduler-worker-task-env: - file: ./dolphinscheduler_env.sh \ No newline at end of file + dolphinscheduler-logs: + dolphinscheduler-resource-local: \ No newline at end of file diff --git a/docker/docker-swarm/docker-stack.yml b/docker/docker-swarm/docker-stack.yml index dff4a47b2c..39c36347bb 100644 --- a/docker/docker-swarm/docker-stack.yml +++ b/docker/docker-swarm/docker-stack.yml @@ -20,13 +20,13 @@ services: dolphinscheduler-postgresql: image: bitnami/postgresql:latest + ports: + - 5432:5432 environment: TZ: Asia/Shanghai POSTGRESQL_USERNAME: root POSTGRESQL_PASSWORD: root POSTGRESQL_DATABASE: dolphinscheduler - ports: - - 5432:5432 volumes: - dolphinscheduler-postgresql:/bitnami/postgresql networks: @@ -37,12 +37,12 @@ services: dolphinscheduler-zookeeper: image: bitnami/zookeeper:latest + ports: + - 2181:2181 environment: TZ: Asia/Shanghai ALLOW_ANONYMOUS_LOGIN: "yes" ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons - ports: - - 2181:2181 volumes: - dolphinscheduler-zookeeper:/bitnami/zookeeper networks: @@ -53,7 +53,9 @@ services: dolphinscheduler-api: image: apache/dolphinscheduler:latest - command: ["api-server"] + command: api-server + ports: + - 12345:12345 environment: TZ: Asia/Shanghai DATABASE_HOST: dolphinscheduler-postgresql @@ -62,39 +64,17 @@ services: DATABASE_PASSWORD: root DATABASE_DATABASE: dolphinscheduler ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 - ports: - - 12345:12345 + RESOURCE_STORAGE_TYPE: HDFS + RESOURCE_UPLOAD_PATH: /dolphinscheduler + FS_DEFAULT_FS: file:/// healthcheck: test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"] - interval: 30 + interval: 30s timeout: 5s retries: 3 start_period: 30s volumes: - dolphinscheduler-logs:/opt/dolphinscheduler/logs - networks: - - dolphinscheduler - deploy: - mode: replicated - replicas: 1 - - dolphinscheduler-frontend: - image: apache/dolphinscheduler:latest - command: ["frontend"] - ports: - - 8888:8888 - environment: - TZ: Asia/Shanghai - FRONTEND_API_SERVER_HOST: dolphinscheduler-api - FRONTEND_API_SERVER_PORT: 12345 - healthcheck: - test: ["CMD", "nc", "-z", "localhost", "8888"] - interval: 30 - timeout: 5s - retries: 3 - start_period: 30s - volumes: - - dolphinscheduler-logs:/var/log/nginx networks: - dolphinscheduler deploy: @@ -103,7 +83,7 @@ services: dolphinscheduler-alert: image: apache/dolphinscheduler:latest - command: ["alert-server"] + command: alert-server environment: TZ: Asia/Shanghai XLS_FILE_PATH: "/tmp/xls" @@ -127,13 +107,13 @@ services: DATABASE_DATABASE: dolphinscheduler healthcheck: test: ["CMD", "/root/checkpoint.sh", "AlertServer"] - interval: 30 + interval: 30s timeout: 5s retries: 3 start_period: 30s volumes: - dolphinscheduler-logs:/opt/dolphinscheduler/logs - networks: + networks: - dolphinscheduler deploy: mode: replicated @@ -141,10 +121,10 @@ services: dolphinscheduler-master: image: apache/dolphinscheduler:latest - command: ["master-server"] - ports: + command: master-server + ports: - 5678:5678 - environment: + environment: TZ: Asia/Shanghai MASTER_EXEC_THREADS: "100" MASTER_EXEC_TASK_NUM: "20" @@ -161,7 +141,7 @@ services: ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 healthcheck: test: ["CMD", "/root/checkpoint.sh", "MasterServer"] - interval: 30 + interval: 30s timeout: 5s retries: 3 start_period: 30s @@ -175,11 +155,11 @@ services: dolphinscheduler-worker: image: apache/dolphinscheduler:latest - command: ["worker-server"] - ports: + command: worker-server + ports: - 1234:1234 - 50051:50051 - environment: + environment: TZ: Asia/Shanghai WORKER_EXEC_THREADS: "100" WORKER_HEARTBEAT_INTERVAL: "10" @@ -188,25 +168,37 @@ services: WORKER_RESERVED_MEMORY: "0.1" WORKER_GROUP: "default" WORKER_WEIGHT: "100" - DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler" + DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler + XLS_FILE_PATH: "/tmp/xls" + MAIL_SERVER_HOST: "" + MAIL_SERVER_PORT: "" + MAIL_SENDER: "" + MAIL_USER: "" + MAIL_PASSWD: "" + MAIL_SMTP_STARTTLS_ENABLE: "false" + MAIL_SMTP_SSL_ENABLE: "false" + MAIL_SMTP_SSL_TRUST: "" DATABASE_HOST: dolphinscheduler-postgresql DATABASE_PORT: 5432 DATABASE_USERNAME: root DATABASE_PASSWORD: root DATABASE_DATABASE: dolphinscheduler ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 + RESOURCE_STORAGE_TYPE: HDFS + RESOURCE_UPLOAD_PATH: /dolphinscheduler + FS_DEFAULT_FS: file:/// healthcheck: test: ["CMD", "/root/checkpoint.sh", "WorkerServer"] - interval: 30 + interval: 30s timeout: 5s retries: 3 start_period: 30s - volumes: - - dolphinscheduler-worker-data:/tmp/dolphinscheduler - - dolphinscheduler-logs:/opt/dolphinscheduler/logs configs: - source: dolphinscheduler-worker-task-env target: /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh + volumes: + - dolphinscheduler-worker-data:/tmp/dolphinscheduler + - dolphinscheduler-logs:/opt/dolphinscheduler/logs networks: - dolphinscheduler deploy: diff --git a/docker/docker-swarm/dolphinscheduler_env.sh b/docker/docker-swarm/dolphinscheduler_env.sh old mode 100644 new mode 100755 index 654318cb41..7fd39335ae --- a/docker/docker-swarm/dolphinscheduler_env.sh +++ b/docker/docker-swarm/dolphinscheduler_env.sh @@ -15,6 +15,14 @@ # limitations under the License. # -export PYTHON_HOME=/usr/bin/python2 +export HADOOP_HOME=/opt/soft/hadoop +export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop +export SPARK_HOME1=/opt/soft/spark1 +export SPARK_HOME2=/opt/soft/spark2 +export PYTHON_HOME=/usr/bin/python export JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk -export PATH=$PYTHON_HOME:$JAVA_HOME/bin:$PATH \ No newline at end of file +export HIVE_HOME=/opt/soft/hive +export FLINK_HOME=/opt/soft/flink +export DATAX_HOME=/opt/soft/datax/bin/datax.py + +export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH:$FLINK_HOME/bin:$DATAX_HOME:$PATH diff --git a/dolphinscheduler-ui/install-dolphinscheduler-ui.sh b/dolphinscheduler-ui/install-dolphinscheduler-ui.sh old mode 100644 new mode 100755 diff --git a/e2e/src/test/resources/config/config.properties b/e2e/src/test/resources/config/config.properties index 352d9375be..135af0baa0 100644 --- a/e2e/src/test/resources/config/config.properties +++ b/e2e/src/test/resources/config/config.properties @@ -17,7 +17,7 @@ ############### project ############## # login url -LOGIN_URL=http://127.0.0.1:8888/dolphinscheduler/ +LOGIN_URL=http://127.0.0.1:12345/dolphinscheduler/ #login username USER_NAME=admin #login password diff --git a/script/create-dolphinscheduler.sh b/script/create-dolphinscheduler.sh old mode 100644 new mode 100755 diff --git a/script/dolphinscheduler-daemon.sh b/script/dolphinscheduler-daemon.sh old mode 100644 new mode 100755 index 5e7652199e..a87e62e187 --- a/script/dolphinscheduler-daemon.sh +++ b/script/dolphinscheduler-daemon.sh @@ -54,6 +54,12 @@ fi log=$DOLPHINSCHEDULER_LOG_DIR/dolphinscheduler-$command-$HOSTNAME.out pid=$DOLPHINSCHEDULER_PID_DIR/dolphinscheduler-$command.pid +# print logs to /dev/null in docker +if [ "$DOCKER" = "true" ]; then + echo "start in docker" + log=/dev/null +fi + cd $DOLPHINSCHEDULER_HOME if [ "$command" = "api-server" ]; then diff --git a/script/env/dolphinscheduler_env.sh b/script/env/dolphinscheduler_env.sh old mode 100644 new mode 100755 diff --git a/script/monitor-server.sh b/script/monitor-server.sh old mode 100644 new mode 100755 diff --git a/script/scp-hosts.sh b/script/scp-hosts.sh old mode 100644 new mode 100755 diff --git a/script/start-all.sh b/script/start-all.sh old mode 100644 new mode 100755 diff --git a/script/status-all.sh b/script/status-all.sh old mode 100644 new mode 100755 diff --git a/script/stop-all.sh b/script/stop-all.sh old mode 100644 new mode 100755 diff --git a/script/upgrade-dolphinscheduler.sh b/script/upgrade-dolphinscheduler.sh old mode 100644 new mode 100755 From 60016fbd669c4a19dd0894ad4eda8a34f5b50f76 Mon Sep 17 00:00:00 2001 From: Shiwen Cheng Date: Sun, 14 Feb 2021 21:28:02 +0800 Subject: [PATCH 4/8] [Improvement-4740][UI] Set the default Tenant as default or the first exist tenant in save process (#4742) --- .../src/js/conf/home/pages/dag/_source/udp/udp.vue | 6 +++--- .../home/pages/security/pages/users/_source/createUser.vue | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/udp/udp.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/udp/udp.vue index a040b4be43..50f6016191 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/udp/udp.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/udp/udp.vue @@ -213,10 +213,10 @@ this.timeout = dag.timeout || 0 this.checkedTimeout = this.timeout !== 0 this.$nextTick(() => { - if (dag.tenantId === -1) { - this.tenantId = this.store.state.user.userInfo.tenantId - } else { + if (dag.tenantId > -1) { this.tenantId = dag.tenantId + } else if (this.store.state.user.userInfo.tenantId) { + this.tenantId = this.store.state.user.userInfo.tenantId } }) }, diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue b/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue index 882a54c230..e53128b23e 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue @@ -293,7 +293,7 @@ this.email = this.item.email this.phone = this.item.phone this.userState = this.item.state + '' || '1' - if (this.item.tenantName) { + if (this.item.tenantId) { this.tenantId = this.item.tenantId } this.$nextTick(() => { @@ -311,7 +311,7 @@ this.email = this.item.email this.phone = this.item.phone this.userState = this.state + '' || '1' - if (this.item.tenantName) { + if (this.item.tenantId) { this.tenantId = this.item.tenantId } if (this.queueList.length > 0) { From a6ea04d4a321db5e74751ff076b679830ca21282 Mon Sep 17 00:00:00 2001 From: Shiwen Cheng Date: Sun, 14 Feb 2021 21:31:41 +0800 Subject: [PATCH 5/8] [Fix-4741][UI] Fix task instance log not change (#4744) --- .../home/pages/projects/pages/taskInstance/_source/list.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/taskInstance/_source/list.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/taskInstance/_source/list.vue index 8aa387afc6..b5bb553865 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/taskInstance/_source/list.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/taskInstance/_source/list.vue @@ -76,7 +76,7 @@ :show-close="false" :visible.sync="logDialog" width="auto"> - + From 53598fdd18a31797d8c6ac796a9c49e537d6e4e9 Mon Sep 17 00:00:00 2001 From: Shiwen Cheng Date: Sun, 14 Feb 2021 21:33:45 +0800 Subject: [PATCH 6/8] [Fix-3298][K8s] Fix task log disappear after recreating or upgrading helm release (#4755) * [Improvement][K8s] Alter column host varchar(15) to varchar(135) for long host * [Improvement][K8s] Improve getHost and getAddr in NetUtils * [Improvement][K8s] Replace getHost with getAddr --- .../dolphinscheduler/common/Constants.java | 8 +++ .../common/utils/NetUtils.java | 26 +++++++-- .../common/utils/NetUtilsTest.java | 9 +++ .../common/utils/OSUtilsTest.java | 6 -- .../master/registry/MasterRegistry.java | 2 +- .../runner/ConditionsTaskExecThread.java | 2 +- .../runner/DependentTaskExecThread.java | 2 +- .../master/runner/MasterSchedulerService.java | 4 +- .../processor/TaskExecuteProcessor.java | 2 +- .../worker/registry/WorkerRegistry.java | 2 +- .../executor/NettyExecutorManagerTest.java | 4 +- .../registry/ZookeeperNodeManagerTest.java | 4 +- .../utils/ExecutionContextTestUtils.java | 2 +- .../worker/registry/WorkerRegistryTest.java | 2 +- pom.xml | 1 + sql/dolphinscheduler-postgre.sql | 4 +- sql/dolphinscheduler_mysql.sql | 4 +- .../mysql/dolphinscheduler_ddl.sql | 58 +++++++++++++++++++ .../mysql/dolphinscheduler_dml.sql | 16 +++++ .../postgresql/dolphinscheduler_ddl.sql | 52 +++++++++++++++++ .../postgresql/dolphinscheduler_dml.sql | 16 +++++ 21 files changed, 197 insertions(+), 29 deletions(-) create mode 100644 sql/upgrade/1.3.5_schema/mysql/dolphinscheduler_ddl.sql create mode 100644 sql/upgrade/1.3.5_schema/mysql/dolphinscheduler_dml.sql create mode 100644 sql/upgrade/1.3.5_schema/postgresql/dolphinscheduler_ddl.sql create mode 100644 sql/upgrade/1.3.5_schema/postgresql/dolphinscheduler_dml.sql diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java index 0520520621..45af3b2700 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.common; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.utils.OSUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; import java.util.regex.Pattern; @@ -1042,4 +1043,11 @@ public final class Constants { * pstree, get pud and sub pid */ public static final String PSTREE = "pstree"; + + /** + * docker & kubernetes + */ + public static final boolean DOCKER_MODE = StringUtils.isNotEmpty(System.getenv("DOCKER")); + public static final boolean KUBERNETES_MODE = StringUtils.isNotEmpty(System.getenv("KUBERNETES_SERVICE_HOST")) && StringUtils.isNotEmpty(System.getenv("KUBERNETES_SERVICE_PORT")); + } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java index ddb29730b7..6c761f3d00 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java @@ -49,8 +49,6 @@ public class NetUtils { private static final String NETWORK_PRIORITY_INNER = "inner"; private static final String NETWORK_PRIORITY_OUTER = "outer"; private static final Logger logger = LoggerFactory.getLogger(NetUtils.class); - private static final String ANY_HOST_VALUE = "0.0.0.0"; - private static final String LOCAL_HOST_VALUE = "127.0.0.1"; private static InetAddress LOCAL_ADDRESS = null; private static volatile String HOST_ADDRESS; @@ -58,6 +56,22 @@ public class NetUtils { throw new UnsupportedOperationException("Construct NetUtils"); } + /** + * get addr like host:port + * @return addr + */ + public static String getAddr(String host, int port) { + return String.format("%s:%d", host, port); + } + + /** + * get addr like host:port + * @return addr + */ + public static String getAddr(int port) { + return getAddr(getHost(), port); + } + public static String getHost() { if (HOST_ADDRESS != null) { return HOST_ADDRESS; @@ -65,10 +79,10 @@ public class NetUtils { InetAddress address = getLocalAddress(); if (address != null) { - HOST_ADDRESS = address.getHostAddress(); + HOST_ADDRESS = Constants.KUBERNETES_MODE ? address.getHostName() : address.getHostAddress(); return HOST_ADDRESS; } - return LOCAL_HOST_VALUE; + return Constants.KUBERNETES_MODE ? "localhost" : "127.0.0.1"; } private static InetAddress getLocalAddress() { @@ -153,8 +167,8 @@ public class NetUtils { String name = address.getHostAddress(); return (name != null && IP_PATTERN.matcher(name).matches() - && !ANY_HOST_VALUE.equals(name) - && !LOCAL_HOST_VALUE.equals(name)); + && !address.isAnyLocalAddress() + && !address.isLoopbackAddress()); } /** diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/NetUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/NetUtilsTest.java index 59276295ae..d2461ecfeb 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/NetUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/NetUtilsTest.java @@ -29,6 +29,13 @@ import static org.mockito.Mockito.when; */ public class NetUtilsTest { + @Test + public void testGetAddr() { + assertEquals(NetUtils.getHost() + ":5678", NetUtils.getAddr(5678)); + assertEquals("127.0.0.1:5678", NetUtils.getAddr("127.0.0.1", 5678)); + assertEquals("localhost:1234", NetUtils.getAddr("localhost", 1234)); + } + @Test public void testGetLocalHost() { assertNotNull(NetUtils.getHost()); @@ -45,9 +52,11 @@ public class NetUtilsTest { assertFalse(NetUtils.isValidV4Address(address)); address = mock(InetAddress.class); when(address.getHostAddress()).thenReturn("0.0.0.0"); + when(address.isAnyLocalAddress()).thenReturn(true); assertFalse(NetUtils.isValidV4Address(address)); address = mock(InetAddress.class); when(address.getHostAddress()).thenReturn("127.0.0.1"); + when(address.isLoopbackAddress()).thenReturn(true); assertFalse(NetUtils.isValidV4Address(address)); address = mock(InetAddress.class); when(address.getHostAddress()).thenReturn("1.2.3.4"); diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java index d005010f93..83cacb758b 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java @@ -99,12 +99,6 @@ public class OSUtilsTest { Assert.assertNotEquals(0, processId); } @Test - public void getHost(){ - String host = NetUtils.getHost(); - Assert.assertNotNull(host); - Assert.assertNotEquals("", host); - } - @Test public void checkResource(){ boolean resource = OSUtils.checkResource(100,0); Assert.assertTrue(resource); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistry.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistry.java index 01218e5d8b..37d6e72243 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistry.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistry.java @@ -135,7 +135,7 @@ public class MasterRegistry { */ private String getLocalAddress() { - return NetUtils.getHost() + ":" + masterConfig.getListenPort(); + return NetUtils.getAddr(masterConfig.getListenPort()); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/ConditionsTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/ConditionsTaskExecThread.java index 11598d9ace..d0b314e0bf 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/ConditionsTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/ConditionsTaskExecThread.java @@ -125,7 +125,7 @@ public class ConditionsTaskExecThread extends MasterBaseTaskExecThread { private void initTaskParameters() { this.taskInstance.setLogPath(LogUtils.getTaskLogPath(taskInstance)); - this.taskInstance.setHost(NetUtils.getHost() + Constants.COLON + masterConfig.getListenPort()); + this.taskInstance.setHost(NetUtils.getAddr(masterConfig.getListenPort())); taskInstance.setState(ExecutionStatus.RUNNING_EXECUTION); taskInstance.setStartTime(new Date()); this.processService.saveTaskInstance(taskInstance); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/DependentTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/DependentTaskExecThread.java index 5b56911fd7..9f78e0c532 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/DependentTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/DependentTaskExecThread.java @@ -185,7 +185,7 @@ public class DependentTaskExecThread extends MasterBaseTaskExecThread { private void initTaskParameters() { taskInstance.setLogPath(LogUtils.getTaskLogPath(taskInstance)); - taskInstance.setHost(NetUtils.getHost() + Constants.COLON + masterConfig.getListenPort()); + taskInstance.setHost(NetUtils.getAddr(masterConfig.getListenPort())); taskInstance.setState(ExecutionStatus.RUNNING_EXECUTION); taskInstance.setStartTime(new Date()); processService.updateTaskInstance(taskInstance); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java index b0e0528c3e..b2659bae52 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java @@ -178,7 +178,7 @@ public class MasterSchedulerService extends Thread { } } - private String getLocalAddress(){ - return NetUtils.getHost() + ":" + masterConfig.getListenPort(); + private String getLocalAddress() { + return NetUtils.getAddr(masterConfig.getListenPort()); } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java index 18971dd363..fc8b33a488 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java @@ -139,7 +139,7 @@ public class TaskExecuteProcessor implements NettyRequestProcessor { taskExecutionContext.getProcessInstanceId(), taskExecutionContext.getTaskInstanceId())); - taskExecutionContext.setHost(NetUtils.getHost() + ":" + workerConfig.getListenPort()); + taskExecutionContext.setHost(NetUtils.getAddr(workerConfig.getListenPort())); taskExecutionContext.setStartTime(new Date()); taskExecutionContext.setLogPath(LogUtils.getTaskLogPath(taskExecutionContext)); taskExecutionContext.setCurrentExecutionStatus(ExecutionStatus.RUNNING_EXECUTION); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistry.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistry.java index 904ea3a807..e779d5deb3 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistry.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistry.java @@ -169,7 +169,7 @@ public class WorkerRegistry { * get local address */ private String getLocalAddress() { - return NetUtils.getHost() + COLON + workerConfig.getListenPort(); + return NetUtils.getAddr(workerConfig.getListenPort()); } /** diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManagerTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManagerTest.java index f7d98baed1..f8e6e65521 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManagerTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManagerTest.java @@ -79,7 +79,7 @@ public class NettyExecutorManagerTest { .buildProcessDefinitionRelatedInfo(processDefinition) .create(); ExecutionContext executionContext = new ExecutionContext(context.toCommand(), ExecutorType.WORKER); - executionContext.setHost(Host.of(NetUtils.getHost() + ":" + serverConfig.getListenPort())); + executionContext.setHost(Host.of(NetUtils.getAddr(serverConfig.getListenPort()))); Boolean execute = nettyExecutorManager.execute(executionContext); Assert.assertTrue(execute); nettyRemotingServer.close(); @@ -98,7 +98,7 @@ public class NettyExecutorManagerTest { .buildProcessDefinitionRelatedInfo(processDefinition) .create(); ExecutionContext executionContext = new ExecutionContext(context.toCommand(), ExecutorType.WORKER); - executionContext.setHost(Host.of(NetUtils.getHost() + ":4444")); + executionContext.setHost(Host.of(NetUtils.getAddr(4444))); nettyExecutorManager.execute(executionContext); } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/registry/ZookeeperNodeManagerTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/registry/ZookeeperNodeManagerTest.java index f6eb861069..c7fa8f3b65 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/registry/ZookeeperNodeManagerTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/registry/ZookeeperNodeManagerTest.java @@ -75,7 +75,7 @@ public class ZookeeperNodeManagerTest { Set masterNodes = zookeeperNodeManager.getMasterNodes(); Assert.assertTrue(CollectionUtils.isNotEmpty(masterNodes)); Assert.assertEquals(1, masterNodes.size()); - Assert.assertEquals(NetUtils.getHost() + ":" + masterConfig.getListenPort(), masterNodes.iterator().next()); + Assert.assertEquals(NetUtils.getAddr(masterConfig.getListenPort()), masterNodes.iterator().next()); workerRegistry.unRegistry(); } @@ -105,7 +105,7 @@ public class ZookeeperNodeManagerTest { Set workerNodes = zookeeperNodeManager.getWorkerGroupNodes("default"); Assert.assertTrue(CollectionUtils.isNotEmpty(workerNodes)); Assert.assertEquals(1, workerNodes.size()); - Assert.assertEquals(NetUtils.getHost() + ":" + workerConfig.getListenPort(), workerNodes.iterator().next()); + Assert.assertEquals(NetUtils.getAddr(workerConfig.getListenPort()), workerNodes.iterator().next()); workerRegistry.unRegistry(); } } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ExecutionContextTestUtils.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ExecutionContextTestUtils.java index 3d570b24b9..0894c1b0dd 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ExecutionContextTestUtils.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ExecutionContextTestUtils.java @@ -47,7 +47,7 @@ public class ExecutionContextTestUtils { .buildProcessDefinitionRelatedInfo(processDefinition) .create(); ExecutionContext executionContext = new ExecutionContext(context.toCommand(), ExecutorType.WORKER); - executionContext.setHost(Host.of(NetUtils.getHost() + ":" + port)); + executionContext.setHost(Host.of(NetUtils.getAddr(port))); return executionContext; } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryTest.java index 0490d934e6..a71e48030d 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryTest.java @@ -113,7 +113,7 @@ public class WorkerRegistryTest { int i = 0; for (String workerGroup : workerConfig.getWorkerGroups()) { - String workerZkPath = workerPath + "/" + workerGroup.trim() + "/" + (NetUtils.getHost() + ":" + workerConfig.getListenPort()); + String workerZkPath = workerPath + "/" + workerGroup.trim() + "/" + (NetUtils.getAddr(workerConfig.getListenPort())); String heartbeat = zookeeperRegistryCenter.getZookeeperCachedOperator().get(workerZkPath); if (0 == i) { Assert.assertTrue(workerZkPath.startsWith("/dolphinscheduler/nodes/worker/test/")); diff --git a/pom.xml b/pom.xml index 6d7446f8d1..48ab95aa0a 100644 --- a/pom.xml +++ b/pom.xml @@ -839,6 +839,7 @@ **/common/utils/IpUtilsTest.java **/common/utils/JSONUtilsTest.java **/common/utils/LoggerUtilsTest.java + **/common/utils/NetUtilsTest.java **/common/utils/OSUtilsTest.java **/common/utils/ParameterUtilsTest.java **/common/utils/TimePlaceholderUtilsTest.java diff --git a/sql/dolphinscheduler-postgre.sql b/sql/dolphinscheduler-postgre.sql index b8f638ff99..b242205d3f 100644 --- a/sql/dolphinscheduler-postgre.sql +++ b/sql/dolphinscheduler-postgre.sql @@ -348,7 +348,7 @@ CREATE TABLE t_ds_process_instance ( start_time timestamp DEFAULT NULL , end_time timestamp DEFAULT NULL , run_times int DEFAULT NULL , - host varchar(45) DEFAULT NULL , + host varchar(135) DEFAULT NULL , command_type int DEFAULT NULL , command_param text , task_depend_type int DEFAULT NULL , @@ -562,7 +562,7 @@ CREATE TABLE t_ds_task_instance ( submit_time timestamp DEFAULT NULL , start_time timestamp DEFAULT NULL , end_time timestamp DEFAULT NULL , - host varchar(45) DEFAULT NULL , + host varchar(135) DEFAULT NULL , execute_path varchar(200) DEFAULT NULL , log_path varchar(200) DEFAULT NULL , alert_flag int DEFAULT NULL , diff --git a/sql/dolphinscheduler_mysql.sql b/sql/dolphinscheduler_mysql.sql index 890e00344b..bb6588dbec 100644 --- a/sql/dolphinscheduler_mysql.sql +++ b/sql/dolphinscheduler_mysql.sql @@ -458,7 +458,7 @@ CREATE TABLE `t_ds_process_instance` ( `start_time` datetime DEFAULT NULL COMMENT 'process instance start time', `end_time` datetime DEFAULT NULL COMMENT 'process instance end time', `run_times` int(11) DEFAULT NULL COMMENT 'process instance run times', - `host` varchar(45) DEFAULT NULL COMMENT 'process instance host', + `host` varchar(135) DEFAULT NULL COMMENT 'process instance host', `command_type` tinyint(4) DEFAULT NULL COMMENT 'command type', `command_param` text COMMENT 'json command parameters', `task_depend_type` tinyint(4) DEFAULT NULL COMMENT 'task depend type. 0: only current node,1:before the node,2:later nodes', @@ -697,7 +697,7 @@ CREATE TABLE `t_ds_task_instance` ( `submit_time` datetime DEFAULT NULL COMMENT 'task submit time', `start_time` datetime DEFAULT NULL COMMENT 'task start time', `end_time` datetime DEFAULT NULL COMMENT 'task end time', - `host` varchar(45) DEFAULT NULL COMMENT 'host of task running on', + `host` varchar(135) DEFAULT NULL COMMENT 'host of task running on', `execute_path` varchar(200) DEFAULT NULL COMMENT 'task execute path in the host', `log_path` varchar(200) DEFAULT NULL COMMENT 'task log path', `alert_flag` tinyint(4) DEFAULT NULL COMMENT 'whether alert', diff --git a/sql/upgrade/1.3.5_schema/mysql/dolphinscheduler_ddl.sql b/sql/upgrade/1.3.5_schema/mysql/dolphinscheduler_ddl.sql new file mode 100644 index 0000000000..5635073390 --- /dev/null +++ b/sql/upgrade/1.3.5_schema/mysql/dolphinscheduler_ddl.sql @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +SET sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY','')); + +-- uc_dolphin_T_t_ds_process_instance_R_host +drop PROCEDURE if EXISTS uc_dolphin_T_t_ds_process_instance_R_host; +delimiter d// +CREATE PROCEDURE uc_dolphin_T_t_ds_process_instance_R_host() +BEGIN + IF EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_NAME='t_ds_process_instance' + AND TABLE_SCHEMA=(SELECT DATABASE()) + AND COLUMN_NAME ='host') + THEN + ALTER TABLE t_ds_process_instance MODIFY COLUMN `host` varchar(135); + END IF; +END; + +d// + +delimiter ; +CALL uc_dolphin_T_t_ds_process_instance_R_host; +DROP PROCEDURE uc_dolphin_T_t_ds_process_instance_R_host; + +-- uc_dolphin_T_t_ds_task_instance_R_host +drop PROCEDURE if EXISTS uc_dolphin_T_t_ds_task_instance_R_host; +delimiter d// +CREATE PROCEDURE uc_dolphin_T_t_ds_task_instance_R_host() +BEGIN + IF EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_NAME='t_ds_task_instance' + AND TABLE_SCHEMA=(SELECT DATABASE()) + AND COLUMN_NAME ='host') + THEN + ALTER TABLE t_ds_task_instance MODIFY COLUMN `host` varchar(135); + END IF; +END; + +d// + +delimiter ; +CALL uc_dolphin_T_t_ds_task_instance_R_host; +DROP PROCEDURE uc_dolphin_T_t_ds_task_instance_R_host; diff --git a/sql/upgrade/1.3.5_schema/mysql/dolphinscheduler_dml.sql b/sql/upgrade/1.3.5_schema/mysql/dolphinscheduler_dml.sql new file mode 100644 index 0000000000..38964cc551 --- /dev/null +++ b/sql/upgrade/1.3.5_schema/mysql/dolphinscheduler_dml.sql @@ -0,0 +1,16 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ \ No newline at end of file diff --git a/sql/upgrade/1.3.5_schema/postgresql/dolphinscheduler_ddl.sql b/sql/upgrade/1.3.5_schema/postgresql/dolphinscheduler_ddl.sql new file mode 100644 index 0000000000..6dd15f7b07 --- /dev/null +++ b/sql/upgrade/1.3.5_schema/postgresql/dolphinscheduler_ddl.sql @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +-- uc_dolphin_T_t_ds_process_instance_A_host +delimiter d// +CREATE OR REPLACE FUNCTION uc_dolphin_T_t_ds_process_instance_A_host() RETURNS void AS $$ +BEGIN + IF EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_NAME='t_ds_process_instance' + AND COLUMN_NAME ='host') + THEN + ALTER TABLE t_ds_process_instance ALTER COLUMN host type varchar(135); + END IF; +END; +$$ LANGUAGE plpgsql; +d// + +delimiter ; +SELECT uc_dolphin_T_t_ds_process_instance_A_host(); +DROP FUNCTION IF EXISTS uc_dolphin_T_t_ds_process_instance_A_host(); + +-- uc_dolphin_T_t_ds_task_instance_A_host +delimiter d// +CREATE OR REPLACE FUNCTION uc_dolphin_T_t_ds_task_instance_A_host() RETURNS void AS $$ +BEGIN + IF EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_NAME='t_ds_task_instance' + AND COLUMN_NAME ='host') + THEN + ALTER TABLE t_ds_task_instance ALTER COLUMN host type varchar(135); + END IF; +END; +$$ LANGUAGE plpgsql; +d// + +delimiter ; +SELECT uc_dolphin_T_t_ds_task_instance_A_host(); +DROP FUNCTION IF EXISTS uc_dolphin_T_t_ds_task_instance_A_host(); diff --git a/sql/upgrade/1.3.5_schema/postgresql/dolphinscheduler_dml.sql b/sql/upgrade/1.3.5_schema/postgresql/dolphinscheduler_dml.sql new file mode 100644 index 0000000000..38964cc551 --- /dev/null +++ b/sql/upgrade/1.3.5_schema/postgresql/dolphinscheduler_dml.sql @@ -0,0 +1,16 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ \ No newline at end of file From 93514ddf316475caf3c8498b48c31c807c29c6f1 Mon Sep 17 00:00:00 2001 From: Shiwen Cheng Date: Sun, 14 Feb 2021 21:35:23 +0800 Subject: [PATCH 7/8] [Improvement][K8s] Improve K8s for helm (#4746) --- docker/kubernetes/dolphinscheduler/Chart.yaml | 10 +- docker/kubernetes/dolphinscheduler/README.md | 37 +-- .../dolphinscheduler/requirements.yaml | 25 -- .../dolphinscheduler/requirements.yaml~HEAD | 25 -- .../dolphinscheduler/requirements.yaml~HEAD_0 | 25 -- .../dolphinscheduler/requirements.yaml~dev | 25 -- .../dolphinscheduler/requirements.yaml~dev_0 | 25 -- .../dolphinscheduler/templates/NOTES.txt | 6 +- .../dolphinscheduler/templates/_helpers.tpl | 4 +- .../configmap-dolphinscheduler-alert.yaml | 1 + ...ml => configmap-dolphinscheduler-api.yaml} | 21 +- .../configmap-dolphinscheduler-common.yaml | 8 +- .../configmap-dolphinscheduler-master.yaml | 2 +- .../configmap-dolphinscheduler-worker.yaml | 7 +- .../deployment-dolphinscheduler-alert.yaml | 68 +--- .../deployment-dolphinscheduler-api.yaml | 48 +-- .../deployment-dolphinscheduler-frontend.yaml | 119 ------- .../dolphinscheduler/templates/ingress.yaml | 2 +- .../templates/pvc-dolphinscheduler-alert.yaml | 2 +- .../templates/pvc-dolphinscheduler-api.yaml | 2 +- ...ntend.yaml => secret-external-fs-s3a.yaml} | 20 +- .../statefulset-dolphinscheduler-master.yaml | 96 +----- .../statefulset-dolphinscheduler-worker.yaml | 184 ++++------- .../svc-dolphinscheduler-worker-headless.yaml | 4 +- .../kubernetes/dolphinscheduler/values.yaml | 310 ++++++++---------- 25 files changed, 269 insertions(+), 807 deletions(-) delete mode 100644 docker/kubernetes/dolphinscheduler/requirements.yaml delete mode 100644 docker/kubernetes/dolphinscheduler/requirements.yaml~HEAD delete mode 100644 docker/kubernetes/dolphinscheduler/requirements.yaml~HEAD_0 delete mode 100644 docker/kubernetes/dolphinscheduler/requirements.yaml~dev delete mode 100644 docker/kubernetes/dolphinscheduler/requirements.yaml~dev_0 rename docker/kubernetes/dolphinscheduler/templates/{svc-dolphinscheduler-frontend.yaml => configmap-dolphinscheduler-api.yaml} (69%) delete mode 100644 docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-frontend.yaml rename docker/kubernetes/dolphinscheduler/templates/{pvc-dolphinscheduler-frontend.yaml => secret-external-fs-s3a.yaml} (66%) diff --git a/docker/kubernetes/dolphinscheduler/Chart.yaml b/docker/kubernetes/dolphinscheduler/Chart.yaml index 9d640869dd..1e0ca755b9 100644 --- a/docker/kubernetes/dolphinscheduler/Chart.yaml +++ b/docker/kubernetes/dolphinscheduler/Chart.yaml @@ -22,7 +22,7 @@ home: https://dolphinscheduler.apache.org icon: https://dolphinscheduler.apache.org/img/hlogo_colorful.svg keywords: - dolphinscheduler -- Scheduler +- scheduler # A chart can be either an 'application' or a 'library' chart. # # Application charts are a collection of templates that can be packaged into versioned archives @@ -35,18 +35,18 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 1.0.0 +version: 1.2.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 1.3.0 +appVersion: 1.4.0 dependencies: - name: postgresql - version: 8.x.x + version: 10.x.x repository: https://charts.bitnami.com/bitnami condition: postgresql.enabled - name: zookeeper - version: 5.x.x + version: 6.x.x repository: https://charts.bitnami.com/bitnami condition: zookeeper.enabled diff --git a/docker/kubernetes/dolphinscheduler/README.md b/docker/kubernetes/dolphinscheduler/README.md index 0691b1ed2e..318c3a9132 100644 --- a/docker/kubernetes/dolphinscheduler/README.md +++ b/docker/kubernetes/dolphinscheduler/README.md @@ -7,19 +7,20 @@ This chart bootstraps a [Dolphin Scheduler](https://dolphinscheduler.apache.org) ## Prerequisites -- Kubernetes 1.10+ +- Helm 3.1.0+ +- Kubernetes 1.12+ - PV provisioner support in the underlying infrastructure ## Installing the Chart -To install the chart with the release name `my-release`: +To install the chart with the release name `dolphinscheduler`: ```bash $ git clone https://github.com/apache/incubator-dolphinscheduler.git -$ cd incubator-dolphinscheduler/kubernetes/dolphinscheduler +$ cd incubator-dolphinscheduler/docker/kubernetes/dolphinscheduler $ helm repo add bitnami https://charts.bitnami.com/bitnami $ helm dependency update . -$ helm install --name dolphinscheduler . +$ helm install dolphinscheduler . ``` These commands deploy Dolphin Scheduler on the Kubernetes cluster in the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation. @@ -30,7 +31,7 @@ These commands deploy Dolphin Scheduler on the Kubernetes cluster in the default To uninstall/delete the `dolphinscheduler` deployment: ```bash -$ helm delete --purge dolphinscheduler +$ helm uninstall dolphinscheduler ``` The command removes all the Kubernetes components associated with the chart and deletes the release. @@ -220,32 +221,6 @@ The following tables lists the configurable parameters of the Dolphins Scheduler | `api.persistentVolumeClaim.storageClassName` | `api` logs data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` | | `api.persistentVolumeClaim.storage` | `PersistentVolumeClaim` Size | `20Gi` | | | | | -| `frontend.strategy.type` | Type of deployment. Can be "Recreate" or "RollingUpdate" | `RollingUpdate` | -| `frontend.strategy.rollingUpdate.maxSurge` | The maximum number of pods that can be scheduled above the desired number of pods | `25%` | -| `frontend.strategy.rollingUpdate.maxUnavailable` | The maximum number of pods that can be unavailable during the update | `25%` | -| `frontend.replicas` | Replicas is the desired number of replicas of the given Template | `1` | -| `frontend.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` | -| `frontend.tolerations` | If specified, the pod's tolerations | `{}` | -| `frontend.affinity` | If specified, the pod's scheduling constraints | `{}` | -| `frontend.resources` | The `resource` limit and request config for frontend server. | `{}` | -| `frontend.annotations` | The `annotations` for frontend server. | `{}` | -| `frontend.livenessProbe.enabled` | Turn on and off liveness probe | `true` | -| `frontend.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `30` | -| `frontend.livenessProbe.periodSeconds` | How often to perform the probe | `30` | -| `frontend.livenessProbe.timeoutSeconds` | When the probe times out | `5` | -| `frontend.livenessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` | -| `frontend.livenessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` | -| `frontend.readinessProbe.enabled` | Turn on and off readiness probe | `true` | -| `frontend.readinessProbe.initialDelaySeconds` | Delay before readiness probe is initiated | `30` | -| `frontend.readinessProbe.periodSeconds` | How often to perform the probe | `30` | -| `frontend.readinessProbe.timeoutSeconds` | When the probe times out | `5` | -| `frontend.readinessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` | -| `frontend.readinessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` | -| `frontend.persistentVolumeClaim.enabled` | Set `frontend.persistentVolumeClaim.enabled` to `true` to mount a new volume for `frontend` | `false` | -| `frontend.persistentVolumeClaim.accessModes` | `PersistentVolumeClaim` Access Modes | `[ReadWriteOnce]` | -| `frontend.persistentVolumeClaim.storageClassName` | `frontend` logs data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` | -| `frontend.persistentVolumeClaim.storage` | `PersistentVolumeClaim` Size | `20Gi` | -| | | | | `ingress.enabled` | Enable ingress | `false` | | `ingress.host` | Ingress host | `dolphinscheduler.org` | | `ingress.path` | Ingress path | `/` | diff --git a/docker/kubernetes/dolphinscheduler/requirements.yaml b/docker/kubernetes/dolphinscheduler/requirements.yaml deleted file mode 100644 index e219975995..0000000000 --- a/docker/kubernetes/dolphinscheduler/requirements.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -dependencies: -- name: postgresql - version: 8.x.x - repository: https://charts.bitnami.com/bitnami - condition: postgresql.enabled -- name: zookeeper - version: 5.x.x - repository: https://charts.bitnami.com/bitnami - condition: redis.enabled \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/requirements.yaml~HEAD b/docker/kubernetes/dolphinscheduler/requirements.yaml~HEAD deleted file mode 100644 index e219975995..0000000000 --- a/docker/kubernetes/dolphinscheduler/requirements.yaml~HEAD +++ /dev/null @@ -1,25 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -dependencies: -- name: postgresql - version: 8.x.x - repository: https://charts.bitnami.com/bitnami - condition: postgresql.enabled -- name: zookeeper - version: 5.x.x - repository: https://charts.bitnami.com/bitnami - condition: redis.enabled \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/requirements.yaml~HEAD_0 b/docker/kubernetes/dolphinscheduler/requirements.yaml~HEAD_0 deleted file mode 100644 index e219975995..0000000000 --- a/docker/kubernetes/dolphinscheduler/requirements.yaml~HEAD_0 +++ /dev/null @@ -1,25 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -dependencies: -- name: postgresql - version: 8.x.x - repository: https://charts.bitnami.com/bitnami - condition: postgresql.enabled -- name: zookeeper - version: 5.x.x - repository: https://charts.bitnami.com/bitnami - condition: redis.enabled \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/requirements.yaml~dev b/docker/kubernetes/dolphinscheduler/requirements.yaml~dev deleted file mode 100644 index e219975995..0000000000 --- a/docker/kubernetes/dolphinscheduler/requirements.yaml~dev +++ /dev/null @@ -1,25 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -dependencies: -- name: postgresql - version: 8.x.x - repository: https://charts.bitnami.com/bitnami - condition: postgresql.enabled -- name: zookeeper - version: 5.x.x - repository: https://charts.bitnami.com/bitnami - condition: redis.enabled \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/requirements.yaml~dev_0 b/docker/kubernetes/dolphinscheduler/requirements.yaml~dev_0 deleted file mode 100644 index e219975995..0000000000 --- a/docker/kubernetes/dolphinscheduler/requirements.yaml~dev_0 +++ /dev/null @@ -1,25 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -dependencies: -- name: postgresql - version: 8.x.x - repository: https://charts.bitnami.com/bitnami - condition: postgresql.enabled -- name: zookeeper - version: 5.x.x - repository: https://charts.bitnami.com/bitnami - condition: redis.enabled \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/templates/NOTES.txt b/docker/kubernetes/dolphinscheduler/templates/NOTES.txt index 256c53ca57..8afe766d27 100644 --- a/docker/kubernetes/dolphinscheduler/templates/NOTES.txt +++ b/docker/kubernetes/dolphinscheduler/templates/NOTES.txt @@ -15,9 +15,9 @@ # limitations under the License. # -** Please be patient while the chart is being deployed ** +** Please be patient while the chart Dolphinscheduler {{ .Chart.AppVersion }} is being deployed ** -1. Get the Dolphinscheduler URL by running: +Get the Dolphinscheduler URL by running: {{- if .Values.ingress.enabled }} @@ -26,6 +26,6 @@ {{- else }} - kubectl port-forward --namespace {{ .Release.Namespace }} svc/{{ template "dolphinscheduler.fullname" . }}-frontend 8888:8888 + kubectl port-forward --namespace {{ .Release.Namespace }} svc/{{ template "dolphinscheduler.fullname" . }}-api 12345:12345 {{- end }} diff --git a/docker/kubernetes/dolphinscheduler/templates/_helpers.tpl b/docker/kubernetes/dolphinscheduler/templates/_helpers.tpl index 1f121dfea8..ccfc746645 100644 --- a/docker/kubernetes/dolphinscheduler/templates/_helpers.tpl +++ b/docker/kubernetes/dolphinscheduler/templates/_helpers.tpl @@ -135,7 +135,7 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this Create a default dolphinscheduler worker base dir. We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). */}} -{{- define "dolphinscheduler.worker.base.dir" -}} -{{- $name := default "/tmp/dolphinscheduler" .Values.worker.configmap.DOLPHINSCHEDULER_DATA_BASEDIR_PATH -}} +{{- define "dolphinscheduler.data.basedir.path" -}} +{{- $name := default "/tmp/dolphinscheduler" .Values.common.configmap.DOLPHINSCHEDULER_DATA_BASEDIR_PATH -}} {{- printf "%s" $name | trunc 63 | trimSuffix "/" -}} {{- end -}} \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml index 2c5b76c3f0..2c7dd67c57 100644 --- a/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml @@ -24,6 +24,7 @@ metadata: app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} data: + DOLPHINSCHEDULER_OPTS: {{ .Values.alert.configmap.DOLPHINSCHEDULER_OPTS | quote }} ALERT_PLUGIN_DIR: {{ .Values.alert.configmap.ALERT_PLUGIN_DIR | quote }} XLS_FILE_PATH: {{ .Values.alert.configmap.XLS_FILE_PATH | quote }} MAIL_SERVER_HOST: {{ .Values.alert.configmap.MAIL_SERVER_HOST | quote }} diff --git a/docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-frontend.yaml b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-api.yaml similarity index 69% rename from docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-frontend.yaml rename to docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-api.yaml index 60d0d6e7b5..3845f1f67c 100644 --- a/docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-frontend.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-api.yaml @@ -14,22 +14,15 @@ # See the License for the specific language governing permissions and # limitations under the License. # +{{- if .Values.api.configmap }} apiVersion: v1 -kind: Service +kind: ConfigMap metadata: - name: {{ include "dolphinscheduler.fullname" . }}-frontend + name: {{ include "dolphinscheduler.fullname" . }}-api labels: - app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-frontend + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} -spec: - ports: - - port: 8888 - targetPort: tcp-port - protocol: TCP - name: tcp-port - selector: - app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-frontend - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/component: frontend \ No newline at end of file +data: + DOLPHINSCHEDULER_OPTS: {{ .Values.api.configmap.DOLPHINSCHEDULER_OPTS | quote }} +{{- end }} \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-common.yaml b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-common.yaml index 931f6d48e4..1fcb5b5992 100644 --- a/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-common.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-common.yaml @@ -24,12 +24,14 @@ metadata: app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} data: - DOLPHINSCHEDULER_ENV_PATH: {{ .Values.common.configmap.DOLPHINSCHEDULER_ENV_PATH | quote }} - DOLPHINSCHEDULER_DATA_BASEDIR_PATH: {{ .Values.common.configmap.DOLPHINSCHEDULER_DATA_BASEDIR_PATH | quote }} + DOLPHINSCHEDULER_ENV: |- + {{- range .Values.common.configmap.DOLPHINSCHEDULER_ENV }} + {{ . }} + {{- end }} + DOLPHINSCHEDULER_DATA_BASEDIR_PATH: {{ include "dolphinscheduler.data.basedir.path" . | quote }} RESOURCE_STORAGE_TYPE: {{ .Values.common.configmap.RESOURCE_STORAGE_TYPE | quote }} RESOURCE_UPLOAD_PATH: {{ .Values.common.configmap.RESOURCE_UPLOAD_PATH | quote }} FS_DEFAULT_FS: {{ .Values.common.configmap.FS_DEFAULT_FS | quote }} FS_S3A_ENDPOINT: {{ .Values.common.configmap.FS_S3A_ENDPOINT | quote }} FS_S3A_ACCESS_KEY: {{ .Values.common.configmap.FS_S3A_ACCESS_KEY | quote }} - FS_S3A_SECRET_KEY: {{ .Values.common.configmap.FS_S3A_SECRET_KEY | quote }} {{- end }} \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml index da82d639cb..7c1be7717e 100644 --- a/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml @@ -24,6 +24,7 @@ metadata: app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} data: + DOLPHINSCHEDULER_OPTS: {{ .Values.master.configmap.DOLPHINSCHEDULER_OPTS | quote }} MASTER_EXEC_THREADS: {{ .Values.master.configmap.MASTER_EXEC_THREADS | quote }} MASTER_EXEC_TASK_NUM: {{ .Values.master.configmap.MASTER_EXEC_TASK_NUM | quote }} MASTER_HEARTBEAT_INTERVAL: {{ .Values.master.configmap.MASTER_HEARTBEAT_INTERVAL | quote }} @@ -32,5 +33,4 @@ data: MASTER_MAX_CPULOAD_AVG: {{ .Values.master.configmap.MASTER_MAX_CPULOAD_AVG | quote }} MASTER_RESERVED_MEMORY: {{ .Values.master.configmap.MASTER_RESERVED_MEMORY | quote }} MASTER_LISTEN_PORT: {{ .Values.master.configmap.MASTER_LISTEN_PORT | quote }} - DOLPHINSCHEDULER_DATA_BASEDIR_PATH: {{ include "dolphinscheduler.worker.base.dir" . | quote }} {{- end }} \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml index 569341c225..78b6ed1036 100644 --- a/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml @@ -24,17 +24,12 @@ metadata: app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} data: + DOLPHINSCHEDULER_OPTS: {{ .Values.worker.configmap.DOLPHINSCHEDULER_OPTS | quote }} WORKER_EXEC_THREADS: {{ .Values.worker.configmap.WORKER_EXEC_THREADS | quote }} WORKER_HEARTBEAT_INTERVAL: {{ .Values.worker.configmap.WORKER_HEARTBEAT_INTERVAL | quote }} - WORKER_FETCH_TASK_NUM: {{ .Values.worker.configmap.WORKER_FETCH_TASK_NUM | quote }} WORKER_MAX_CPULOAD_AVG: {{ .Values.worker.configmap.WORKER_MAX_CPULOAD_AVG | quote }} WORKER_RESERVED_MEMORY: {{ .Values.worker.configmap.WORKER_RESERVED_MEMORY | quote }} WORKER_LISTEN_PORT: {{ .Values.worker.configmap.WORKER_LISTEN_PORT | quote }} WORKER_GROUP: {{ .Values.worker.configmap.WORKER_GROUP | quote }} WORKER_WEIGHT: {{ .Values.worker.configmap.WORKER_WEIGHT | quote }} - DOLPHINSCHEDULER_DATA_BASEDIR_PATH: {{ include "dolphinscheduler.worker.base.dir" . | quote }} - dolphinscheduler_env.sh: |- - {{- range .Values.worker.configmap.DOLPHINSCHEDULER_ENV }} - {{ . }} - {{- end }} {{- end }} \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml index fb9a23786b..e4a2b21ffa 100644 --- a/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml @@ -57,35 +57,6 @@ spec: {{- if .Values.alert.tolerations }} tolerations: {{- toYaml . | nindent 8 }} {{- end }} - initContainers: - - name: init-database - image: busybox:1.31.0 - command: - - /bin/sh - - -ec - - | - while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do - counter=$((counter+1)) - if [ $counter == 5 ]; then - echo "Error: Couldn't connect to database." - exit 1 - fi - echo "Trying to connect to database at ${DATABASE_HOST}:${DATABASE_PORT}. Attempt $counter." - sleep 60 - done - env: - - name: DATABASE_HOST - {{- if .Values.postgresql.enabled }} - value: {{ template "dolphinscheduler.postgresql.fullname" . }} - {{- else }} - value: {{ .Values.externalDatabase.host | quote }} - {{- end }} - - name: DATABASE_PORT - {{- if .Values.postgresql.enabled }} - value: "5432" - {{- else }} - value: {{ .Values.externalDatabase.port | quote }} - {{- end }} {{- if .Values.image.pullSecrets }} imagePullSecrets: - name: {{ include "dolphinscheduler.image.pullSecrets" . }} @@ -93,14 +64,17 @@ spec: containers: - name: {{ include "dolphinscheduler.fullname" . }}-alert image: {{ include "dolphinscheduler.image.repository" . | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} args: - "alert-server" - imagePullPolicy: {{ .Values.image.pullPolicy }} env: - - name: DOLPHINSCHEDULER_OPTS - value: {{ default "" .Values.alert.jvmOptions }} - name: TZ value: {{ .Values.timezone }} + - name: DOLPHINSCHEDULER_OPTS + valueFrom: + configMapKeyRef: + key: DOLPHINSCHEDULER_OPTS + name: {{ include "dolphinscheduler.fullname" . }}-alert - name: ALERT_PLUGIN_DIR valueFrom: configMapKeyRef: @@ -228,36 +202,6 @@ spec: {{- else }} value: {{ .Values.externalDatabase.params | quote }} {{- end }} - - name: RESOURCE_STORAGE_TYPE - valueFrom: - configMapKeyRef: - key: RESOURCE_STORAGE_TYPE - name: {{ include "dolphinscheduler.fullname" . }}-common - - name: RESOURCE_UPLOAD_PATH - valueFrom: - configMapKeyRef: - key: RESOURCE_UPLOAD_PATH - name: {{ include "dolphinscheduler.fullname" . }}-common - - name: FS_DEFAULT_FS - valueFrom: - configMapKeyRef: - key: FS_DEFAULT_FS - name: {{ include "dolphinscheduler.fullname" . }}-common - - name: FS_S3A_ENDPOINT - valueFrom: - configMapKeyRef: - key: FS_S3A_ENDPOINT - name: {{ include "dolphinscheduler.fullname" . }}-common - - name: FS_S3A_ACCESS_KEY - valueFrom: - configMapKeyRef: - key: FS_S3A_ACCESS_KEY - name: {{ include "dolphinscheduler.fullname" . }}-common - - name: FS_S3A_SECRET_KEY - valueFrom: - configMapKeyRef: - key: FS_S3A_SECRET_KEY - name: {{ include "dolphinscheduler.fullname" . }}-common {{- if .Values.alert.resources }} resources: limits: diff --git a/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml index 92c2c72398..563f9ebbf9 100644 --- a/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml @@ -57,35 +57,6 @@ spec: {{- if .Values.api.tolerations }} tolerations: {{- toYaml . | nindent 8 }} {{- end }} - initContainers: - - name: init-database - image: busybox:1.31.0 - command: - - /bin/sh - - -ec - - | - while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do - counter=$((counter+1)) - if [ $counter == 5 ]; then - echo "Error: Couldn't connect to database." - exit 1 - fi - echo "Trying to connect to database at ${DATABASE_HOST}:${DATABASE_PORT}. Attempt $counter." - sleep 60 - done - env: - - name: DATABASE_HOST - {{- if .Values.postgresql.enabled }} - value: {{ template "dolphinscheduler.postgresql.fullname" . }} - {{- else }} - value: {{ .Values.externalDatabase.host | quote }} - {{- end }} - - name: DATABASE_PORT - {{- if .Values.postgresql.enabled }} - value: "5432" - {{- else }} - value: {{ .Values.externalDatabase.port | quote }} - {{- end }} {{- if .Values.image.pullSecrets }} imagePullSecrets: - name: {{ include "dolphinscheduler.image.pullSecrets" . }} @@ -93,17 +64,20 @@ spec: containers: - name: {{ include "dolphinscheduler.fullname" . }}-api image: {{ include "dolphinscheduler.image.repository" . | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} args: - "api-server" ports: - containerPort: 12345 name: tcp-port - imagePullPolicy: {{ .Values.image.pullPolicy }} env: - - name: DOLPHINSCHEDULER_OPTS - value: {{ default "" .Values.api.jvmOptions }} - name: TZ value: {{ .Values.timezone }} + - name: DOLPHINSCHEDULER_OPTS + valueFrom: + configMapKeyRef: + key: DOLPHINSCHEDULER_OPTS + name: {{ include "dolphinscheduler.fullname" . }}-api - name: DATABASE_TYPE {{- if .Values.postgresql.enabled }} value: "postgresql" @@ -164,7 +138,7 @@ spec: {{- end }} - name: ZOOKEEPER_ROOT {{- if .Values.zookeeper.enabled }} - value: "/dolphinscheduler" + value: {{ .Values.zookeeper.zookeeperRoot }} {{- else }} value: {{ .Values.externalZookeeper.zookeeperRoot }} {{- end }} @@ -183,6 +157,7 @@ spec: configMapKeyRef: key: FS_DEFAULT_FS name: {{ include "dolphinscheduler.fullname" . }}-common + {{- if eq .Values.common.configmap.RESOURCE_STORAGE_TYPE "S3" }} - name: FS_S3A_ENDPOINT valueFrom: configMapKeyRef: @@ -195,9 +170,10 @@ spec: name: {{ include "dolphinscheduler.fullname" . }}-common - name: FS_S3A_SECRET_KEY valueFrom: - configMapKeyRef: - key: FS_S3A_SECRET_KEY - name: {{ include "dolphinscheduler.fullname" . }}-common + secretKeyRef: + key: fs-s3a-secret-key + name: {{ printf "%s-%s" .Release.Name "fs-s3a" }} + {{- end }} {{- if .Values.api.resources }} resources: limits: diff --git a/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-frontend.yaml b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-frontend.yaml deleted file mode 100644 index 04b9408d9f..0000000000 --- a/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-frontend.yaml +++ /dev/null @@ -1,119 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "dolphinscheduler.fullname" . }}-frontend - labels: - app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-frontend - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/component: frontend -spec: - replicas: {{ .Values.frontend.replicas }} - selector: - matchLabels: - app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-frontend - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/component: frontend - strategy: - type: {{ .Values.frontend.strategy.type | quote }} - rollingUpdate: - maxSurge: {{ .Values.frontend.strategy.rollingUpdate.maxSurge | quote }} - maxUnavailable: {{ .Values.frontend.strategy.rollingUpdate.maxUnavailable | quote }} - template: - metadata: - labels: - app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-frontend - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/component: frontend - {{- if .Values.alert.annotations }} - annotations: - {{- toYaml .Values.alert.annotations | nindent 8 }} - {{- end }} - spec: - {{- if .Values.frontend.affinity }} - affinity: {{- toYaml .Values.frontend.affinity | nindent 8 }} - {{- end }} - {{- if .Values.frontend.nodeSelector }} - nodeSelector: {{- toYaml .Values.frontend.nodeSelector | nindent 8 }} - {{- end }} - {{- if .Values.frontend.tolerations }} - tolerations: {{- toYaml . | nindent 8 }} - {{- end }} - {{- if .Values.image.pullSecrets }} - imagePullSecrets: - - name: {{ include "dolphinscheduler.image.pullSecrets" . }} - {{- end }} - containers: - - name: {{ include "dolphinscheduler.fullname" . }}-frontend - image: {{ include "dolphinscheduler.image.repository" . | quote }} - args: - - "frontend" - ports: - - containerPort: 8888 - name: tcp-port - imagePullPolicy: {{ .Values.image.pullPolicy }} - env: - - name: TZ - value: {{ .Values.timezone }} - - name: FRONTEND_API_SERVER_HOST - value: '{{ include "dolphinscheduler.fullname" . }}-api' - - name: FRONTEND_API_SERVER_PORT - value: "12345" - {{- if .Values.frontend.resources }} - resources: - limits: - memory: {{ .Values.frontend.resources.limits.memory | quote }} - cpu: {{ .Values.frontend.resources.limits.cpu | quote }} - requests: - memory: {{ .Values.frontend.resources.requests.memory | quote }} - cpu: {{ .Values.frontend.resources.requests.cpu | quote }} - {{- end }} - {{- if .Values.frontend.livenessProbe.enabled }} - livenessProbe: - tcpSocket: - port: 8888 - initialDelaySeconds: {{ .Values.frontend.livenessProbe.initialDelaySeconds }} - periodSeconds: {{ .Values.frontend.livenessProbe.periodSeconds }} - timeoutSeconds: {{ .Values.frontend.livenessProbe.timeoutSeconds }} - successThreshold: {{ .Values.frontend.livenessProbe.successThreshold }} - failureThreshold: {{ .Values.frontend.livenessProbe.failureThreshold }} - {{- end }} - {{- if .Values.frontend.readinessProbe.enabled }} - readinessProbe: - tcpSocket: - port: 8888 - initialDelaySeconds: {{ .Values.frontend.readinessProbe.initialDelaySeconds }} - periodSeconds: {{ .Values.frontend.readinessProbe.periodSeconds }} - timeoutSeconds: {{ .Values.frontend.readinessProbe.timeoutSeconds }} - successThreshold: {{ .Values.frontend.readinessProbe.successThreshold }} - failureThreshold: {{ .Values.frontend.readinessProbe.failureThreshold }} - {{- end }} - volumeMounts: - - mountPath: "/var/log/nginx" - name: {{ include "dolphinscheduler.fullname" . }}-frontend - volumes: - - name: {{ include "dolphinscheduler.fullname" . }}-frontend - {{- if .Values.frontend.persistentVolumeClaim.enabled }} - persistentVolumeClaim: - claimName: {{ include "dolphinscheduler.fullname" . }}-frontend - {{- else }} - emptyDir: {} - {{- end }} \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/templates/ingress.yaml b/docker/kubernetes/dolphinscheduler/templates/ingress.yaml index d0f923dcf1..6edc82b12a 100644 --- a/docker/kubernetes/dolphinscheduler/templates/ingress.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/ingress.yaml @@ -30,7 +30,7 @@ spec: paths: - path: {{ .Values.ingress.path }} backend: - serviceName: {{ include "dolphinscheduler.fullname" . }}-frontend + serviceName: {{ include "dolphinscheduler.fullname" . }}-api servicePort: tcp-port {{- if .Values.ingress.tls.enabled }} tls: diff --git a/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-alert.yaml b/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-alert.yaml index 7f74cd94ae..fd5e88afb9 100644 --- a/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-alert.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-alert.yaml @@ -25,7 +25,7 @@ metadata: app.kubernetes.io/managed-by: {{ .Release.Service }} spec: accessModes: - {{- range .Values.alert.persistentVolumeClaim.accessModes }} + {{- range .Values.alert.persistentVolumeClaim.accessModes }} - {{ . | quote }} {{- end }} storageClassName: {{ .Values.alert.persistentVolumeClaim.storageClassName | quote }} diff --git a/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-api.yaml b/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-api.yaml index c1074cc2b1..48292966ad 100644 --- a/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-api.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-api.yaml @@ -25,7 +25,7 @@ metadata: app.kubernetes.io/managed-by: {{ .Release.Service }} spec: accessModes: - {{- range .Values.api.persistentVolumeClaim.accessModes }} + {{- range .Values.api.persistentVolumeClaim.accessModes }} - {{ . | quote }} {{- end }} storageClassName: {{ .Values.api.persistentVolumeClaim.storageClassName | quote }} diff --git a/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-frontend.yaml b/docker/kubernetes/dolphinscheduler/templates/secret-external-fs-s3a.yaml similarity index 66% rename from docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-frontend.yaml rename to docker/kubernetes/dolphinscheduler/templates/secret-external-fs-s3a.yaml index ac9fe02a9e..78e7440392 100644 --- a/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-frontend.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/secret-external-fs-s3a.yaml @@ -14,22 +14,16 @@ # See the License for the specific language governing permissions and # limitations under the License. # -{{- if .Values.frontend.persistentVolumeClaim.enabled }} +{{- if eq .Values.common.configmap.RESOURCE_STORAGE_TYPE "S3" }} apiVersion: v1 -kind: PersistentVolumeClaim +kind: Secret metadata: - name: {{ include "dolphinscheduler.fullname" . }}-frontend + name: {{ printf "%s-%s" .Release.Name "fs-s3a" }} labels: - app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-frontend + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-fs-s3a app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} -spec: - accessModes: - {{- range .Values.frontend.persistentVolumeClaim.accessModes }} - - {{ . | quote }} - {{- end }} - storageClassName: {{ .Values.frontend.persistentVolumeClaim.storageClassName | quote }} - resources: - requests: - storage: {{ .Values.frontend.persistentVolumeClaim.storage | quote }} +type: Opaque +data: + fs-s3a-secret-key: {{ .Values.common.configmap.FS_S3A_SECRET_KEY | b64enc | quote }} {{- end }} \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml index e9dc7919ca..85633a9f33 100644 --- a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml @@ -54,59 +54,6 @@ spec: {{- if .Values.master.tolerations }} tolerations: {{- toYaml . | nindent 8 }} {{- end }} - initContainers: - - name: init-zookeeper - image: busybox:1.31.0 - command: - - /bin/sh - - -ec - - | - echo "${ZOOKEEPER_QUORUM}" | awk -F ',' 'BEGIN{ i=1 }{ while( i <= NF ){ print $i; i++ } }' | while read line; do - while ! nc -z ${line%:*} ${line#*:}; do - counter=$((counter+1)) - if [ $counter == 5 ]; then - echo "Error: Couldn't connect to zookeeper." - exit 1 - fi - echo "Trying to connect to zookeeper at ${line}. Attempt $counter." - sleep 60 - done - done - env: - - name: ZOOKEEPER_QUORUM - {{- if .Values.zookeeper.enabled }} - value: "{{ template "dolphinscheduler.zookeeper.quorum" . }}" - {{- else }} - value: {{ .Values.externalZookeeper.zookeeperQuorum }} - {{- end }} - - name: init-database - image: busybox:1.31.0 - command: - - /bin/sh - - -ec - - | - while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do - counter=$((counter+1)) - if [ $counter == 5 ]; then - echo "Error: Couldn't connect to database." - exit 1 - fi - echo "Trying to connect to database at ${DATABASE_HOST}:${DATABASE_PORT}. Attempt $counter." - sleep 60 - done - env: - - name: DATABASE_HOST - {{- if .Values.postgresql.enabled }} - value: {{ template "dolphinscheduler.postgresql.fullname" . }} - {{- else }} - value: {{ .Values.externalDatabase.host | quote }} - {{- end }} - - name: DATABASE_PORT - {{- if .Values.postgresql.enabled }} - value: "5432" - {{- else }} - value: {{ .Values.externalDatabase.port | quote }} - {{- end }} {{- if .Values.image.pullSecrets }} imagePullSecrets: - name: {{ include "dolphinscheduler.image.pullSecrets" . }} @@ -114,17 +61,20 @@ spec: containers: - name: {{ include "dolphinscheduler.fullname" . }}-master image: {{ include "dolphinscheduler.image.repository" . | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} args: - "master-server" ports: - containerPort: {{ .Values.master.configmap.MASTER_LISTEN_PORT }} name: "master-port" - imagePullPolicy: {{ .Values.image.pullPolicy }} env: - - name: DOLPHINSCHEDULER_OPTS - value: {{ default "" .Values.master.jvmOptions }} - name: TZ value: {{ .Values.timezone }} + - name: DOLPHINSCHEDULER_OPTS + valueFrom: + configMapKeyRef: + key: DOLPHINSCHEDULER_OPTS + name: {{ include "dolphinscheduler.fullname" . }}-master - name: MASTER_EXEC_THREADS valueFrom: configMapKeyRef: @@ -168,7 +118,7 @@ spec: - name: DOLPHINSCHEDULER_DATA_BASEDIR_PATH valueFrom: configMapKeyRef: - name: {{ include "dolphinscheduler.fullname" . }}-master + name: {{ include "dolphinscheduler.fullname" . }}-common key: DOLPHINSCHEDULER_DATA_BASEDIR_PATH - name: DATABASE_TYPE {{- if .Values.postgresql.enabled }} @@ -230,40 +180,10 @@ spec: {{- end }} - name: ZOOKEEPER_ROOT {{- if .Values.zookeeper.enabled }} - value: "/dolphinscheduler" + value: {{ .Values.zookeeper.zookeeperRoot }} {{- else }} value: {{ .Values.externalZookeeper.zookeeperRoot }} {{- end }} - - name: RESOURCE_STORAGE_TYPE - valueFrom: - configMapKeyRef: - key: RESOURCE_STORAGE_TYPE - name: {{ include "dolphinscheduler.fullname" . }}-common - - name: RESOURCE_UPLOAD_PATH - valueFrom: - configMapKeyRef: - key: RESOURCE_UPLOAD_PATH - name: {{ include "dolphinscheduler.fullname" . }}-common - - name: FS_DEFAULT_FS - valueFrom: - configMapKeyRef: - key: FS_DEFAULT_FS - name: {{ include "dolphinscheduler.fullname" . }}-common - - name: FS_S3A_ENDPOINT - valueFrom: - configMapKeyRef: - key: FS_S3A_ENDPOINT - name: {{ include "dolphinscheduler.fullname" . }}-common - - name: FS_S3A_ACCESS_KEY - valueFrom: - configMapKeyRef: - key: FS_S3A_ACCESS_KEY - name: {{ include "dolphinscheduler.fullname" . }}-common - - name: FS_S3A_SECRET_KEY - valueFrom: - configMapKeyRef: - key: FS_S3A_SECRET_KEY - name: {{ include "dolphinscheduler.fullname" . }}-common {{- if .Values.master.resources }} resources: limits: diff --git a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml index ae562cc62b..b8a62fa6ae 100644 --- a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml @@ -54,59 +54,6 @@ spec: {{- if .Values.worker.tolerations }} tolerations: {{- toYaml . | nindent 8 }} {{- end }} - initContainers: - - name: init-zookeeper - image: busybox:1.31.0 - command: - - /bin/sh - - -ec - - | - echo "${ZOOKEEPER_QUORUM}" | awk -F ',' 'BEGIN{ i=1 }{ while( i <= NF ){ print $i; i++ } }' | while read line; do - while ! nc -z ${line%:*} ${line#*:}; do - counter=$((counter+1)) - if [ $counter == 5 ]; then - echo "Error: Couldn't connect to zookeeper." - exit 1 - fi - echo "Trying to connect to zookeeper at ${line}. Attempt $counter." - sleep 60 - done - done - env: - - name: ZOOKEEPER_QUORUM - {{- if .Values.zookeeper.enabled }} - value: "{{ template "dolphinscheduler.zookeeper.quorum" . }}" - {{- else }} - value: {{ .Values.externalZookeeper.zookeeperQuorum }} - {{- end }} - - name: init-database - image: busybox:1.31.0 - command: - - /bin/sh - - -ec - - | - while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do - counter=$((counter+1)) - if [ $counter == 5 ]; then - echo "Error: Couldn't connect to database." - exit 1 - fi - echo "Trying to connect to database at ${DATABASE_HOST}:${DATABASE_PORT}. Attempt $counter." - sleep 60 - done - env: - - name: DATABASE_HOST - {{- if .Values.postgresql.enabled }} - value: {{ template "dolphinscheduler.postgresql.fullname" . }} - {{- else }} - value: {{ .Values.externalDatabase.host | quote }} - {{- end }} - - name: DATABASE_PORT - {{- if .Values.postgresql.enabled }} - value: "5432" - {{- else }} - value: {{ .Values.externalDatabase.port | quote }} - {{- end }} {{- if .Values.image.pullSecrets }} imagePullSecrets: - name: {{ include "dolphinscheduler.image.pullSecrets" . }} @@ -114,29 +61,27 @@ spec: containers: - name: {{ include "dolphinscheduler.fullname" . }}-worker image: {{ include "dolphinscheduler.image.repository" . | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} args: - "worker-server" ports: - containerPort: {{ .Values.worker.configmap.WORKER_LISTEN_PORT }} name: "worker-port" - containerPort: 50051 - name: "logs-port" - imagePullPolicy: {{ .Values.image.pullPolicy }} + name: "logger-port" env: - - name: DOLPHINSCHEDULER_OPTS - value: {{ default "" .Values.worker.jvmOptions }} - name: TZ value: {{ .Values.timezone }} - - name: WORKER_EXEC_THREADS + - name: DOLPHINSCHEDULER_OPTS valueFrom: configMapKeyRef: + key: DOLPHINSCHEDULER_OPTS name: {{ include "dolphinscheduler.fullname" . }}-worker - key: WORKER_EXEC_THREADS - - name: WORKER_FETCH_TASK_NUM + - name: WORKER_EXEC_THREADS valueFrom: configMapKeyRef: name: {{ include "dolphinscheduler.fullname" . }}-worker - key: WORKER_FETCH_TASK_NUM + key: WORKER_EXEC_THREADS - name: WORKER_HEARTBEAT_INTERVAL valueFrom: configMapKeyRef: @@ -170,8 +115,58 @@ spec: - name: DOLPHINSCHEDULER_DATA_BASEDIR_PATH valueFrom: configMapKeyRef: - name: {{ include "dolphinscheduler.fullname" . }}-master + name: {{ include "dolphinscheduler.fullname" . }}-common key: DOLPHINSCHEDULER_DATA_BASEDIR_PATH + - name: ALERT_PLUGIN_DIR + valueFrom: + configMapKeyRef: + key: ALERT_PLUGIN_DIR + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: XLS_FILE_PATH + valueFrom: + configMapKeyRef: + key: XLS_FILE_PATH + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_SERVER_HOST + valueFrom: + configMapKeyRef: + key: MAIL_SERVER_HOST + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_SERVER_PORT + valueFrom: + configMapKeyRef: + key: MAIL_SERVER_PORT + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_SENDER + valueFrom: + configMapKeyRef: + key: MAIL_SENDER + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_USER + valueFrom: + configMapKeyRef: + key: MAIL_USER + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_PASSWD + valueFrom: + configMapKeyRef: + key: MAIL_PASSWD + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_SMTP_STARTTLS_ENABLE + valueFrom: + configMapKeyRef: + key: MAIL_SMTP_STARTTLS_ENABLE + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_SMTP_SSL_ENABLE + valueFrom: + configMapKeyRef: + key: MAIL_SMTP_SSL_ENABLE + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_SMTP_SSL_TRUST + valueFrom: + configMapKeyRef: + key: MAIL_SMTP_SSL_TRUST + name: {{ include "dolphinscheduler.fullname" . }}-alert - name: DATABASE_TYPE {{- if .Values.postgresql.enabled }} value: "postgresql" @@ -232,7 +227,7 @@ spec: {{- end }} - name: ZOOKEEPER_ROOT {{- if .Values.zookeeper.enabled }} - value: "/dolphinscheduler" + value: {{ .Values.zookeeper.zookeeperRoot }} {{- else }} value: {{ .Values.externalZookeeper.zookeeperRoot }} {{- end }} @@ -251,6 +246,7 @@ spec: configMapKeyRef: key: FS_DEFAULT_FS name: {{ include "dolphinscheduler.fullname" . }}-common + {{- if eq .Values.common.configmap.RESOURCE_STORAGE_TYPE "S3" }} - name: FS_S3A_ENDPOINT valueFrom: configMapKeyRef: @@ -263,54 +259,10 @@ spec: name: {{ include "dolphinscheduler.fullname" . }}-common - name: FS_S3A_SECRET_KEY valueFrom: - configMapKeyRef: - key: FS_S3A_SECRET_KEY - name: {{ include "dolphinscheduler.fullname" . }}-common - - name: XLS_FILE_PATH - valueFrom: - configMapKeyRef: - key: XLS_FILE_PATH - name: {{ include "dolphinscheduler.fullname" . }}-alert - - name: MAIL_SERVER_HOST - valueFrom: - configMapKeyRef: - key: MAIL_SERVER_HOST - name: {{ include "dolphinscheduler.fullname" . }}-alert - - name: MAIL_SERVER_PORT - valueFrom: - configMapKeyRef: - key: MAIL_SERVER_PORT - name: {{ include "dolphinscheduler.fullname" . }}-alert - - name: MAIL_SENDER - valueFrom: - configMapKeyRef: - key: MAIL_SENDER - name: {{ include "dolphinscheduler.fullname" . }}-alert - - name: MAIL_USER - valueFrom: - configMapKeyRef: - key: MAIL_USER - name: {{ include "dolphinscheduler.fullname" . }}-alert - - name: MAIL_PASSWD - valueFrom: - configMapKeyRef: - key: MAIL_PASSWD - name: {{ include "dolphinscheduler.fullname" . }}-alert - - name: MAIL_SMTP_STARTTLS_ENABLE - valueFrom: - configMapKeyRef: - key: MAIL_SMTP_STARTTLS_ENABLE - name: {{ include "dolphinscheduler.fullname" . }}-alert - - name: MAIL_SMTP_SSL_ENABLE - valueFrom: - configMapKeyRef: - key: MAIL_SMTP_SSL_ENABLE - name: {{ include "dolphinscheduler.fullname" . }}-alert - - name: MAIL_SMTP_SSL_TRUST - valueFrom: - configMapKeyRef: - key: MAIL_SMTP_SSL_TRUST - name: {{ include "dolphinscheduler.fullname" . }}-alert + secretKeyRef: + key: fs-s3a-secret-key + name: {{ printf "%s-%s" .Release.Name "fs-s3a" }} + {{- end }} - name: ENTERPRISE_WECHAT_ENABLE valueFrom: configMapKeyRef: @@ -372,13 +324,13 @@ spec: failureThreshold: {{ .Values.worker.readinessProbe.failureThreshold }} {{- end }} volumeMounts: - - mountPath: {{ include "dolphinscheduler.worker.base.dir" . | quote }} + - mountPath: {{ include "dolphinscheduler.data.basedir.path" . | quote }} name: {{ include "dolphinscheduler.fullname" . }}-worker-data - mountPath: "/opt/dolphinscheduler/logs" name: {{ include "dolphinscheduler.fullname" . }}-worker-logs - mountPath: "/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh" subPath: "dolphinscheduler_env.sh" - name: {{ include "dolphinscheduler.fullname" . }}-worker-configmap + name: {{ include "dolphinscheduler.fullname" . }}-common-env volumes: - name: {{ include "dolphinscheduler.fullname" . }}-worker-data {{- if .Values.worker.persistentVolumeClaim.dataPersistentVolume.enabled }} @@ -394,12 +346,12 @@ spec: {{- else }} emptyDir: {} {{- end }} - - name: {{ include "dolphinscheduler.fullname" . }}-worker-configmap + - name: {{ include "dolphinscheduler.fullname" . }}-common-env configMap: defaultMode: 0777 - name: {{ include "dolphinscheduler.fullname" . }}-worker + name: {{ include "dolphinscheduler.fullname" . }}-common items: - - key: dolphinscheduler_env.sh + - key: DOLPHINSCHEDULER_ENV path: dolphinscheduler_env.sh {{- if .Values.worker.persistentVolumeClaim.enabled }} volumeClaimTemplates: diff --git a/docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml b/docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml index fb3b85b5c3..5c4e75436f 100644 --- a/docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml @@ -30,9 +30,9 @@ spec: protocol: TCP name: worker-port - port: 50051 - targetPort: logs-port + targetPort: logger-port protocol: TCP - name: logs-port + name: logger-port selector: app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/docker/kubernetes/dolphinscheduler/values.yaml b/docker/kubernetes/dolphinscheduler/values.yaml index 3261b08401..61201dfaa7 100644 --- a/docker/kubernetes/dolphinscheduler/values.yaml +++ b/docker/kubernetes/dolphinscheduler/values.yaml @@ -58,62 +58,74 @@ externalDatabase: # If not exists external zookeeper, by default, Dolphinscheduler's zookeeper will use it. zookeeper: enabled: true - taskQueue: "zookeeper" - config: null + fourlwCommandsWhitelist: srvr,ruok,wchs,cons service: port: "2181" persistence: enabled: false size: "20Gi" storageClass: "-" + zookeeperRoot: "/dolphinscheduler" # If exists external zookeeper, and set zookeeper.enable value to false. -# If zookeeper.enable is false, Dolphinscheduler's zookeeper will use it. +# If zookeeper.enable is false, Dolphinscheduler's zookeeper will use it. externalZookeeper: - taskQueue: "zookeeper" zookeeperQuorum: "127.0.0.1:2181" zookeeperRoot: "/dolphinscheduler" common: + ## ConfigMap configmap: - DOLPHINSCHEDULER_ENV_PATH: "/tmp/dolphinscheduler/env" - DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler/files" - RESOURCE_STORAGE_TYPE: "NONE" - RESOURCE_UPLOAD_PATH: "/ds" - FS_DEFAULT_FS: "s3a://xxxx" + DOLPHINSCHEDULER_ENV: + - "export HADOOP_HOME=/opt/soft/hadoop" + - "export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop" + - "export SPARK_HOME1=/opt/soft/spark1" + - "export SPARK_HOME2=/opt/soft/spark2" + - "export PYTHON_HOME=/usr/bin/python" + - "export JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk" + - "export HIVE_HOME=/opt/soft/hive" + - "export FLINK_HOME=/opt/soft/flink" + - "export DATAX_HOME=/opt/soft/datax/bin/datax.py" + - "export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$PATH" + DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler" + RESOURCE_STORAGE_TYPE: "HDFS" + RESOURCE_UPLOAD_PATH: "/dolphinscheduler" + FS_DEFAULT_FS: "file:///" FS_S3A_ENDPOINT: "s3.xxx.amazonaws.com" FS_S3A_ACCESS_KEY: "xxxxxxx" FS_S3A_SECRET_KEY: "xxxxxxx" master: + ## PodManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down. podManagementPolicy: "Parallel" + ## Replicas is the desired number of replicas of the given Template. replicas: "3" - # NodeSelector is a selector which must be true for the pod to fit on a node. - # Selector which must match a node's labels for the pod to be scheduled on that node. - # More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## You can use annotations to attach arbitrary non-identifying metadata to objects. + ## Clients such as tools and libraries can retrieve this metadata. + annotations: {} + ## Affinity is a group of affinity scheduling rules. If specified, the pod's scheduling constraints. + ## More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core + affinity: {} + ## NodeSelector is a selector which must be true for the pod to fit on a node. + ## Selector which must match a node's labels for the pod to be scheduled on that node. + ## More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ nodeSelector: {} - # Tolerations are appended (excluding duplicates) to pods running with this RuntimeClass during admission, - # effectively unioning the set of nodes tolerated by the pod and the RuntimeClass. + ## Tolerations are appended (excluding duplicates) to pods running with this RuntimeClass during admission, + ## effectively unioning the set of nodes tolerated by the pod and the RuntimeClass. tolerations: [] - # Affinity is a group of affinity scheduling rules. - # If specified, the pod's scheduling constraints. - # More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core - affinity: {} - # The jvm options for java instance startup - jvmOptions: "" + ## Compute Resources required by this container. Cannot be updated. + ## More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container resources: {} - # limits: - # memory: "18Gi" - # cpu: "4" - # requests: - # memory: "2Gi" - # cpu: "500m" - # You can use annotations to attach arbitrary non-identifying metadata to objects. - # Clients such as tools and libraries can retrieve this metadata. - annotations: {} - ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. - ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + # resources: + # limits: + # memory: "8Gi" + # cpu: "4" + # requests: + # memory: "2Gi" + # cpu: "500m" + ## ConfigMap configmap: + DOLPHINSCHEDULER_OPTS: "" MASTER_EXEC_THREADS: "100" MASTER_EXEC_TASK_NUM: "20" MASTER_HEARTBEAT_INTERVAL: "10" @@ -122,6 +134,8 @@ master: MASTER_MAX_CPULOAD_AVG: "100" MASTER_RESERVED_MEMORY: "0.1" MASTER_LISTEN_PORT: "5678" + ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. + ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes livenessProbe: enabled: true initialDelaySeconds: "30" @@ -138,7 +152,7 @@ master: timeoutSeconds: "5" failureThreshold: "3" successThreshold: "1" - ## volumeClaimTemplates is a list of claims that pods are allowed to reference. + ## PersistentVolumeClaim represents a reference to a PersistentVolumeClaim in the same namespace. ## The StatefulSet controller is responsible for mapping network identities to claims in a way that maintains the identity of a pod. ## Every claim in this list must have at least one matching (by name) volumeMount in one container in the template. ## A claim in this list takes precedence over any volumes in the template, with the same name. @@ -150,31 +164,43 @@ master: storage: "20Gi" worker: + ## PodManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down. podManagementPolicy: "Parallel" + ## Replicas is the desired number of replicas of the given Template. replicas: "3" - # NodeSelector is a selector which must be true for the pod to fit on a node. - # Selector which must match a node's labels for the pod to be scheduled on that node. - # More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## You can use annotations to attach arbitrary non-identifying metadata to objects. + ## Clients such as tools and libraries can retrieve this metadata. + annotations: {} + ## Affinity is a group of affinity scheduling rules. If specified, the pod's scheduling constraints. + ## More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core + affinity: {} + ## NodeSelector is a selector which must be true for the pod to fit on a node. + ## Selector which must match a node's labels for the pod to be scheduled on that node. + ## More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ nodeSelector: {} - # Tolerations are appended (excluding duplicates) to pods running with this RuntimeClass during admission, - # effectively unioning the set of nodes tolerated by the pod and the RuntimeClass. + ## Tolerations are appended (excluding duplicates) to pods running with this RuntimeClass during admission, + ## effectively unioning the set of nodes tolerated by the pod and the RuntimeClass. tolerations: [] - # Affinity is a group of affinity scheduling rules. - # If specified, the pod's scheduling constraints. - # More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core - affinity: {} - # The jvm options for java instance startup - jvmOptions: "" + ## Compute Resources required by this container. Cannot be updated. + ## More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container resources: {} - # limits: - # memory: "18Gi" - # cpu: "4" - # requests: - # memory: "2Gi" - # cpu: "500m" - # You can use annotations to attach arbitrary non-identifying metadata to objects. - # Clients such as tools and libraries can retrieve this metadata. - annotations: {} + # resources: + # limits: + # memory: "8Gi" + # cpu: "4" + # requests: + # memory: "2Gi" + # cpu: "500m" + ## ConfigMap + configmap: + DOLPHINSCHEDULER_OPTS: "" + WORKER_EXEC_THREADS: "100" + WORKER_HEARTBEAT_INTERVAL: "10" + WORKER_MAX_CPULOAD_AVG: "100" + WORKER_RESERVED_MEMORY: "0.1" + WORKER_LISTEN_PORT: "1234" + WORKER_GROUP: "default" + WORKER_WEIGHT: "100" ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes livenessProbe: @@ -193,27 +219,7 @@ worker: timeoutSeconds: "5" failureThreshold: "3" successThreshold: "1" - configmap: - WORKER_EXEC_THREADS: "100" - WORKER_HEARTBEAT_INTERVAL: "10" - WORKER_FETCH_TASK_NUM: "3" - WORKER_MAX_CPULOAD_AVG: "100" - WORKER_RESERVED_MEMORY: "0.1" - WORKER_LISTEN_PORT: "1234" - WORKER_GROUP: "default" - WORKER_WEIGHT: "100" - DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler" - DOLPHINSCHEDULER_ENV: - - "export HADOOP_HOME=/opt/soft/hadoop" - - "export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop" - - "export SPARK_HOME1=/opt/soft/spark1" - - "export SPARK_HOME2=/opt/soft/spark2" - - "export PYTHON_HOME=/opt/soft/python" - - "export JAVA_HOME=/opt/soft/java" - - "export HIVE_HOME=/opt/soft/hive" - - "export FLINK_HOME=/opt/soft/flink" - - "export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$PATH" - ## volumeClaimTemplates is a list of claims that pods are allowed to reference. + ## PersistentVolumeClaim represents a reference to a PersistentVolumeClaim in the same namespace. ## The StatefulSet controller is responsible for mapping network identities to claims in a way that maintains the identity of a pod. ## Every claim in this list must have at least one matching (by name) volumeMount in one container in the template. ## A claim in this list takes precedence over any volumes in the template, with the same name. @@ -235,38 +241,40 @@ worker: storage: "20Gi" alert: + ## Number of desired pods. This is a pointer to distinguish between explicit zero and not specified. Defaults to 1. + replicas: "1" + ## The deployment strategy to use to replace existing pods with new ones. strategy: type: "RollingUpdate" rollingUpdate: maxSurge: "25%" maxUnavailable: "25%" - replicas: "1" - # NodeSelector is a selector which must be true for the pod to fit on a node. - # Selector which must match a node's labels for the pod to be scheduled on that node. - # More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## You can use annotations to attach arbitrary non-identifying metadata to objects. + ## Clients such as tools and libraries can retrieve this metadata. + annotations: {} + ## NodeSelector is a selector which must be true for the pod to fit on a node. + ## Selector which must match a node's labels for the pod to be scheduled on that node. + ## More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + affinity: {} + ## Compute Resources required by this container. Cannot be updated. + ## More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container nodeSelector: {} - # Tolerations are appended (excluding duplicates) to pods running with this RuntimeClass during admission, - # effectively unioning the set of nodes tolerated by the pod and the RuntimeClass. + ## Tolerations are appended (excluding duplicates) to pods running with this RuntimeClass during admission, + ## effectively unioning the set of nodes tolerated by the pod and the RuntimeClass. tolerations: [] - # Affinity is a group of affinity scheduling rules. - # If specified, the pod's scheduling constraints. - # More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core - affinity: {} - # The jvm options for java instance startup - jvmOptions: "" + ## Affinity is a group of affinity scheduling rules. If specified, the pod's scheduling constraints. + ## More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core resources: {} - # limits: - # memory: "4Gi" - # cpu: "1" - # requests: - # memory: "2Gi" - # cpu: "500m" - # You can use annotations to attach arbitrary non-identifying metadata to objects. - # Clients such as tools and libraries can retrieve this metadata. - annotations: {} - ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. - ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + # resources: + # limits: + # memory: "2Gi" + # cpu: "1" + # requests: + # memory: "1Gi" + # cpu: "500m" + ## ConfigMap configmap: + DOLPHINSCHEDULER_OPTS: "" ALERT_PLUGIN_DIR: "/opt/dolphinscheduler/alert/plugin" XLS_FILE_PATH: "/tmp/xls" MAIL_SERVER_HOST: "" @@ -282,63 +290,6 @@ alert: ENTERPRISE_WECHAT_SECRET: "" ENTERPRISE_WECHAT_AGENT_ID: "" ENTERPRISE_WECHAT_USERS: "" - livenessProbe: - enabled: true - initialDelaySeconds: "30" - periodSeconds: "30" - timeoutSeconds: "5" - failureThreshold: "3" - successThreshold: "1" - ## Periodic probe of container service readiness. Container will be removed from service endpoints if the probe fails. Cannot be updated. - ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - readinessProbe: - enabled: true - initialDelaySeconds: "30" - periodSeconds: "30" - timeoutSeconds: "5" - failureThreshold: "3" - successThreshold: "1" - ## volumeClaimTemplates is a list of claims that pods are allowed to reference. - ## The StatefulSet controller is responsible for mapping network identities to claims in a way that maintains the identity of a pod. - ## Every claim in this list must have at least one matching (by name) volumeMount in one container in the template. - ## A claim in this list takes precedence over any volumes in the template, with the same name. - persistentVolumeClaim: - enabled: false - accessModes: - - "ReadWriteOnce" - storageClassName: "-" - storage: "20Gi" - -api: - strategy: - type: "RollingUpdate" - rollingUpdate: - maxSurge: "25%" - maxUnavailable: "25%" - replicas: "1" - # NodeSelector is a selector which must be true for the pod to fit on a node. - # Selector which must match a node's labels for the pod to be scheduled on that node. - # More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ - nodeSelector: {} - # Tolerations are appended (excluding duplicates) to pods running with this RuntimeClass during admission, - # effectively unioning the set of nodes tolerated by the pod and the RuntimeClass. - tolerations: [] - # Affinity is a group of affinity scheduling rules. - # If specified, the pod's scheduling constraints. - # More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core - affinity: {} - # The jvm options for java instance startup - jvmOptions: "" - resources: {} - # limits: - # memory: "4Gi" - # cpu: "2" - # requests: - # memory: "2Gi" - # cpu: "500m" - # You can use annotations to attach arbitrary non-identifying metadata to objects. - # Clients such as tools and libraries can retrieve this metadata. - annotations: {} ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes livenessProbe: @@ -357,10 +308,8 @@ api: timeoutSeconds: "5" failureThreshold: "3" successThreshold: "1" - ## volumeClaimTemplates is a list of claims that pods are allowed to reference. - ## The StatefulSet controller is responsible for mapping network identities to claims in a way that maintains the identity of a pod. - ## Every claim in this list must have at least one matching (by name) volumeMount in one container in the template. - ## A claim in this list takes precedence over any volumes in the template, with the same name. + ## PersistentVolumeClaim represents a reference to a PersistentVolumeClaim in the same namespace. + ## More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims persistentVolumeClaim: enabled: false accessModes: @@ -368,34 +317,41 @@ api: storageClassName: "-" storage: "20Gi" -frontend: +api: + ## Number of desired pods. This is a pointer to distinguish between explicit zero and not specified. Defaults to 1. + replicas: "1" + ## The deployment strategy to use to replace existing pods with new ones. strategy: type: "RollingUpdate" rollingUpdate: maxSurge: "25%" maxUnavailable: "25%" - replicas: "1" - # NodeSelector is a selector which must be true for the pod to fit on a node. - # Selector which must match a node's labels for the pod to be scheduled on that node. - # More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## You can use annotations to attach arbitrary non-identifying metadata to objects. + ## Clients such as tools and libraries can retrieve this metadata. + annotations: {} + ## NodeSelector is a selector which must be true for the pod to fit on a node. + ## Selector which must match a node's labels for the pod to be scheduled on that node. + ## More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + affinity: {} + ## Compute Resources required by this container. Cannot be updated. + ## More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container nodeSelector: {} - # Tolerations are appended (excluding duplicates) to pods running with this RuntimeClass during admission, - # effectively unioning the set of nodes tolerated by the pod and the RuntimeClass. + ## Tolerations are appended (excluding duplicates) to pods running with this RuntimeClass during admission, + ## effectively unioning the set of nodes tolerated by the pod and the RuntimeClass. tolerations: [] - # Affinity is a group of affinity scheduling rules. - # If specified, the pod's scheduling constraints. - # More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core - affinity: {} + ## Affinity is a group of affinity scheduling rules. If specified, the pod's scheduling constraints. + ## More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core resources: {} - # limits: - # memory: "256Mi" + # resources: + # limits: + # memory: "2Gi" # cpu: "1" # requests: - # memory: "256Mi" + # memory: "1Gi" # cpu: "500m" - # You can use annotations to attach arbitrary non-identifying metadata to objects. - # Clients such as tools and libraries can retrieve this metadata. - annotations: {} + ## ConfigMap + configmap: + DOLPHINSCHEDULER_OPTS: "" ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes livenessProbe: @@ -414,10 +370,8 @@ frontend: timeoutSeconds: "5" failureThreshold: "3" successThreshold: "1" - ## volumeClaimTemplates is a list of claims that pods are allowed to reference. - ## The StatefulSet controller is responsible for mapping network identities to claims in a way that maintains the identity of a pod. - ## Every claim in this list must have at least one matching (by name) volumeMount in one container in the template. - ## A claim in this list takes precedence over any volumes in the template, with the same name. + ## PersistentVolumeClaim represents a reference to a PersistentVolumeClaim in the same namespace. + ## More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims persistentVolumeClaim: enabled: false accessModes: From 5f40fce1f5544baffe34356e4244aaf0b364fd83 Mon Sep 17 00:00:00 2001 From: Shiwen Cheng Date: Sun, 14 Feb 2021 21:37:25 +0800 Subject: [PATCH 8/8] [Improvement][UI] Improve long host display in K8s (#4756) --- .../pages/monitor/pages/servers/master.vue | 22 ++++++++++++++----- .../pages/monitor/pages/servers/worker.vue | 2 +- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/monitor/pages/servers/master.vue b/dolphinscheduler-ui/src/js/conf/home/pages/monitor/pages/servers/master.vue index 40ff14b53d..3f257e5fe8 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/monitor/pages/servers/master.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/monitor/pages/servers/master.vue @@ -21,8 +21,8 @@
- IP: {{item.host}} - {{$t('Zk registration directory')}}: {{item.zkDirectory}} + Host: {{item.host}} + {{$t('Zk registration directory')}}: {{$t('Directory detail')}}
{{$t('Create Time')}}: {{item.createTime | formatDate}} @@ -57,6 +57,11 @@
+ + +
@@ -72,6 +77,7 @@ import mNoData from '@/module/components/noData/noData' import themeData from '@/module/echarts/themeData.json' import mListConstruction from '@/module/components/listConstruction/listConstruction' + import zookeeperDirectoriesPopup from './_source/zookeeperDirectories' export default { name: 'servers-master', @@ -79,12 +85,18 @@ return { isLoading: false, masterList: [], - color: themeData.color + color: themeData.color, + drawer: false, + zkDirectories: [] } }, props: {}, methods: { - ...mapActions('monitor', ['getMasterData']) + ...mapActions('monitor', ['getMasterData']), + _showZkDirectories (item) { + this.zkDirectories = [{ zkDirectory: item.zkDirectory }] + this.drawer = true + } }, watch: {}, created () { @@ -103,7 +115,7 @@ this.isLoading = false }) }, - components: { mListConstruction, mSpin, mNoData, mGauge } + components: { mListConstruction, mSpin, mNoData, mGauge, zookeeperDirectoriesPopup } }