diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
index 2a4f625d0d..99ba08006c 100644
--- a/.github/workflows/backend.yml
+++ b/.github/workflows/backend.yml
@@ -47,8 +47,6 @@ jobs:
submodules: true
- name: Sanity Check
uses: ./.github/actions/sanity-check
- with:
- token: ${{ secrets.GITHUB_TOKEN }} # We only need to pass this token in one workflow
- uses: actions/cache@v2
with:
path: ~/.m2/repository
@@ -56,7 +54,7 @@ jobs:
- name: Build and Package
run: |
./mvnw -B clean install \
- -Prelease \
+ -Prelease,docker \
-Dmaven.test.skip=true \
-Dcheckstyle.skip=true \
-Dhttp.keepAlive=false \
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index 5ccf937dda..7773759f28 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -61,7 +61,13 @@ jobs:
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: ${{ runner.os }}-maven-
- name: Build Image
- run: TAG=ci sh ./docker/build/hooks/build
+ run: |
+ ./mvnw -B clean install \
+ -Dmaven.test.skip \
+ -Dmaven.javadoc.skip \
+ -Dmaven.checkstyle.skip \
+ -Pdocker,release -Ddocker.tag=ci \
+ -pl dolphinscheduler-standalone-server -am
- name: Run Test
run: |
./mvnw -B -f dolphinscheduler-e2e/pom.xml -am \
diff --git a/.gitignore b/.gitignore
index 0ab7a2f1f3..3ddb8241b9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -45,7 +45,6 @@ dolphinscheduler-alert/src/main/resources/logback.xml
dolphinscheduler-server/src/main/resources/logback.xml
dolphinscheduler-ui/dist
dolphinscheduler-ui/node
-docker/build/apache-dolphinscheduler*
dolphinscheduler-common/sql
dolphinscheduler-common/test
diff --git a/docker/README.md b/deploy/README.md
similarity index 100%
rename from docker/README.md
rename to deploy/README.md
diff --git a/dolphinscheduler-dao/src/main/resources/application-postgresql.yaml b/deploy/docker/.env
old mode 100644
new mode 100755
similarity index 61%
rename from dolphinscheduler-dao/src/main/resources/application-postgresql.yaml
rename to deploy/docker/.env
index e5ad49ffae..afe8f11900
--- a/dolphinscheduler-dao/src/main/resources/application-postgresql.yaml
+++ b/deploy/docker/.env
@@ -15,20 +15,9 @@
# specific language governing permissions and limitations
# under the License.
#
-spring:
- datasource:
- driver-class-name: org.postgresql.Driver
- url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler
- username: root
- password: root
- hikari:
- connection-test-query: select 1
- minimum-idle: 5
- auto-commit: true
- validation-timeout: 3000
- pool-name: DolphinScheduler
- maximum-pool-size: 50
- connection-timeout: 30000
- idle-timeout: 600000
- leak-detection-threshold: 0
- initialization-fail-timeout: 1
+HUB=apache
+TAG=latest
+
+TZ=Asia/Shanghai
+SPRING_DATASOURCE_URL=jdbc:postgresql://dolphinscheduler-postgresql:5432/dolphinscheduler
+REGISTRY_ZOOKEEPER_CONNECT_STRING=dolphinscheduler-zookeeper:2181
diff --git a/deploy/docker/docker-compose.yml b/deploy/docker/docker-compose.yml
new file mode 100644
index 0000000000..2c557c4dd7
--- /dev/null
+++ b/deploy/docker/docker-compose.yml
@@ -0,0 +1,153 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3.8"
+
+services:
+ dolphinscheduler-postgresql:
+ image: bitnami/postgresql:11.11.0
+ environment:
+ POSTGRESQL_USERNAME: root
+ POSTGRESQL_PASSWORD: root
+ POSTGRESQL_DATABASE: dolphinscheduler
+ volumes:
+ - dolphinscheduler-postgresql:/bitnami/postgresql
+ healthcheck:
+ test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/5432"]
+ interval: 5s
+ timeout: 60s
+ retries: 120
+ networks:
+ - dolphinscheduler
+
+ dolphinscheduler-zookeeper:
+ image: bitnami/zookeeper:3.6.2
+ environment:
+ ALLOW_ANONYMOUS_LOGIN: "yes"
+ ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons
+ volumes:
+ - dolphinscheduler-zookeeper:/bitnami/zookeeper
+ healthcheck:
+ test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/2181"]
+ interval: 5s
+ timeout: 60s
+ retries: 120
+ networks:
+ - dolphinscheduler
+
+ dolphinscheduler-schema-initializer:
+ image: ${HUB}/dolphinscheduler-tools:${TAG}
+ env_file: .env
+ command: [ bin/create-schema.sh ]
+ depends_on:
+ dolphinscheduler-postgresql:
+ condition: service_healthy
+ volumes:
+ - dolphinscheduler-logs:/opt/dolphinscheduler/logs
+ - dolphinscheduler-shared-local:/opt/soft
+ - dolphinscheduler-resource-local:/dolphinscheduler
+ networks:
+ - dolphinscheduler
+
+ dolphinscheduler-api:
+ image: ${HUB}/dolphinscheduler-api:${TAG}
+ ports:
+ - "12345:12345"
+ env_file: .env
+ healthcheck:
+ test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ]
+ interval: 30s
+ timeout: 5s
+ retries: 3
+ depends_on:
+ dolphinscheduler-schema-initializer:
+ condition: service_completed_successfully
+ dolphinscheduler-zookeeper:
+ condition: service_healthy
+ volumes:
+ - dolphinscheduler-logs:/opt/dolphinscheduler/logs
+ - dolphinscheduler-shared-local:/opt/soft
+ - dolphinscheduler-resource-local:/dolphinscheduler
+ networks:
+ - dolphinscheduler
+
+ dolphinscheduler-alert:
+ image: ${HUB}/dolphinscheduler-alert-server:${TAG}
+ env_file: .env
+ healthcheck:
+ test: [ "CMD", "curl", "http://localhost:50053/actuator/health" ]
+ interval: 30s
+ timeout: 5s
+ retries: 3
+ depends_on:
+ dolphinscheduler-schema-initializer:
+ condition: service_completed_successfully
+ volumes:
+ - dolphinscheduler-logs:/opt/dolphinscheduler/logs
+ networks:
+ - dolphinscheduler
+
+ dolphinscheduler-master:
+ image: ${HUB}/dolphinscheduler-master:${TAG}
+ env_file: .env
+ healthcheck:
+ test: [ "CMD", "curl", "http://localhost:5679/actuator/health" ]
+ interval: 30s
+ timeout: 5s
+ retries: 3
+ depends_on:
+ dolphinscheduler-schema-initializer:
+ condition: service_completed_successfully
+ dolphinscheduler-zookeeper:
+ condition: service_healthy
+ volumes:
+ - dolphinscheduler-logs:/opt/dolphinscheduler/logs
+ - dolphinscheduler-shared-local:/opt/soft
+ networks:
+ - dolphinscheduler
+
+ dolphinscheduler-worker:
+ image: ${HUB}/dolphinscheduler-worker:${TAG}
+ env_file: .env
+ healthcheck:
+ test: [ "CMD", "curl", "http://localhost:1235/actuator/health" ]
+ interval: 30s
+ timeout: 5s
+ retries: 3
+ depends_on:
+ dolphinscheduler-schema-initializer:
+ condition: service_completed_successfully
+ dolphinscheduler-zookeeper:
+ condition: service_healthy
+ volumes:
+ - dolphinscheduler-worker-data:/tmp/dolphinscheduler
+ - dolphinscheduler-logs:/opt/dolphinscheduler/logs
+ - dolphinscheduler-shared-local:/opt/soft
+ - dolphinscheduler-resource-local:/dolphinscheduler
+ networks:
+ - dolphinscheduler
+
+networks:
+ dolphinscheduler:
+ driver: bridge
+
+volumes:
+ dolphinscheduler-postgresql:
+ dolphinscheduler-zookeeper:
+ dolphinscheduler-worker-data:
+ dolphinscheduler-logs:
+ dolphinscheduler-shared-local:
+ dolphinscheduler-resource-local:
diff --git a/docker/docker-swarm/docker-stack.yml b/deploy/docker/docker-stack.yml
similarity index 56%
rename from docker/docker-swarm/docker-stack.yml
rename to deploy/docker/docker-stack.yml
index 80a95c2923..4ec363eee3 100644
--- a/docker/docker-swarm/docker-stack.yml
+++ b/deploy/docker/docker-stack.yml
@@ -17,7 +17,6 @@
version: "3.1"
services:
-
dolphinscheduler-postgresql:
image: bitnami/postgresql:11.11.0
environment:
@@ -26,9 +25,9 @@ services:
POSTGRESQL_PASSWORD: root
POSTGRESQL_DATABASE: dolphinscheduler
volumes:
- - dolphinscheduler-postgresql:/bitnami/postgresql
+ - dolphinscheduler-postgresql:/bitnami/postgresql
networks:
- - dolphinscheduler
+ - dolphinscheduler
deploy:
mode: replicated
replicas: 1
@@ -40,93 +39,81 @@ services:
ALLOW_ANONYMOUS_LOGIN: "yes"
ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons
volumes:
- - dolphinscheduler-zookeeper:/bitnami/zookeeper
+ - dolphinscheduler-zookeeper:/bitnami/zookeeper
networks:
- - dolphinscheduler
+ - dolphinscheduler
deploy:
mode: replicated
replicas: 1
dolphinscheduler-api:
- image: dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler:2.0.0-SNAPSHOT
- command: api-server
+ image: apache/dolphinscheduler-tools
ports:
- - 12345:12345
- environment:
- TZ: Asia/Shanghai
- env_file: config.env.sh
+ - 12345:12345
+ env_file: .env
healthcheck:
- test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"]
+ test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ]
interval: 30s
timeout: 5s
retries: 3
volumes:
- - dolphinscheduler-logs:/opt/dolphinscheduler/logs
- - dolphinscheduler-shared-local:/opt/soft
- - dolphinscheduler-resource-local:/dolphinscheduler
+ - dolphinscheduler-logs:/opt/dolphinscheduler/logs
+ - dolphinscheduler-shared-local:/opt/soft
+ - dolphinscheduler-resource-local:/dolphinscheduler
networks:
- - dolphinscheduler
+ - dolphinscheduler
deploy:
mode: replicated
replicas: 1
dolphinscheduler-alert:
- image: dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler:2.0.0-SNAPSHOT
- command: alert-server
- environment:
- TZ: Asia/Shanghai
- env_file: config.env.sh
+ image: apache/dolphinscheduler-alert-server
+ env_file: .env
healthcheck:
- test: ["CMD", "/root/checkpoint.sh", "AlertServer"]
+ test: [ "CMD", "curl", "http://localhost:50053/actuator/health" ]
interval: 30s
timeout: 5s
retries: 3
volumes:
- - dolphinscheduler-logs:/opt/dolphinscheduler/logs
+ - dolphinscheduler-logs:/opt/dolphinscheduler/logs
networks:
- - dolphinscheduler
+ - dolphinscheduler
deploy:
mode: replicated
replicas: 1
dolphinscheduler-master:
- image: dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler:2.0.0-SNAPSHOT
- command: master-server
- environment:
- TZ: Asia/Shanghai
- env_file: config.env.sh
+ image: apache/dolphinscheduler-master
+ env_file: .env
healthcheck:
- test: ["CMD", "/root/checkpoint.sh", "MasterServer"]
+ test: [ "CMD", "curl", "http://localhost:50053/actuator/health" ]
interval: 30s
timeout: 5s
retries: 3
volumes:
- - dolphinscheduler-logs:/opt/dolphinscheduler/logs
- - dolphinscheduler-shared-local:/opt/soft
+ - dolphinscheduler-logs:/opt/dolphinscheduler/logs
+ - dolphinscheduler-shared-local:/opt/soft
networks:
- - dolphinscheduler
+ - dolphinscheduler
deploy:
mode: replicated
replicas: 1
dolphinscheduler-worker:
- image: dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler:2.0.0-SNAPSHOT
- command: worker-server
- environment:
- TZ: Asia/Shanghai
- env_file: config.env.sh
+ image: apache/dolphinscheduler-worker
+ env_file: .env
healthcheck:
- test: ["CMD", "/root/checkpoint.sh", "WorkerServer"]
+ test: [ "CMD", "curl", "http://localhost:50053/actuator/health" ]
interval: 30s
timeout: 5s
retries: 3
volumes:
- - dolphinscheduler-worker-data:/tmp/dolphinscheduler
- - dolphinscheduler-logs:/opt/dolphinscheduler/logs
- - dolphinscheduler-shared-local:/opt/soft
- - dolphinscheduler-resource-local:/dolphinscheduler
+ - dolphinscheduler-worker-data:/tmp/dolphinscheduler
+ - dolphinscheduler-logs:/opt/dolphinscheduler/logs
+ - dolphinscheduler-shared-local:/opt/soft
+ - dolphinscheduler-resource-local:/dolphinscheduler
networks:
- - dolphinscheduler
+ - dolphinscheduler
deploy:
mode: replicated
replicas: 1
@@ -141,4 +128,4 @@ volumes:
dolphinscheduler-worker-data:
dolphinscheduler-logs:
dolphinscheduler-shared-local:
- dolphinscheduler-resource-local:
\ No newline at end of file
+ dolphinscheduler-resource-local:
diff --git a/docker/kubernetes/dolphinscheduler/Chart.yaml b/deploy/kubernetes/dolphinscheduler/Chart.yaml
similarity index 100%
rename from docker/kubernetes/dolphinscheduler/Chart.yaml
rename to deploy/kubernetes/dolphinscheduler/Chart.yaml
diff --git a/docker/kubernetes/dolphinscheduler/templates/NOTES.txt b/deploy/kubernetes/dolphinscheduler/templates/NOTES.txt
similarity index 100%
rename from docker/kubernetes/dolphinscheduler/templates/NOTES.txt
rename to deploy/kubernetes/dolphinscheduler/templates/NOTES.txt
diff --git a/docker/kubernetes/dolphinscheduler/templates/_helpers.tpl b/deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl
similarity index 100%
rename from docker/kubernetes/dolphinscheduler/templates/_helpers.tpl
rename to deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl
diff --git a/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml b/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml
similarity index 100%
rename from docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml
rename to deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml
diff --git a/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-api.yaml b/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-api.yaml
similarity index 100%
rename from docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-api.yaml
rename to deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-api.yaml
diff --git a/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-common.yaml b/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-common.yaml
similarity index 100%
rename from docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-common.yaml
rename to deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-common.yaml
diff --git a/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml b/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml
similarity index 100%
rename from docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml
rename to deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml
diff --git a/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml b/deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml
similarity index 100%
rename from docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml
rename to deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml
diff --git a/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml b/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml
similarity index 100%
rename from docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml
rename to deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml
diff --git a/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml b/deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml
similarity index 100%
rename from docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml
rename to deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml
diff --git a/docker/kubernetes/dolphinscheduler/templates/ingress.yaml b/deploy/kubernetes/dolphinscheduler/templates/ingress.yaml
similarity index 100%
rename from docker/kubernetes/dolphinscheduler/templates/ingress.yaml
rename to deploy/kubernetes/dolphinscheduler/templates/ingress.yaml
diff --git a/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-alert.yaml b/deploy/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-alert.yaml
similarity index 100%
rename from docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-alert.yaml
rename to deploy/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-alert.yaml
diff --git a/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-api.yaml b/deploy/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-api.yaml
similarity index 100%
rename from docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-api.yaml
rename to deploy/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-api.yaml
diff --git a/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-fs-file.yaml b/deploy/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-fs-file.yaml
similarity index 100%
rename from docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-fs-file.yaml
rename to deploy/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-fs-file.yaml
diff --git a/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-shared.yaml b/deploy/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-shared.yaml
similarity index 100%
rename from docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-shared.yaml
rename to deploy/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-shared.yaml
diff --git a/docker/kubernetes/dolphinscheduler/templates/secret-external-database.yaml b/deploy/kubernetes/dolphinscheduler/templates/secret-external-database.yaml
similarity index 100%
rename from docker/kubernetes/dolphinscheduler/templates/secret-external-database.yaml
rename to deploy/kubernetes/dolphinscheduler/templates/secret-external-database.yaml
diff --git a/docker/kubernetes/dolphinscheduler/templates/secret-external-fs-s3a.yaml b/deploy/kubernetes/dolphinscheduler/templates/secret-external-fs-s3a.yaml
similarity index 100%
rename from docker/kubernetes/dolphinscheduler/templates/secret-external-fs-s3a.yaml
rename to deploy/kubernetes/dolphinscheduler/templates/secret-external-fs-s3a.yaml
diff --git a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml b/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml
similarity index 100%
rename from docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml
rename to deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml
diff --git a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml b/deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml
similarity index 100%
rename from docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml
rename to deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml
diff --git a/docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-alert.yaml b/deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-alert.yaml
similarity index 100%
rename from docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-alert.yaml
rename to deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-alert.yaml
diff --git a/docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-api.yaml b/deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-api.yaml
similarity index 100%
rename from docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-api.yaml
rename to deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-api.yaml
diff --git a/docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-master-headless.yaml b/deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-master-headless.yaml
similarity index 100%
rename from docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-master-headless.yaml
rename to deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-master-headless.yaml
diff --git a/docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml b/deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml
similarity index 100%
rename from docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml
rename to deploy/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml
diff --git a/docker/kubernetes/dolphinscheduler/values.yaml b/deploy/kubernetes/dolphinscheduler/values.yaml
similarity index 100%
rename from docker/kubernetes/dolphinscheduler/values.yaml
rename to deploy/kubernetes/dolphinscheduler/values.yaml
diff --git a/docker/build/Dockerfile b/docker/build/Dockerfile
deleted file mode 100644
index 65ab79cbe3..0000000000
--- a/docker/build/Dockerfile
+++ /dev/null
@@ -1,71 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-FROM openjdk:8-jre-slim-buster
-
-ARG VERSION
-ARG DEBIAN_FRONTEND=noninteractive
-
-ENV TZ Asia/Shanghai
-ENV LANG C.UTF-8
-ENV DOCKER true
-ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler
-
-# 1. install command/library/software
-# If install slowly, you can replcae debian's mirror with new mirror, Example:
-# RUN { \
-# echo "deb http://mirrors.tuna.tsinghua.edu.cn/debian/ buster main contrib non-free"; \
-# echo "deb http://mirrors.tuna.tsinghua.edu.cn/debian/ buster-updates main contrib non-free"; \
-# echo "deb http://mirrors.tuna.tsinghua.edu.cn/debian/ buster-backports main contrib non-free"; \
-# echo "deb http://mirrors.tuna.tsinghua.edu.cn/debian-security buster/updates main contrib non-free"; \
-# } > /etc/apt/sources.list
-RUN apt-get update && \
- apt-get install -y --no-install-recommends tzdata dos2unix python supervisor procps psmisc netcat sudo tini && \
- echo "Asia/Shanghai" > /etc/timezone && \
- rm -f /etc/localtime && \
- dpkg-reconfigure tzdata && \
- rm -rf /var/lib/apt/lists/* /tmp/*
-
-# 2. add dolphinscheduler
-ADD ./apache-dolphinscheduler-${VERSION}-bin.tar.gz /opt/
-RUN ln -s -r /opt/apache-dolphinscheduler-${VERSION}-bin /opt/dolphinscheduler
-WORKDIR /opt/apache-dolphinscheduler-${VERSION}-bin
-
-# 3. add configuration and modify permissions and set soft links
-COPY ./checkpoint.sh /root/checkpoint.sh
-COPY ./startup-init-conf.sh /root/startup-init-conf.sh
-COPY ./startup.sh /root/startup.sh
-COPY ./conf/dolphinscheduler/*.tpl /opt/dolphinscheduler/conf/
-COPY ./conf/dolphinscheduler/logback/* /opt/dolphinscheduler/conf/
-COPY ./conf/dolphinscheduler/supervisor/supervisor.ini /etc/supervisor/conf.d/
-COPY ./conf/dolphinscheduler/env/dolphinscheduler_env.sh.tpl /opt/dolphinscheduler/conf/env/
-RUN sed -i 's/*.conf$/*.ini/' /etc/supervisor/supervisord.conf && \
- dos2unix /root/checkpoint.sh && \
- dos2unix /root/startup-init-conf.sh && \
- dos2unix /root/startup.sh && \
- dos2unix /opt/dolphinscheduler/script/*.sh && \
- dos2unix /opt/dolphinscheduler/bin/*.sh && \
- rm -f /bin/sh && \
- ln -s /bin/bash /bin/sh && \
- mkdir -p /tmp/xls && \
- echo PS1=\'\\w \\$ \' >> ~/.bashrc && \
- echo "Set disable_coredump false" >> /etc/sudo.conf
-
-# 4. expose port
-EXPOSE 5678 1234 12345 50051 50052
-
-ENTRYPOINT ["/usr/bin/tini", "--", "/root/startup.sh"]
diff --git a/docker/build/conf/dolphinscheduler/common.properties.tpl b/docker/build/conf/dolphinscheduler/common.properties.tpl
deleted file mode 100644
index 83776a00a2..0000000000
--- a/docker/build/conf/dolphinscheduler/common.properties.tpl
+++ /dev/null
@@ -1,91 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# user data local directory path, please make sure the directory exists and have read write permissions
-data.basedir.path=${DATA_BASEDIR_PATH}
-
-# resource storage type: HDFS, S3, NONE
-resource.storage.type=${RESOURCE_STORAGE_TYPE}
-
-# resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended
-resource.upload.path=${RESOURCE_UPLOAD_PATH}
-
-# whether to startup kerberos
-hadoop.security.authentication.startup.state=${HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE}
-
-# java.security.krb5.conf path
-java.security.krb5.conf.path=${JAVA_SECURITY_KRB5_CONF_PATH}
-
-# login user from keytab username
-login.user.keytab.username=${LOGIN_USER_KEYTAB_USERNAME}
-
-# login user from keytab path
-login.user.keytab.path=${LOGIN_USER_KEYTAB_PATH}
-
-# kerberos expire time, the unit is hour
-kerberos.expire.time=${KERBEROS_EXPIRE_TIME}
-
-# resource view suffixs
-#resource.view.suffixs=txt,log,sh,bat,conf,cfg,py,java,sql,xml,hql,properties,json,yml,yaml,ini,js
-
-# if resource.storage.type=HDFS, the user must have the permission to create directories under the HDFS root path
-hdfs.root.user=${HDFS_ROOT_USER}
-
-# if resource.storage.type=S3, the value like: s3a://dolphinscheduler; if resource.storage.type=HDFS and namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir
-fs.defaultFS=${FS_DEFAULT_FS}
-
-# if resource.storage.type=S3, s3 endpoint
-fs.s3a.endpoint=${FS_S3A_ENDPOINT}
-
-# if resource.storage.type=S3, s3 access key
-fs.s3a.access.key=${FS_S3A_ACCESS_KEY}
-
-# if resource.storage.type=S3, s3 secret key
-fs.s3a.secret.key=${FS_S3A_SECRET_KEY}
-
-# resourcemanager port, the default value is 8088 if not specified
-resource.manager.httpaddress.port=${RESOURCE_MANAGER_HTTPADDRESS_PORT}
-
-# if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single, keep this value empty
-yarn.resourcemanager.ha.rm.ids=${YARN_RESOURCEMANAGER_HA_RM_IDS}
-
-# if resourcemanager HA is enabled or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname
-yarn.application.status.address=${YARN_APPLICATION_STATUS_ADDRESS}
-
-# job history status url when application number threshold is reached(default 10000, maybe it was set to 1000)
-yarn.job.history.status.address=${YARN_JOB_HISTORY_STATUS_ADDRESS}
-
-# datasource encryption enable
-datasource.encryption.enable=${DATASOURCE_ENCRYPTION_ENABLE}
-
-# datasource encryption salt
-datasource.encryption.salt=${DATASOURCE_ENCRYPTION_SALT}
-
-# use sudo or not, if set true, executing user is tenant user and deploy user needs sudo permissions; if set false, executing user is the deploy user and doesn't need sudo permissions
-sudo.enable=${SUDO_ENABLE}
-
-# network interface preferred like eth0, default: empty
-#dolphin.scheduler.network.interface.preferred=
-
-# network IP gets priority, default: inner outer
-#dolphin.scheduler.network.priority.strategy=default
-
-# system env path
-#dolphinscheduler.env.path=env/dolphinscheduler_env.sh
-
-# development state
-development.state=false
diff --git a/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh.tpl b/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh.tpl
deleted file mode 100755
index b9b04eb83d..0000000000
--- a/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh.tpl
+++ /dev/null
@@ -1,28 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-export HADOOP_HOME=$HADOOP_HOME
-export HADOOP_CONF_DIR=$HADOOP_CONF_DIR
-export SPARK_HOME1=$SPARK_HOME1
-export SPARK_HOME2=$SPARK_HOME2
-export PYTHON_HOME=$PYTHON_HOME
-export JAVA_HOME=$JAVA_HOME
-export HIVE_HOME=$HIVE_HOME
-export FLINK_HOME=$FLINK_HOME
-export DATAX_HOME=$DATAX_HOME
-
-export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH
diff --git a/docker/build/conf/dolphinscheduler/logback/logback-api.xml b/docker/build/conf/dolphinscheduler/logback/logback-api.xml
deleted file mode 100644
index bec3d9aa49..0000000000
--- a/docker/build/conf/dolphinscheduler/logback/logback-api.xml
+++ /dev/null
@@ -1,62 +0,0 @@
-
-
-
-
-
-
-
-
-
-
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
-
- UTF-8
-
-
-
-
-
- ${log.base}/dolphinscheduler-api.log
-
- INFO
-
-
- ${log.base}/dolphinscheduler-api.%d{yyyy-MM-dd_HH}.%i.log
- 168
- 64MB
-
-
-
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
-
- UTF-8
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docker/build/conf/dolphinscheduler/logback/logback-master.xml b/docker/build/conf/dolphinscheduler/logback/logback-master.xml
deleted file mode 100644
index f0d2c81df2..0000000000
--- a/docker/build/conf/dolphinscheduler/logback/logback-master.xml
+++ /dev/null
@@ -1,82 +0,0 @@
-
-
-
-
-
-
-
-
-
-
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
-
- UTF-8
-
-
-
-
-
-
-
-
- taskAppId
- ${log.base}
-
-
-
- ${log.base}/${taskAppId}.log
-
-
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %messsage%n
-
- UTF-8
-
- true
-
-
-
-
-
- ${log.base}/dolphinscheduler-master.log
-
-
- ${log.base}/dolphinscheduler-master.%d{yyyy-MM-dd_HH}.%i.log
- 168
- 200MB
-
-
-
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
-
- UTF-8
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docker/build/conf/dolphinscheduler/logback/logback-worker.xml b/docker/build/conf/dolphinscheduler/logback/logback-worker.xml
deleted file mode 100644
index 7127219873..0000000000
--- a/docker/build/conf/dolphinscheduler/logback/logback-worker.xml
+++ /dev/null
@@ -1,82 +0,0 @@
-
-
-
-
-
-
-
-
-
-
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
-
- UTF-8
-
-
-
-
-
-
-
-
-
- taskAppId
- ${log.base}
-
-
-
- ${log.base}/${taskAppId}.log
-
-
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %messsage%n
-
- UTF-8
-
- true
-
-
-
-
- ${log.base}/dolphinscheduler-worker.log
-
-
- ${log.base}/dolphinscheduler-worker.%d{yyyy-MM-dd_HH}.%i.log
- 168
- 200MB
-
-
-
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %messsage%n
-
- UTF-8
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/docker/build/conf/dolphinscheduler/registry.properties.tpl b/docker/build/conf/dolphinscheduler/registry.properties.tpl
deleted file mode 100644
index 9ee8add211..0000000000
--- a/docker/build/conf/dolphinscheduler/registry.properties.tpl
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-registry.plugin.name=${REGISTRY_PLUGIN_NAME}
-registry.servers=${REGISTRY_SERVERS}
diff --git a/docker/build/conf/dolphinscheduler/supervisor/supervisor.ini b/docker/build/conf/dolphinscheduler/supervisor/supervisor.ini
deleted file mode 100644
index 19166f48d9..0000000000
--- a/docker/build/conf/dolphinscheduler/supervisor/supervisor.ini
+++ /dev/null
@@ -1,107 +0,0 @@
-; Licensed to the Apache Software Foundation (ASF) under one
-; or more contributor license agreements. See the NOTICE file
-; distributed with this work for additional information
-; regarding copyright ownership. The ASF licenses this file
-; to you under the Apache License, Version 2.0 (the
-; "License"); you may not use this file except in compliance
-; with the License. You may obtain a copy of the License at
-;
-; http://www.apache.org/licenses/LICENSE-2.0
-;
-; Unless required by applicable law or agreed to in writing, software
-; distributed under the License is distributed on an "AS IS" BASIS,
-; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-; See the License for the specific language governing permissions and
-; limitations under the License.
-
-; program config file
-
-[program:master]
-command=%(ENV_DOLPHINSCHEDULER_BIN)s/dolphinscheduler-daemon.sh start master-server
-directory=%(ENV_DOLPHINSCHEDULER_HOME)s
-priority=999
-autostart=%(ENV_MASTER_START_ENABLED)s
-autorestart=true
-startsecs=10
-stopwaitsecs=3
-exitcodes=0
-stopasgroup=true
-killasgroup=true
-redirect_stderr=true
-stdout_logfile=/dev/fd/1
-stdout_logfile_maxbytes=0
-
-[program:worker]
-command=%(ENV_DOLPHINSCHEDULER_BIN)s/dolphinscheduler-daemon.sh start worker-server
-directory=%(ENV_DOLPHINSCHEDULER_HOME)s
-priority=999
-autostart=%(ENV_WORKER_START_ENABLED)s
-autorestart=true
-startsecs=10
-stopwaitsecs=3
-exitcodes=0
-stopasgroup=true
-killasgroup=true
-redirect_stderr=true
-stdout_logfile=/dev/fd/1
-stdout_logfile_maxbytes=0
-
-[program:api]
-command=%(ENV_DOLPHINSCHEDULER_BIN)s/dolphinscheduler-daemon.sh start api-server
-directory=%(ENV_DOLPHINSCHEDULER_HOME)s
-priority=999
-autostart=%(ENV_API_START_ENABLED)s
-autorestart=true
-startsecs=10
-stopwaitsecs=3
-exitcodes=0
-stopasgroup=true
-killasgroup=true
-redirect_stderr=true
-stdout_logfile=/dev/fd/1
-stdout_logfile_maxbytes=0
-
-[program:alert]
-command=%(ENV_DOLPHINSCHEDULER_BIN)s/dolphinscheduler-daemon.sh start alert-server
-directory=%(ENV_DOLPHINSCHEDULER_HOME)s
-priority=999
-autostart=%(ENV_ALERT_START_ENABLED)s
-autorestart=true
-startsecs=5
-stopwaitsecs=3
-exitcodes=0
-stopasgroup=true
-killasgroup=true
-redirect_stderr=true
-stdout_logfile=/dev/fd/1
-stdout_logfile_maxbytes=0
-
-[program:logger]
-command=%(ENV_DOLPHINSCHEDULER_BIN)s/dolphinscheduler-daemon.sh start logger-server
-directory=%(ENV_DOLPHINSCHEDULER_HOME)s
-priority=999
-autostart=%(ENV_LOGGER_START_ENABLED)s
-autorestart=true
-startsecs=5
-stopwaitsecs=3
-exitcodes=0
-stopasgroup=true
-killasgroup=true
-redirect_stderr=true
-stdout_logfile=/dev/fd/1
-stdout_logfile_maxbytes=0
-
-[program:standalone]
-command=%(ENV_DOLPHINSCHEDULER_BIN)s/dolphinscheduler-daemon.sh start standalone-server
-directory=%(ENV_DOLPHINSCHEDULER_HOME)s
-priority=999
-autostart=%(ENV_STANDALONE_START_ENABLED)s
-autorestart=true
-startsecs=5
-stopwaitsecs=3
-exitcodes=0
-stopasgroup=true
-killasgroup=true
-redirect_stderr=true
-stdout_logfile=/dev/fd/1
-stdout_logfile_maxbytes=0
diff --git a/docker/build/hooks/build b/docker/build/hooks/build
deleted file mode 100755
index 1e6a96549b..0000000000
--- a/docker/build/hooks/build
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-set -e
-
-ROOT_DIR=$(dirname "$0")/../../..
-MVN="$ROOT_DIR"/mvnw
-VERSION=$("$MVN" -q -DforceStdout -N org.apache.maven.plugins:maven-help-plugin:3.2.0:evaluate -Dexpression=project.version)
-
-DOCKER_REPO=${DOCKER_REPO:-"apache/dolphinscheduler"}
-TAG=${TAG:-"$VERSION"}
-
-echo "Building Docker image as: $DOCKER_REPO:$TAG"
-
-"$MVN" -B clean package -Prelease -Dmaven.test.skip=true -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120
-
-cp "$ROOT_DIR"/dolphinscheduler-dist/target/apache-dolphinscheduler-$VERSION-bin.tar.gz "$ROOT_DIR"/docker/build/
-
-docker build --build-arg VERSION=$VERSION -t $DOCKER_REPO:$TAG "$ROOT_DIR"/docker/build/
diff --git a/docker/build/hooks/build.bat b/docker/build/hooks/build.bat
deleted file mode 100644
index 6aa3726ceb..0000000000
--- a/docker/build/hooks/build.bat
+++ /dev/null
@@ -1,61 +0,0 @@
-:: Licensed to the Apache Software Foundation (ASF) under one or more
-:: contributor license agreements. See the NOTICE file distributed with
-:: this work for additional information regarding copyright ownership.
-:: The ASF licenses this file to You under the Apache License, Version 2.0
-:: (the "License"); you may not use this file except in compliance with
-:: the License. You may obtain a copy of the License at
-::
-:: http://www.apache.org/licenses/LICENSE-2.0
-::
-:: Unless required by applicable law or agreed to in writing, software
-:: distributed under the License is distributed on an "AS IS" BASIS,
-:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-:: See the License for the specific language governing permissions and
-:: limitations under the License.
-::
-@echo off
-
-echo "------ dolphinscheduler start - build -------"
-set
-setlocal enableextensions enabledelayedexpansion
-
-if not defined VERSION (
- echo "set environment variable [VERSION]"
- set first=1
- for /f "tokens=3 delims=<>" %%a in ('findstr "[0-9].*" %cd%\pom.xml') do (
- if !first! EQU 1 (set VERSION=%%a)
- set first=0
- )
-)
-
-if not defined DOCKER_REPO (
- echo "set environment variable [DOCKER_REPO]"
- set DOCKER_REPO=dolphinscheduler
-)
-
-echo "Version: %VERSION%"
-echo "Repo: %DOCKER_REPO%"
-
-echo "Current Directory is %cd%"
-
-:: maven package(Project Directory)
-echo "mvn clean package -Prelease -DskipTests=true -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120"
-call mvn clean package -Prelease -DskipTests=true -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120
-if "%errorlevel%"=="1" goto :mvnFailed
-
-:: move dolphinscheduler-bin.tar.gz file to docker/build directory
-echo "move %cd%\dolphinscheduler-dist\target\apache-dolphinscheduler-%VERSION%-bin.tar.gz %cd%\docker\build\"
-move %cd%\dolphinscheduler-dist\target\apache-dolphinscheduler-%VERSION%-bin.tar.gz %cd%\docker\build\
-
-:: docker build
-echo "docker build --build-arg VERSION=%VERSION% -t %DOCKER_REPO%:%VERSION% %cd%\docker\build\"
-docker build --build-arg VERSION=%VERSION% -t %DOCKER_REPO%:%VERSION% %cd%\docker\build\
-if "%errorlevel%"=="1" goto :dockerBuildFailed
-
-echo "------ dolphinscheduler end - build -------"
-
-:mvnFailed
-echo "MAVEN PACKAGE FAILED!"
-
-:dockerBuildFailed
-echo "DOCKER BUILD FAILED!"
diff --git a/docker/build/hooks/push.bat b/docker/build/hooks/push.bat
deleted file mode 100644
index 458a693f97..0000000000
--- a/docker/build/hooks/push.bat
+++ /dev/null
@@ -1,23 +0,0 @@
-:: Licensed to the Apache Software Foundation (ASF) under one or more
-:: contributor license agreements. See the NOTICE file distributed with
-:: this work for additional information regarding copyright ownership.
-:: The ASF licenses this file to You under the Apache License, Version 2.0
-:: (the "License"); you may not use this file except in compliance with
-:: the License. You may obtain a copy of the License at
-::
-:: http://www.apache.org/licenses/LICENSE-2.0
-::
-:: Unless required by applicable law or agreed to in writing, software
-:: distributed under the License is distributed on an "AS IS" BASIS,
-:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-:: See the License for the specific language governing permissions and
-:: limitations under the License.
-::
-@echo off
-
-echo "------ push start -------"
-set
-
-docker push %DOCKER_REPO%:%VERSION%
-
-echo "------ push end -------"
diff --git a/docker/build/startup-init-conf.sh b/docker/build/startup-init-conf.sh
deleted file mode 100755
index ad61829074..0000000000
--- a/docker/build/startup-init-conf.sh
+++ /dev/null
@@ -1,143 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-set -e
-
-echo "init env variables"
-
-# Define parameters default value
-
-#============================================================================
-# Database
-#============================================================================
-[ "h2" == ${DATABASE_TYPE} ] || export DATABASE_TYPE=${DATABASE_TYPE:-"postgresql"}
-# export DATABASE_DRIVER=${DATABASE_DRIVER:-"org.postgresql.Driver"}
-[ "h2" == ${DATABASE_TYPE} ] || export DATABASE_HOST=${DATABASE_HOST:-"127.0.0.1"}
-[ "h2" == ${DATABASE_TYPE} ] || export DATABASE_PORT=${DATABASE_PORT:-"5432"}
-[ "h2" == ${DATABASE_TYPE} ] || export DATABASE_USERNAME=${DATABASE_USERNAME:-"root"}
-[ "h2" == ${DATABASE_TYPE} ] || export DATABASE_PASSWORD=${DATABASE_PASSWORD:-"root"}
-[ "h2" == ${DATABASE_TYPE} ] || export DATABASE_DATABASE=${DATABASE_DATABASE:-"dolphinscheduler"}
-[ "h2" == ${DATABASE_TYPE} ] || export DATABASE_PARAMS=${DATABASE_PARAMS:-"characterEncoding=utf8"}
-# transform params to SPRING_DATASOURCE_XXX
-[ "h2" == ${DATABASE_TYPE} ] || export SPRING_DATASOURCE_URL="jdbc:${DATABASE_TYPE}://${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DATABASE}?${DATABASE_PARAMS}"
-[ "h2" == ${DATABASE_TYPE} ] || export SPRING_DATASOURCE_USERNAME=${DATABASE_USERNAME}
-[ "h2" == ${DATABASE_TYPE} ] || export SPRING_DATASOURCE_PASSWORD=${DATABASE_PASSWORD}
-# export SPRING_DATASOURCE_DRIVER_CLASS_NAME=${DATABASE_DRIVER}
-
-
-#============================================================================
-# Registry
-#============================================================================
-export REGISTRY_PLUGIN_NAME=${REGISTRY_PLUGIN_NAME:-"zookeeper"}
-export REGISTRY_SERVERS=${REGISTRY_SERVERS:-"127.0.0.1:2181"}
-
-#============================================================================
-# Common
-#============================================================================
-# common opts
-export DOLPHINSCHEDULER_OPTS=${DOLPHINSCHEDULER_OPTS:-""}
-# common env
-export DATA_BASEDIR_PATH=${DATA_BASEDIR_PATH:-"/tmp/dolphinscheduler"}
-export RESOURCE_STORAGE_TYPE=${RESOURCE_STORAGE_TYPE:-"HDFS"}
-export RESOURCE_UPLOAD_PATH=${RESOURCE_UPLOAD_PATH:-"/dolphinscheduler"}
-export FS_DEFAULT_FS=${FS_DEFAULT_FS:-"file:///"}
-export FS_S3A_ENDPOINT=${FS_S3A_ENDPOINT:-"s3.xxx.amazonaws.com"}
-export FS_S3A_ACCESS_KEY=${FS_S3A_ACCESS_KEY:-"xxxxxxx"}
-export FS_S3A_SECRET_KEY=${FS_S3A_SECRET_KEY:-"xxxxxxx"}
-export HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE=${HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE:-"false"}
-export JAVA_SECURITY_KRB5_CONF_PATH=${JAVA_SECURITY_KRB5_CONF_PATH:-"/opt/krb5.conf"}
-export LOGIN_USER_KEYTAB_USERNAME=${LOGIN_USER_KEYTAB_USERNAME:-"hdfs@HADOOP.COM"}
-export LOGIN_USER_KEYTAB_PATH=${LOGIN_USER_KEYTAB_PATH:-"/opt/hdfs.keytab"}
-export KERBEROS_EXPIRE_TIME=${KERBEROS_EXPIRE_TIME:-"2"}
-export HDFS_ROOT_USER=${HDFS_ROOT_USER:-"hdfs"}
-export RESOURCE_MANAGER_HTTPADDRESS_PORT=${RESOURCE_MANAGER_HTTPADDRESS_PORT:-"8088"}
-export YARN_RESOURCEMANAGER_HA_RM_IDS=${YARN_RESOURCEMANAGER_HA_RM_IDS:-""}
-export YARN_APPLICATION_STATUS_ADDRESS=${YARN_APPLICATION_STATUS_ADDRESS:-"http://ds1:%s/ws/v1/cluster/apps/%s"}
-export YARN_JOB_HISTORY_STATUS_ADDRESS=${YARN_JOB_HISTORY_STATUS_ADDRESS:-"http://ds1:19888/ws/v1/history/mapreduce/jobs/%s"}
-export DATASOURCE_ENCRYPTION_ENABLE=${DATASOURCE_ENCRYPTION_ENABLE:-"false"}
-export DATASOURCE_ENCRYPTION_SALT=${DATASOURCE_ENCRYPTION_SALT:-"!@#$%^&*"}
-export SUDO_ENABLE=${SUDO_ENABLE:-"true"}
-# dolphinscheduler env
-export HADOOP_HOME=${HADOOP_HOME:-"/opt/soft/hadoop"}
-export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-"/opt/soft/hadoop/etc/hadoop"}
-export SPARK_HOME1=${SPARK_HOME1:-"/opt/soft/spark1"}
-export SPARK_HOME2=${SPARK_HOME2:-"/opt/soft/spark2"}
-export PYTHON_HOME=${PYTHON_HOME:-"/usr/bin/python"}
-export JAVA_HOME=${JAVA_HOME:-"/usr/local/openjdk-8"}
-export HIVE_HOME=${HIVE_HOME:-"/opt/soft/hive"}
-export FLINK_HOME=${FLINK_HOME:-"/opt/soft/flink"}
-export DATAX_HOME=${DATAX_HOME:-"/opt/soft/datax"}
-
-#============================================================================
-# Master Server
-#============================================================================
-export MASTER_SERVER_OPTS=${MASTER_SERVER_OPTS:-"-Xms1g -Xmx1g -Xmn512m"}
-export MASTER_EXEC_THREADS=${MASTER_EXEC_THREADS:-"100"}
-export MASTER_EXEC_TASK_NUM=${MASTER_EXEC_TASK_NUM:-"20"}
-export MASTER_DISPATCH_TASK_NUM=${MASTER_DISPATCH_TASK_NUM:-"3"}
-export MASTER_HOST_SELECTOR=${MASTER_HOST_SELECTOR:-"LowerWeight"}
-export MASTER_HEARTBEAT_INTERVAL=${MASTER_HEARTBEAT_INTERVAL:-"10"}
-export MASTER_TASK_COMMIT_RETRYTIMES=${MASTER_TASK_COMMIT_RETRYTIMES:-"5"}
-export MASTER_TASK_COMMIT_INTERVAL=${MASTER_TASK_COMMIT_INTERVAL:-"1000"}
-export MASTER_MAX_CPULOAD_AVG=${MASTER_MAX_CPULOAD_AVG:-"-1"}
-export MASTER_RESERVED_MEMORY=${MASTER_RESERVED_MEMORY:-"0.3"}
-
-#============================================================================
-# Worker Server
-#============================================================================
-export WORKER_SERVER_OPTS=${WORKER_SERVER_OPTS:-"-Xms1g -Xmx1g -Xmn512m"}
-export WORKER_EXEC_THREADS=${WORKER_EXEC_THREADS:-"100"}
-export WORKER_HEARTBEAT_INTERVAL=${WORKER_HEARTBEAT_INTERVAL:-"10"}
-export WORKER_HOST_WEIGHT=${WORKER_HOST_WEIGHT:-"100"}
-export WORKER_MAX_CPULOAD_AVG=${WORKER_MAX_CPULOAD_AVG:-"-1"}
-export WORKER_RESERVED_MEMORY=${WORKER_RESERVED_MEMORY:-"0.3"}
-export WORKER_GROUPS=${WORKER_GROUPS:-"default"}
-export ALERT_LISTEN_HOST=${ALERT_LISTEN_HOST:-"localhost"}
-
-#============================================================================
-# Alert Server
-#============================================================================
-export ALERT_SERVER_OPTS=${ALERT_SERVER_OPTS:-"-Xms512m -Xmx512m -Xmn256m"}
-
-#============================================================================
-# Api Server
-#============================================================================
-export API_SERVER_OPTS=${API_SERVER_OPTS:-"-Xms512m -Xmx512m -Xmn256m"}
-
-#============================================================================
-# Logger Server
-#============================================================================
-export LOGGER_SERVER_OPTS=${LOGGER_SERVER_OPTS:-"-Xms512m -Xmx512m -Xmn256m"}
-
-echo "generate dolphinscheduler config"
-ls ${DOLPHINSCHEDULER_HOME}/conf/ | grep ".tpl" | while read line; do
-eval "cat << EOF
-$(cat ${DOLPHINSCHEDULER_HOME}/conf/${line})
-EOF
-" > ${DOLPHINSCHEDULER_HOME}/conf/${line%.*}
-done
-
-# generate dolphinscheduler env
-DOLPHINSCHEDULER_ENV_PATH=${DOLPHINSCHEDULER_HOME}/conf/env/dolphinscheduler_env.sh
-if [ -r "${DOLPHINSCHEDULER_ENV_PATH}.tpl" ]; then
-eval "cat << EOF
-$(cat ${DOLPHINSCHEDULER_ENV_PATH}.tpl)
-EOF
-" > ${DOLPHINSCHEDULER_ENV_PATH}
-chmod +x ${DOLPHINSCHEDULER_ENV_PATH}
-fi
diff --git a/docker/build/startup.sh b/docker/build/startup.sh
deleted file mode 100755
index 7f3b7d0d20..0000000000
--- a/docker/build/startup.sh
+++ /dev/null
@@ -1,132 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-set -e
-
-export DOLPHINSCHEDULER_BIN=${DOLPHINSCHEDULER_HOME}/bin
-export MASTER_START_ENABLED=false
-export WORKER_START_ENABLED=false
-export API_START_ENABLED=false
-export ALERT_START_ENABLED=false
-export LOGGER_START_ENABLED=false
-export STANDALONE_START_ENABLED=false
-
-# wait database
-waitDatabase() {
- echo "try to connect ${DATABASE_TYPE} ..."
- while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do
- local counter=$((counter+1))
- if [ $counter == 30 ]; then
- echo "Error: Couldn't connect to ${DATABASE_TYPE}."
- exit 1
- fi
- echo "Trying to connect to ${DATABASE_TYPE} at ${DATABASE_HOST}:${DATABASE_PORT}. Attempt $counter."
- sleep 5
- done
- echo "${DATABASE_TYPE} connection is ok"
-}
-
-# init database
-initDatabase() {
- echo "import sql data"
- ${DOLPHINSCHEDULER_HOME}/script/create-dolphinscheduler.sh
-}
-
-# wait zk
-waitZK() {
- echo "try to connect zookeeper ..."
- echo "${ZOOKEEPER_QUORUM}" | awk -F ',' 'BEGIN{ i=1 }{ while( i <= NF ){ print $i; i++ } }' | while read line; do
- while ! nc -z ${line%:*} ${line#*:}; do
- local counter=$((counter+1))
- if [ $counter == 30 ]; then
- echo "Error: Couldn't connect to zookeeper."
- exit 1
- fi
- echo "Trying to connect to zookeeper at ${line}. Attempt $counter."
- sleep 5
- done
- done
- echo "zookeeper connection is ok"
-}
-
-# print usage
-printUsage() {
- echo -e "Dolphin Scheduler is a distributed and easy-to-expand visual DAG workflow scheduling system,"
- echo -e "dedicated to solving the complex dependencies in data processing, making the scheduling system out of the box for data processing.\n"
- echo -e "Usage: [ all | master-server | worker-server | api-server | alert-server | standalone-server ]\n"
- printf "%-13s: %s\n" "all" "Run master-server, worker-server, api-server and alert-server"
- printf "%-13s: %s\n" "master-server" "MasterServer is mainly responsible for DAG task split, task submission monitoring."
- printf "%-13s: %s\n" "worker-server" "WorkerServer is mainly responsible for task execution and providing log services."
- printf "%-13s: %s\n" "api-server" "ApiServer is mainly responsible for processing requests and providing the front-end UI layer."
- printf "%-13s: %s\n" "alert-server" "AlertServer mainly include Alarms."
- printf "%-13s: %s\n" "standalone-server" "Standalone server that uses embedded zookeeper and database, only for testing and demostration."
-}
-
-# init config file
-source /root/startup-init-conf.sh
-
-case "$1" in
- (all)
- waitZK
- waitDatabase
- initDatabase
- export MASTER_START_ENABLED=true
- export WORKER_START_ENABLED=true
- export API_START_ENABLED=true
- export ALERT_START_ENABLED=true
- export LOGGER_START_ENABLED=true
- ;;
- (master-server)
- waitZK
- waitDatabase
- export MASTER_START_ENABLED=true
- ;;
- (worker-server)
- waitZK
- waitDatabase
- export WORKER_START_ENABLED=true
- export LOGGER_START_ENABLED=true
- ;;
- (api-server)
- waitZK
- waitDatabase
- initDatabase
- export API_START_ENABLED=true
- ;;
- (alert-server)
- waitDatabase
- export ALERT_START_ENABLED=true
- ;;
- (standalone-server)
- export STANDALONE_START_ENABLED=true
- ;;
- (help)
- printUsage
- exit 1
- ;;
- (*)
- printUsage
- exit 1
- ;;
-esac
-
-# init directories
-mkdir -p ${DOLPHINSCHEDULER_HOME}/logs
-
-# start supervisord
-supervisord -n -u root
diff --git a/docker/docker-swarm/check b/docker/docker-swarm/check
deleted file mode 100755
index 66484ae0c5..0000000000
--- a/docker/docker-swarm/check
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-echo "------- check server status -------"
-cd $(dirname $0)
-until [ $(docker-compose top | grep java | grep "dolphinscheduler" | awk -F 'classpath ' '{print $2}' | awk '{print $2}' | sort | uniq -c | wc -l) -eq 5 ]; do
- counter=$((counter+1))
- if [ $counter -eq 30 ]; then
- echo "error: all servers start failed"
- exit 1
- fi
- echo "waiting for all servers started ..."
- sleep 5
-done
-echo "all servers start successfully"
-
-counter=0
-until docker logs docker-swarm_dolphinscheduler-api_1 2>&1 | grep '0.0.0.0:12345' &>/dev/null; do
- counter=$((counter+1))
- if [ $counter -eq 30 ]; then
- echo "error: api server is not ready"
- exit 1
- fi
- echo "waiting for api server ready ..."
- sleep 5
-done
-echo "api server is ready"
diff --git a/docker/docker-swarm/config.env.sh b/docker/docker-swarm/config.env.sh
deleted file mode 100755
index e0193de986..0000000000
--- a/docker/docker-swarm/config.env.sh
+++ /dev/null
@@ -1,121 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-#============================================================================
-# Database
-#============================================================================
-# postgresql
-DATABASE_TYPE=postgresql
-DATABASE_DRIVER=org.postgresql.Driver
-DATABASE_HOST=dolphinscheduler-postgresql
-DATABASE_PORT=5432
-DATABASE_USERNAME=root
-DATABASE_PASSWORD=root
-DATABASE_DATABASE=dolphinscheduler
-DATABASE_PARAMS=characterEncoding=utf8
-# mysql
-# DATABASE_TYPE=mysql
-# DATABASE_DRIVER=com.mysql.cj.jdbc.Driver
-# DATABASE_HOST=dolphinscheduler-mysql
-# DATABASE_PORT=3306
-# DATABASE_USERNAME=root
-# DATABASE_PASSWORD=root
-# DATABASE_DATABASE=dolphinscheduler
-# DATABASE_PARAMS=useUnicode=true&characterEncoding=UTF-8
-
-#============================================================================
-# Registry
-#============================================================================
-REGISTRY_PLUGIN_NAME=zookeeper
-REGISTRY_SERVERS=dolphinscheduler-zookeeper:2181
-
-#============================================================================
-# Common
-#============================================================================
-# common opts
-DOLPHINSCHEDULER_OPTS=
-# common env
-DATA_BASEDIR_PATH=/tmp/dolphinscheduler
-RESOURCE_STORAGE_TYPE=HDFS
-RESOURCE_UPLOAD_PATH=/dolphinscheduler
-FS_DEFAULT_FS=file:///
-FS_S3A_ENDPOINT=s3.xxx.amazonaws.com
-FS_S3A_ACCESS_KEY=xxxxxxx
-FS_S3A_SECRET_KEY=xxxxxxx
-HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE=false
-JAVA_SECURITY_KRB5_CONF_PATH=/opt/krb5.conf
-LOGIN_USER_KEYTAB_USERNAME=hdfs@HADOOP.COM
-LOGIN_USER_KEYTAB_PATH=/opt/hdfs.keytab
-KERBEROS_EXPIRE_TIME=2
-HDFS_ROOT_USER=hdfs
-RESOURCE_MANAGER_HTTPADDRESS_PORT=8088
-YARN_RESOURCEMANAGER_HA_RM_IDS=
-YARN_APPLICATION_STATUS_ADDRESS=http://ds1:%s/ws/v1/cluster/apps/%s
-YARN_JOB_HISTORY_STATUS_ADDRESS=http://ds1:19888/ws/v1/history/mapreduce/jobs/%s
-DATASOURCE_ENCRYPTION_ENABLE=false
-DATASOURCE_ENCRYPTION_SALT=!@#$%^&*
-SUDO_ENABLE=true
-# dolphinscheduler env
-HADOOP_HOME=/opt/soft/hadoop
-HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop
-SPARK_HOME1=/opt/soft/spark1
-SPARK_HOME2=/opt/soft/spark2
-PYTHON_HOME=/usr/bin/python
-JAVA_HOME=/usr/local/openjdk-8
-HIVE_HOME=/opt/soft/hive
-FLINK_HOME=/opt/soft/flink
-DATAX_HOME=/opt/soft/datax
-
-#============================================================================
-# Master Server
-#============================================================================
-MASTER_SERVER_OPTS=-Xms1g -Xmx1g -Xmn512m
-MASTER_EXEC_THREADS=100
-MASTER_EXEC_TASK_NUM=20
-MASTER_DISPATCH_TASK_NUM=3
-MASTER_HOST_SELECTOR=LowerWeight
-MASTER_HEARTBEAT_INTERVAL=10
-MASTER_TASK_COMMIT_RETRYTIMES=5
-MASTER_TASK_COMMIT_INTERVAL=1000
-MASTER_MAX_CPULOAD_AVG=-1
-MASTER_RESERVED_MEMORY=0.3
-
-#============================================================================
-# Worker Server
-#============================================================================
-WORKER_SERVER_OPTS=-Xms1g -Xmx1g -Xmn512m
-WORKER_EXEC_THREADS=100
-WORKER_HEARTBEAT_INTERVAL=10
-WORKER_HOST_WEIGHT=100
-WORKER_MAX_CPULOAD_AVG=-1
-WORKER_RESERVED_MEMORY=0.3
-WORKER_GROUPS=default
-ALERT_LISTEN_HOST=dolphinscheduler-alert
-
-#============================================================================
-# Alert Server
-#============================================================================
-ALERT_SERVER_OPTS=-Xms512m -Xmx512m -Xmn256m
-
-#============================================================================
-# Api Server
-#============================================================================
-API_SERVER_OPTS=-Xms512m -Xmx512m -Xmn256m
-
-#============================================================================
-# Logger Server
-#============================================================================
-LOGGER_SERVER_OPTS=-Xms512m -Xmx512m -Xmn256m
diff --git a/docker/docker-swarm/docker-compose.yml b/docker/docker-swarm/docker-compose.yml
deleted file mode 100644
index 04ac8b3e8b..0000000000
--- a/docker/docker-swarm/docker-compose.yml
+++ /dev/null
@@ -1,143 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-version: "3.1"
-
-services:
-
- dolphinscheduler-postgresql:
- image: bitnami/postgresql:11.11.0
- environment:
- TZ: Asia/Shanghai
- POSTGRESQL_USERNAME: root
- POSTGRESQL_PASSWORD: root
- POSTGRESQL_DATABASE: dolphinscheduler
- volumes:
- - dolphinscheduler-postgresql:/bitnami/postgresql
- restart: unless-stopped
- networks:
- - dolphinscheduler
-
- dolphinscheduler-zookeeper:
- image: bitnami/zookeeper:3.6.2
- environment:
- TZ: Asia/Shanghai
- ALLOW_ANONYMOUS_LOGIN: "yes"
- ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons
- volumes:
- - dolphinscheduler-zookeeper:/bitnami/zookeeper
- restart: unless-stopped
- networks:
- - dolphinscheduler
-
- dolphinscheduler-api:
- image: dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler:2.0.0-SNAPSHOT
- command: api-server
- ports:
- - 12345:12345
- environment:
- TZ: Asia/Shanghai
- env_file: config.env.sh
- healthcheck:
- test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"]
- interval: 30s
- timeout: 5s
- retries: 3
- depends_on:
- - dolphinscheduler-postgresql
- - dolphinscheduler-zookeeper
- volumes:
- - dolphinscheduler-logs:/opt/dolphinscheduler/logs
- - dolphinscheduler-shared-local:/opt/soft
- - dolphinscheduler-resource-local:/dolphinscheduler
- restart: unless-stopped
- networks:
- - dolphinscheduler
-
- dolphinscheduler-alert:
- image: dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler:2.0.0-SNAPSHOT
- command: alert-server
- environment:
- TZ: Asia/Shanghai
- env_file: config.env.sh
- healthcheck:
- test: ["CMD", "/root/checkpoint.sh", "AlertServer"]
- interval: 30s
- timeout: 5s
- retries: 3
- depends_on:
- - dolphinscheduler-postgresql
- volumes:
- - dolphinscheduler-logs:/opt/dolphinscheduler/logs
- restart: unless-stopped
- networks:
- - dolphinscheduler
-
- dolphinscheduler-master:
- image: dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler:2.0.0-SNAPSHOT
- command: master-server
- environment:
- TZ: Asia/Shanghai
- env_file: config.env.sh
- healthcheck:
- test: ["CMD", "/root/checkpoint.sh", "MasterServer"]
- interval: 30s
- timeout: 5s
- retries: 3
- depends_on:
- - dolphinscheduler-postgresql
- - dolphinscheduler-zookeeper
- volumes:
- - dolphinscheduler-logs:/opt/dolphinscheduler/logs
- - dolphinscheduler-shared-local:/opt/soft
- restart: unless-stopped
- networks:
- - dolphinscheduler
-
- dolphinscheduler-worker:
- image: dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler:2.0.0-SNAPSHOT
- command: worker-server
- environment:
- TZ: Asia/Shanghai
- env_file: config.env.sh
- healthcheck:
- test: ["CMD", "/root/checkpoint.sh", "WorkerServer"]
- interval: 30s
- timeout: 5s
- retries: 3
- depends_on:
- - dolphinscheduler-postgresql
- - dolphinscheduler-zookeeper
- volumes:
- - dolphinscheduler-worker-data:/tmp/dolphinscheduler
- - dolphinscheduler-logs:/opt/dolphinscheduler/logs
- - dolphinscheduler-shared-local:/opt/soft
- - dolphinscheduler-resource-local:/dolphinscheduler
- restart: unless-stopped
- networks:
- - dolphinscheduler
-
-networks:
- dolphinscheduler:
- driver: bridge
-
-volumes:
- dolphinscheduler-postgresql:
- dolphinscheduler-zookeeper:
- dolphinscheduler-worker-data:
- dolphinscheduler-logs:
- dolphinscheduler-shared-local:
- dolphinscheduler-resource-local:
diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml
index ac71840e93..bac21d2c18 100644
--- a/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml
+++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml
@@ -85,18 +85,17 @@
com.google.guava
guava
-
-
- jsr305
- com.google.code.findbugs
-
-
ch.qos.logback
logback-classic
+
+ org.codehaus.janino
+ janino
+
+
org.apache.poi
@@ -124,11 +123,43 @@
*.yaml
- *.yml
*.xml
+
+ maven-assembly-plugin
+
+
+ dolphinscheduler-alert-server
+ package
+
+ single
+
+
+ alert-server
+
+ src/main/assembly/dolphinscheduler-alert-server.xml
+
+ false
+
+
+
+
+
+
+
+ docker
+
+
+
+ org.codehaus.mojo
+ exec-maven-plugin
+
+
+
+
+
diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/assembly/dolphinscheduler-alert-server.xml b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/assembly/dolphinscheduler-alert-server.xml
new file mode 100644
index 0000000000..259051fed4
--- /dev/null
+++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/assembly/dolphinscheduler-alert-server.xml
@@ -0,0 +1,64 @@
+
+
+
+ dolphinscheduler-alert-server
+
+ dir
+
+ false
+ alert-server
+
+
+ ${basedir}/src/main/resources
+
+ *.yaml
+ *.xml
+
+ conf
+
+
+ ${basedir}/src/main/bin
+ bin
+ 0755
+ 0755
+
+
+ ${basedir}/../../script/env
+ bin
+
+ dolphinscheduler_env.sh
+
+ 0755
+ 0755
+
+
+ ${basedir}/../../dolphinscheduler-common/src/main/resources
+
+ **/*.properties
+
+ conf
+
+
+
+
+ libs
+
+
+
diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/bin/start.sh b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/bin/start.sh
new file mode 100644
index 0000000000..eda02bb6c9
--- /dev/null
+++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/bin/start.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+BIN_DIR=$(dirname $0)
+DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/..; pwd)}
+
+source "$BIN_DIR/dolphinscheduler_env.sh"
+
+JAVA_OPTS=${JAVA_OPTS:-"-server -Xms1g -Xmx1g -Xmn512m -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"}
+
+if [[ "$DOCKER" == "true" ]]; then
+ JAVA_OPTS="${JAVA_OPTS} -XX:-UseContainerSupport"
+fi
+
+java $JAVA_OPTS \
+ -cp "$DOLPHINSCHEDULER_HOME/conf":"$DOLPHINSCHEDULER_HOME/libs/*" \
+ org.apache.dolphinscheduler.alert.AlertServer
diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/docker/Dockerfile b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/docker/Dockerfile
new file mode 100644
index 0000000000..84df818d9f
--- /dev/null
+++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/docker/Dockerfile
@@ -0,0 +1,34 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+FROM openjdk:8-jre-slim-buster
+
+ENV DOCKER true
+ENV TZ Asia/Shanghai
+ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler
+
+RUN apt update ; \
+ apt install -y curl ; \
+ rm -rf /var/lib/apt/lists/*
+
+WORKDIR $DOLPHINSCHEDULER_HOME
+
+ADD ./target/alert-server $DOLPHINSCHEDULER_HOME
+
+EXPOSE 50052 50053
+
+CMD [ "/bin/bash", "./bin/start.sh" ]
diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertConfig.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertConfig.java
new file mode 100644
index 0000000000..f7c9daaa7f
--- /dev/null
+++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertConfig.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.alert;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.stereotype.Component;
+
+@Component
+@ConfigurationProperties("alert")
+public final class AlertConfig {
+ private int port;
+
+ public int getPort() {
+ return port;
+ }
+
+ public void setPort(final int port) {
+ this.port = port;
+ }
+}
diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java
index 901172b057..1d2ffb114e 100644
--- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java
+++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java
@@ -17,10 +17,7 @@
package org.apache.dolphinscheduler.alert;
-import static org.apache.dolphinscheduler.common.Constants.ALERT_RPC_PORT;
-
import org.apache.dolphinscheduler.common.thread.Stopper;
-import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.dao.AlertDao;
import org.apache.dolphinscheduler.dao.PluginDao;
import org.apache.dolphinscheduler.dao.entity.Alert;
@@ -38,15 +35,13 @@ import javax.annotation.PreDestroy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.SpringApplication;
-import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.ComponentScan;
-@EnableAutoConfiguration
-@ComponentScan(value = {
- "org.apache.dolphinscheduler.alert",
- "org.apache.dolphinscheduler.dao"
-})
+@SpringBootApplication
+@ComponentScan("org.apache.dolphinscheduler")
public class AlertServer implements Closeable {
private static final Logger logger = LoggerFactory.getLogger(AlertServer.class);
@@ -58,6 +53,9 @@ public class AlertServer implements Closeable {
private NettyRemotingServer server;
+ @Autowired
+ private AlertConfig config;
+
public AlertServer(PluginDao pluginDao, AlertDao alertDao, AlertPluginManager alertPluginManager, AlertSender alertSender, AlertRequestProcessor alertRequestProcessor) {
this.pluginDao = pluginDao;
this.alertDao = alertDao;
@@ -101,7 +99,7 @@ public class AlertServer implements Closeable {
private void startServer() {
NettyServerConfig serverConfig = new NettyServerConfig();
- serverConfig.setListenPort(PropertyUtils.getInt(ALERT_RPC_PORT, 50052));
+ serverConfig.setListenPort(config.getPort());
server = new NettyRemotingServer(serverConfig);
server.registerProcessor(CommandType.ALERT_SEND_REQUEST, alertRequestProcessor);
diff --git a/dolphinscheduler-api/src/main/resources/application-api.yaml b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml
similarity index 54%
rename from dolphinscheduler-api/src/main/resources/application-api.yaml
rename to dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml
index ce76b9a9de..d2bb0cfac0 100644
--- a/dolphinscheduler-api/src/main/resources/application-api.yaml
+++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml
@@ -15,29 +15,30 @@
# limitations under the License.
#
-server:
- port: 12345
- servlet:
- session:
- timeout: 120m
- context-path: /dolphinscheduler/
- compression:
- enabled: true
- mime-types: text/html,text/xml,text/plain,text/css,text/javascript,application/javascript,application/json,application/xml
- jetty:
- max-http-form-post-size: 5000000
-
spring:
application:
- name: api-server
- jackson:
- time-zone: GMT+8
- servlet:
- multipart:
- max-file-size: 1024MB
- max-request-size: 1024MB
- messages:
- basename: i18n/messages
+ name: alert-server
+ main:
+ banner-mode: off
+ datasource:
+ driver-class-name: org.postgresql.Driver
+ url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler
+ username: root
+ password: root
+ hikari:
+ connection-test-query: select 1
+ minimum-idle: 5
+ auto-commit: true
+ validation-timeout: 3000
+ pool-name: DolphinScheduler
+ maximum-pool-size: 50
+ connection-timeout: 30000
+ idle-timeout: 600000
+ leak-detection-threshold: 0
+ initialization-fail-timeout: 1
+
+server:
+ port: 50053
management:
endpoints:
@@ -47,3 +48,17 @@ management:
metrics:
tags:
application: ${spring.application.name}
+
+alert:
+ port: 50052
+
+# Override by profile
+
+---
+spring:
+ config:
+ activate:
+ on-profile: mysql
+ datasource:
+ driver-class-name: com.mysql.jdbc.Driver
+ url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8
diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/logback-alert.xml b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/logback-spring.xml
similarity index 93%
rename from dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/logback-alert.xml
rename to dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/logback-spring.xml
index 48b3c35bff..de3654fedb 100644
--- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/logback-alert.xml
+++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/logback-spring.xml
@@ -16,10 +16,9 @@
~ limitations under the License.
-->
-
-
+
@@ -45,8 +44,11 @@
-
+
+
+
+
+
-
diff --git a/dolphinscheduler-alert/pom.xml b/dolphinscheduler-alert/pom.xml
index a47b25055d..2a7c264a18 100644
--- a/dolphinscheduler-alert/pom.xml
+++ b/dolphinscheduler-alert/pom.xml
@@ -41,10 +41,5 @@
org.slf4j
slf4j-api
-
- org.springframework.boot
- spring-boot-starter-test
- test
-
diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml
index 6a857b97a4..8c1b900fac 100644
--- a/dolphinscheduler-api/pom.xml
+++ b/dolphinscheduler-api/pom.xml
@@ -45,6 +45,11 @@
${project.version}
+
+ org.apache.dolphinscheduler
+ dolphinscheduler-ui
+
+
org.springframework.boot
@@ -112,7 +117,7 @@
com.zaxxer
- HikariCP-java6
+ HikariCP-java7
@@ -212,6 +217,11 @@
hadoop-aws
+
+ org.codehaus.janino
+ janino
+
+
org.springframework.boot
@@ -239,11 +249,10 @@
${project.basedir}/../dolphinscheduler-dao/src/main/resources
sql/**
- *.yaml
- ${project.basedir}/../dolphinscheduler-service/src/main/resources
+ ${project.basedir}/src/test/resources
@@ -253,11 +262,43 @@
*.yaml
- *.yml
*.xml
+
+ maven-assembly-plugin
+
+
+ dolphinscheduler-api-server
+ package
+
+ single
+
+
+ api-server
+
+ src/main/assembly/dolphinscheduler-api-server.xml
+
+ false
+
+
+
+
+
+
+
+ docker
+
+
+
+ org.codehaus.mojo
+ exec-maven-plugin
+
+
+
+
+
diff --git a/dolphinscheduler-api/src/main/assembly/dolphinscheduler-api-server.xml b/dolphinscheduler-api/src/main/assembly/dolphinscheduler-api-server.xml
new file mode 100644
index 0000000000..792f92179f
--- /dev/null
+++ b/dolphinscheduler-api/src/main/assembly/dolphinscheduler-api-server.xml
@@ -0,0 +1,68 @@
+
+
+
+ dolphinscheduler-api-server
+
+ dir
+
+ false
+ api-server
+
+
+ ${basedir}/src/main/resources
+
+ *.yaml
+ *.xml
+
+ conf
+
+
+ ${basedir}/src/main/bin
+ bin
+ 0755
+ 0755
+
+
+ ${basedir}/../script/env
+ bin
+
+ dolphinscheduler_env.sh
+
+ 0755
+ 0755
+
+
+ ${basedir}/../dolphinscheduler-common/src/main/resources
+
+ **/*.properties
+
+ conf
+
+
+ ${basedir}/../dolphinscheduler-ui/dist
+ ./ui
+
+
+
+
+ libs
+
+
+
diff --git a/dolphinscheduler-api/src/main/bin/start.sh b/dolphinscheduler-api/src/main/bin/start.sh
new file mode 100644
index 0000000000..66655069c1
--- /dev/null
+++ b/dolphinscheduler-api/src/main/bin/start.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+BIN_DIR=$(dirname $0)
+DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/..; pwd)}
+
+source "$BIN_DIR/dolphinscheduler_env.sh"
+
+JAVA_OPTS=${JAVA_OPTS:-"-server -Xms1g -Xmx1g -Xmn512m -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"}
+
+if [[ "$DOCKER" == "true" ]]; then
+ JAVA_OPTS="${JAVA_OPTS} -XX:-UseContainerSupport"
+fi
+
+java $JAVA_OPTS \
+ -cp "$DOLPHINSCHEDULER_HOME/conf":"$DOLPHINSCHEDULER_HOME/libs/*" \
+ org.apache.dolphinscheduler.api.ApiApplicationServer
diff --git a/dolphinscheduler-api/src/main/docker/Dockerfile b/dolphinscheduler-api/src/main/docker/Dockerfile
new file mode 100644
index 0000000000..9a1a8c0ca1
--- /dev/null
+++ b/dolphinscheduler-api/src/main/docker/Dockerfile
@@ -0,0 +1,34 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+FROM openjdk:8-jre-slim-buster
+
+ENV DOCKER true
+ENV TZ Asia/Shanghai
+ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler
+
+RUN apt update ; \
+ apt install -y curl ; \
+ rm -rf /var/lib/apt/lists/*
+
+WORKDIR $DOLPHINSCHEDULER_HOME
+
+ADD ./target/api-server $DOLPHINSCHEDULER_HOME
+
+EXPOSE 12345
+
+CMD [ "/bin/bash", "./bin/start.sh" ]
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java
index 9d04857d90..7981f8a48d 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java
@@ -17,25 +17,18 @@
package org.apache.dolphinscheduler.api;
-import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
-import org.springframework.boot.builder.SpringApplicationBuilder;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.web.servlet.ServletComponentScan;
-import org.springframework.boot.web.servlet.support.SpringBootServletInitializer;
import org.springframework.context.annotation.ComponentScan;
-import org.springframework.context.annotation.FilterType;
-@EnableAutoConfiguration
@ServletComponentScan
-@ComponentScan(value = "org.apache.dolphinscheduler", excludeFilters = {
- @ComponentScan.Filter(type = FilterType.REGEX, pattern = {
- "org.apache.dolphinscheduler.server.*",
- "org.apache.dolphinscheduler.alert.*"
- })
-})
-public class ApiApplicationServer extends SpringBootServletInitializer {
+@SpringBootApplication
+@ComponentScan("org.apache.dolphinscheduler")
+public class ApiApplicationServer {
public static void main(String[] args) {
- new SpringApplicationBuilder(ApiApplicationServer.class).profiles("api").run(args);
+ SpringApplication.run(ApiApplicationServer.class);
}
}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/aspect/CacheEvictAspect.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/aspect/CacheEvictAspect.java
index 39d697545c..b036f50bca 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/aspect/CacheEvictAspect.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/aspect/CacheEvictAspect.java
@@ -127,7 +127,7 @@ public class CacheEvictAspect {
SpelExpressionParser spelParser = new SpelExpressionParser();
EvaluationContext ctx = new StandardEvaluationContext();
for (int i = 0; i < paramNameList.size(); i++) {
- ctx.setVariable(paramNameList.get(i), paramList.get(i));
+ ctx.setVariable("p" + i, paramList.get(i));
}
Object obj = spelParser.parseExpression(key).getValue(ctx);
if (null == obj) {
diff --git a/dolphinscheduler-api/src/main/resources/application.yaml b/dolphinscheduler-api/src/main/resources/application.yaml
new file mode 100644
index 0000000000..53447b96f0
--- /dev/null
+++ b/dolphinscheduler-api/src/main/resources/application.yaml
@@ -0,0 +1,116 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+server:
+ port: 12345
+ servlet:
+ session:
+ timeout: 120m
+ context-path: /dolphinscheduler/
+ compression:
+ enabled: true
+ mime-types: text/html,text/xml,text/plain,text/css,text/javascript,application/javascript,application/json,application/xml
+ jetty:
+ max-http-form-post-size: 5000000
+
+spring:
+ application:
+ name: api-server
+ main:
+ banner-mode: off
+ jackson:
+ time-zone: GMT+8
+ servlet:
+ multipart:
+ max-file-size: 1024MB
+ max-request-size: 1024MB
+ messages:
+ basename: i18n/messages
+ datasource:
+ driver-class-name: org.postgresql.Driver
+ url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler
+ username: root
+ password: root
+ hikari:
+ connection-test-query: select 1
+ minimum-idle: 5
+ auto-commit: true
+ validation-timeout: 3000
+ pool-name: DolphinScheduler
+ maximum-pool-size: 50
+ connection-timeout: 30000
+ idle-timeout: 600000
+ leak-detection-threshold: 0
+ initialization-fail-timeout: 1
+ quartz:
+ auto-startup: false
+ job-store-type: jdbc
+ jdbc:
+ initialize-schema: never
+ properties:
+ org.quartz.threadPool:threadPriority: 5
+ org.quartz.jobStore.isClustered: true
+ org.quartz.jobStore.class: org.quartz.impl.jdbcjobstore.JobStoreTX
+ org.quartz.scheduler.instanceId: AUTO
+ org.quartz.jobStore.tablePrefix: QRTZ_
+ org.quartz.jobStore.acquireTriggersWithinLock: true
+ org.quartz.scheduler.instanceName: DolphinScheduler
+ org.quartz.threadPool.class: org.quartz.simpl.SimpleThreadPool
+ org.quartz.jobStore.useProperties: false
+ org.quartz.threadPool.makeThreadsDaemons: true
+ org.quartz.threadPool.threadCount: 25
+ org.quartz.jobStore.misfireThreshold: 60000
+ org.quartz.scheduler.makeSchedulerThreadDaemon: true
+ org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.PostgreSQLDelegate
+ org.quartz.jobStore.clusterCheckinInterval: 5000
+
+management:
+ endpoints:
+ web:
+ exposure:
+ include: '*'
+ metrics:
+ tags:
+ application: ${spring.application.name}
+
+registry:
+ type: zookeeper
+ zookeeper:
+ namespace: dolphinscheduler
+ connect-string: localhost:2181
+ retry-policy:
+ base-sleep-time: 60ms
+ max-sleep: 300ms
+ max-retries: 5
+ session-timeout: 30s
+ connection-timeout: 9s
+ block-until-connected: 600ms
+ digest: ~
+
+# Override by profile
+
+---
+spring:
+ config:
+ activate:
+ on-profile: mysql
+ datasource:
+ driver-class-name: com.mysql.jdbc.Driver
+ url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8
+ quartz:
+ properties:
+ org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.StdJDBCDelegate
diff --git a/dolphinscheduler-api/src/main/resources/logback-api.xml b/dolphinscheduler-api/src/main/resources/logback-spring.xml
similarity index 90%
rename from dolphinscheduler-api/src/main/resources/logback-api.xml
rename to dolphinscheduler-api/src/main/resources/logback-spring.xml
index ed655a0870..55badff5df 100644
--- a/dolphinscheduler-api/src/main/resources/logback-api.xml
+++ b/dolphinscheduler-api/src/main/resources/logback-spring.xml
@@ -16,10 +16,9 @@
~ limitations under the License.
-->
-
-
-
+
+
@@ -29,7 +28,6 @@
-
${log.base}/dolphinscheduler-api.log
@@ -47,15 +45,17 @@
UTF-8
-
-
+
+
+
+
+
-
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AbstractControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AbstractControllerTest.java
index 75038295bf..62e64acec5 100644
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AbstractControllerTest.java
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AbstractControllerTest.java
@@ -53,7 +53,6 @@ import org.springframework.test.web.servlet.MockMvc;
/**
* abstract controller test
*/
-@ActiveProfiles(value = {ProfileType.H2})
@RunWith(SpringRunner.class)
@SpringBootTest(classes = {ApiApplicationServer.class, DaoConfiguration.class, RegistryServer.class})
@AutoConfigureMockMvc
@@ -112,12 +111,11 @@ public abstract class AbstractControllerTest {
}
@Configuration
- @Profile(ProfileType.H2)
public static class RegistryServer {
@PostConstruct
public void startEmbedRegistryServer() throws Exception {
final TestingServer server = new TestingServer(true);
- System.setProperty("registry.servers", server.getConnectString());
+ System.setProperty("registry.zookeeper.connect-string", server.getConnectString());
}
}
}
diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application-alert.yaml b/dolphinscheduler-api/src/test/resources/application.yaml
similarity index 73%
rename from dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application-alert.yaml
rename to dolphinscheduler-api/src/test/resources/application.yaml
index 8e5c4c5355..95a484cf3e 100644
--- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application-alert.yaml
+++ b/dolphinscheduler-api/src/test/resources/application.yaml
@@ -16,17 +16,13 @@
#
spring:
- application:
- name: alert-server
+ main:
+ banner-mode: off
+ datasource:
+ driver-class-name: org.h2.Driver
+ url: jdbc:h2:mem:dolphinscheduler;MODE=MySQL;DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=true;INIT=runscript from 'classpath:sql/dolphinscheduler_h2.sql'
+ username: sa
+ password: ""
-server:
- port: 50053
-
-management:
- endpoints:
- web:
- exposure:
- include: '*'
- metrics:
- tags:
- application: ${spring.application.name}
+registry:
+ type: zookeeper
diff --git a/dolphinscheduler-common/pom.xml b/dolphinscheduler-common/pom.xml
index ac98e9fe1e..12f3ab5b94 100644
--- a/dolphinscheduler-common/pom.xml
+++ b/dolphinscheduler-common/pom.xml
@@ -46,13 +46,6 @@
com.google.guava
guava
- provided
-
-
- jsr305
- com.google.code.findbugs
-
-
org.mockito
@@ -139,10 +132,6 @@
com.google.code.gson
gson
-
- org.apache.commons
- commons-math3
-
xmlenc
xmlenc
@@ -159,10 +148,6 @@
org.apache.zookeeper
zookeeper
-
- jsr305
- com.google.code.findbugs
-
javax.servlet.jsp
jsp-api
@@ -179,10 +164,6 @@
jersey-core
com.sun.jersey
-
- xz
- org.tukaani
-
@@ -346,10 +327,6 @@
com.google.code.gson
gson
-
- com.google.code.findbugs
- jsr305
-
io.dropwizard.metrics
metrics-core
@@ -362,10 +339,6 @@
org.apache.avro
avro
-
- org.apache.commons
- commons-compress
-
org.apache.curator
curator-client
@@ -579,22 +552,11 @@
com.baomidou
mybatis-plus-annotation
${mybatis-plus.version}
- compile
com.github.rholder
guava-retrying
-
-
- com.google.guava
- guava
-
-
- com.google.code.findbugs
- jsr305
-
-
io.netty
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
index 5ce364ca75..7d6428c4cc 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
@@ -38,11 +38,6 @@ public final class Constants {
*/
public static final String COMMON_PROPERTIES_PATH = "/common.properties";
- /**
- * alert properties
- */
- public static final String ALERT_RPC_PORT = "alert.rpc.port";
-
/**
* registry properties
*/
diff --git a/dolphinscheduler-dao/pom.xml b/dolphinscheduler-dao/pom.xml
index d3b1639b54..17e665d76e 100644
--- a/dolphinscheduler-dao/pom.xml
+++ b/dolphinscheduler-dao/pom.xml
@@ -34,6 +34,21 @@
dolphinscheduler-common
+
+ org.springframework.boot
+ spring-boot-starter
+
+
+ org.springframework.boot
+ spring-boot-starter-tomcat
+
+
+ log4j-to-slf4j
+ org.apache.logging.log4j
+
+
+
+
com.zaxxer
HikariCP
@@ -103,17 +118,11 @@
org.yaml
snakeyaml
-
- org.springframework
- spring-test
- test
-
org.springframework.boot
spring-boot-starter-test
test
-
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SpringConnectionFactory.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SpringConnectionFactory.java
index 2b4f99ceaa..516e16ad42 100644
--- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SpringConnectionFactory.java
+++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SpringConnectionFactory.java
@@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.dao.datasource;
import org.apache.ibatis.mapping.DatabaseIdProvider;
import org.apache.ibatis.mapping.VendorDatabaseIdProvider;
-import org.apache.ibatis.session.SqlSession;
import org.apache.ibatis.session.SqlSessionFactory;
import org.apache.ibatis.type.JdbcType;
@@ -27,7 +26,6 @@ import java.util.Properties;
import javax.sql.DataSource;
-import org.mybatis.spring.SqlSessionTemplate;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
@@ -69,7 +67,7 @@ public class SpringConnectionFactory {
GlobalConfig.DbConfig dbConfig = new GlobalConfig.DbConfig();
dbConfig.setIdType(IdType.AUTO);
- GlobalConfig globalConfig = new GlobalConfig();
+ GlobalConfig globalConfig = new GlobalConfig().setBanner(false);
globalConfig.setDbConfig(dbConfig);
sqlSessionFactoryBean.setGlobalConfig(globalConfig);
sqlSessionFactoryBean.setTypeAliasesPackage("org.apache.dolphinscheduler.dao.entity");
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.java
index 280e04938b..7fc5322947 100644
--- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.java
+++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.java
@@ -53,7 +53,7 @@ public interface ProcessDefinitionMapper extends BaseMapper {
/**
* update
*/
- @CacheEvict(key = "#processDefinition.code")
+ @CacheEvict(key = "#p0.code")
int updateById(@Name("processDefinition") @Param("et") ProcessDefinition processDefinition);
/**
diff --git a/dolphinscheduler-dao/src/main/resources/application-h2.yaml b/dolphinscheduler-dao/src/main/resources/application-h2.yaml
deleted file mode 100644
index 147892c350..0000000000
--- a/dolphinscheduler-dao/src/main/resources/application-h2.yaml
+++ /dev/null
@@ -1,34 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-spring:
- datasource:
- driver-class-name: org.h2.Driver
- url: jdbc:h2:mem:dolphinscheduler;MODE=MySQL;DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=true;INIT=runscript from 'classpath:sql/dolphinscheduler_h2.sql'
- username: sa
- password: ""
- hikari:
- connection-test-query: select 1
- minimum-idle: 5
- auto-commit: true
- validation-timeout: 3000
- pool-name: DolphinScheduler
- maximum-pool-size: 50
- connection-timeout: 30000
- idle-timeout: 600000
- leak-detection-threshold: 0
- initialization-fail-timeout: 1
diff --git a/dolphinscheduler-dao/src/main/resources/application-mysql.yaml b/dolphinscheduler-dao/src/main/resources/application-mysql.yaml
deleted file mode 100644
index be5872b906..0000000000
--- a/dolphinscheduler-dao/src/main/resources/application-mysql.yaml
+++ /dev/null
@@ -1,34 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-spring:
- datasource:
- driver-class-name: com.mysql.jdbc.Driver
- url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8
- username: ds_user
- password: dolphinscheduler
- hikari:
- connection-test-query: select 1
- minimum-idle: 5
- auto-commit: true
- validation-timeout: 3000
- pool-name: DolphinScheduler
- maximum-pool-size: 50
- connection-timeout: 30000
- idle-timeout: 600000
- leak-detection-threshold: 0
- initialization-fail-timeout: 1
diff --git a/dolphinscheduler-dao/src/main/resources/sql/soft_version b/dolphinscheduler-dao/src/main/resources/sql/soft_version
index 359a5b952d..38f77a65b3 100644
--- a/dolphinscheduler-dao/src/main/resources/sql/soft_version
+++ b/dolphinscheduler-dao/src/main/resources/sql/soft_version
@@ -1 +1 @@
-2.0.0
\ No newline at end of file
+2.0.1
diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/BaseDaoTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/BaseDaoTest.java
index 769830cf21..c7208146d0 100644
--- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/BaseDaoTest.java
+++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/BaseDaoTest.java
@@ -17,13 +17,10 @@
package org.apache.dolphinscheduler.dao;
-import org.apache.dolphinscheduler.common.enums.ProfileType;
-
import org.junit.runner.RunWith;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.annotation.Rollback;
-import org.springframework.test.context.ActiveProfiles;
import org.springframework.test.context.junit4.SpringRunner;
import org.springframework.transaction.annotation.EnableTransactionManagement;
import org.springframework.transaction.annotation.Transactional;
@@ -31,7 +28,6 @@ import org.springframework.transaction.annotation.Transactional;
@RunWith(SpringRunner.class)
@SpringBootTest(classes = DaoConfiguration.class)
@SpringBootApplication(scanBasePackageClasses = DaoConfiguration.class)
-@ActiveProfiles(value = ProfileType.H2)
@Transactional
@Rollback
@EnableTransactionManagement
diff --git a/dolphinscheduler-standalone-server/src/main/resources/application-standalone.yaml b/dolphinscheduler-dao/src/test/resources/application.yaml
similarity index 77%
rename from dolphinscheduler-standalone-server/src/main/resources/application-standalone.yaml
rename to dolphinscheduler-dao/src/test/resources/application.yaml
index 532c85a3b3..22087b9afa 100644
--- a/dolphinscheduler-standalone-server/src/main/resources/application-standalone.yaml
+++ b/dolphinscheduler-dao/src/test/resources/application.yaml
@@ -16,19 +16,8 @@
#
spring:
- application:
- name: standalone-server
-
-server:
- port: 12345
-
-management:
- endpoints:
- web:
- exposure:
- include: '*'
- server:
- port: 8080
- metrics:
- tags:
- application: ${spring.application.name}
+ datasource:
+ driver-class-name: org.h2.Driver
+ url: jdbc:h2:mem:dolphinscheduler;MODE=MySQL;DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=true;INIT=runscript from 'classpath:sql/dolphinscheduler_h2.sql'
+ username: sa
+ password: ""
diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/pom.xml
index cb44ca1d3c..ee00ef47f8 100644
--- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/pom.xml
+++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/pom.xml
@@ -135,10 +135,6 @@
com.google.code.gson
gson
-
- com.google.code.findbugs
- jsr305
-
io.dropwizard.metrics
metrics-core
@@ -151,10 +147,6 @@
org.apache.avro
avro
-
- org.apache.commons
- commons-compress
-
org.apache.curator
curator-client
diff --git a/dolphinscheduler-dist/pom.xml b/dolphinscheduler-dist/pom.xml
index b75262e8f0..f27d16f321 100644
--- a/dolphinscheduler-dist/pom.xml
+++ b/dolphinscheduler-dist/pom.xml
@@ -51,6 +51,11 @@
org.apache.dolphinscheduler
dolphinscheduler-alert-server
+
+
+ org.apache.dolphinscheduler
+ dolphinscheduler-ui
+
@@ -92,20 +97,6 @@
-
-
- org.apache.maven.plugins
- maven-source-plugin
-
-
- attach-sources
- verify
-
- jar-no-fork
-
-
-
-
diff --git a/dolphinscheduler-dist/release-docs/LICENSE b/dolphinscheduler-dist/release-docs/LICENSE
index 5f1f7d0907..6c4e13456d 100644
--- a/dolphinscheduler-dist/release-docs/LICENSE
+++ b/dolphinscheduler-dist/release-docs/LICENSE
@@ -233,7 +233,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
commons-codec 1.11: https://mvnrepository.com/artifact/commons-codec/commons-codec/1.11, Apache 2.0
commons-collections 3.2.2: https://mvnrepository.com/artifact/commons-collections/commons-collections/3.2.2, Apache 2.0
commons-collections4 4.1: https://mvnrepository.com/artifact/org.apache.commons/commons-collections4/4.1, Apache 2.0
- commons-compress 1.4.1: https://mvnrepository.com/artifact/org.apache.commons/commons-compress/1.4.1, Apache 2.0
+ commons-compress 1.19: https://mvnrepository.com/artifact/org.apache.commons/commons-compress/1.19, Apache 2.0
commons-configuration 1.10: https://mvnrepository.com/artifact/commons-configuration/commons-configuration/1.10, Apache 2.0
commons-daemon 1.0.13 https://mvnrepository.com/artifact/commons-daemon/commons-daemon/1.0.13, Apache 2.0
commons-dbcp 1.4: https://github.com/apache/commons-dbcp, Apache 2.0
@@ -321,7 +321,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
jetty-xml 9.4.44.v20210927: https://mvnrepository.com/artifact/org.eclipse.jetty/jetty-xml/9.4.44.v20210927, Apache 2.0 and EPL 1.0
jna 4.5.2: https://mvnrepository.com/artifact/net.java.dev.jna/jna/4.5.2, Apache 2.0 and LGPL 2.1
jna-platform 4.5.2: https://mvnrepository.com/artifact/net.java.dev.jna/jna-platform/4.5.2, Apache 2.0 and LGPL 2.1
- joda-time 2.5: https://github.com/JodaOrg/joda-time, Apache 2.0
+ joda-time 2.10.13: https://github.com/JodaOrg/joda-time, Apache 2.0
jpam 1.1: https://mvnrepository.com/artifact/net.sf.jpam/jpam/1.1, Apache 2.0
jsqlparser 2.1: https://github.com/JSQLParser/JSqlParser, Apache 2.0 or LGPL 2.1
jsr305 3.0.0: https://mvnrepository.com/artifact/com.google.code.findbugs/jsr305, Apache 2.0
@@ -348,8 +348,8 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
poi 4.1.2: https://mvnrepository.com/artifact/org.apache.poi/poi/4.1.2, Apache 2.0
poi-ooxml 4.1.2: https://mvnrepository.com/artifact/org.apache.poi/poi-ooxml/4.1.2, Apache 2.0
poi-ooxml-schemas-4.1.2: https://mvnrepository.com/artifact/org.apache.poi/poi-ooxml-schemas/4.1.2, Apache 2.0
- quartz 2.3.0: https://mvnrepository.com/artifact/org.quartz-scheduler/quartz/2.3.0, Apache 2.0
- quartz-jobs 2.3.0: https://mvnrepository.com/artifact/org.quartz-scheduler/quartz-jobs/2.3.0, Apache 2.0
+ quartz 2.3.2: https://mvnrepository.com/artifact/org.quartz-scheduler/quartz/2.3.2, Apache 2.0
+ quartz-jobs 2.3.2: https://mvnrepository.com/artifact/org.quartz-scheduler/quartz-jobs/2.3.2, Apache 2.0
snakeyaml 1.28: https://mvnrepository.com/artifact/org.yaml/snakeyaml/1.28, Apache 2.0
snappy 0.2: https://mvnrepository.com/artifact/org.iq80.snappy/snappy/0.2, Apache 2.0
snappy-java 1.0.4.1: https://github.com/xerial/snappy-java, Apache 2.0
@@ -426,6 +426,8 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
xmlenc 0.52: https://mvnrepository.com/artifact/xmlenc/xmlenc/0.52, BSD
py4j 0.10.9: https://mvnrepository.com/artifact/net.sf.py4j/py4j/0.10.9, BSD 2-clause
LatencyUtils 2.0.3: https://github.com/LatencyUtils/LatencyUtils, BSD-2-Clause
+ janino 3.1.6: https://mvnrepository.com/artifact/org.codehaus.janino/janino/3.1.6, BSD 3-clause
+ commons-compiler 3.1.6: https://mvnrepository.com/artifact/org.codehaus.janino/janino/3.1.6, BSD 3-clause
========================================================================
CDDL licenses
@@ -490,7 +492,6 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
Public Domain licenses
========================================================================
- xz 1.0: https://mvnrepository.com/artifact/org.tukaani/xz/1.0, Public Domain
aopalliance 1.0: https://mvnrepository.com/artifact/aopalliance/aopalliance/1.0, Public Domain
========================================
diff --git a/dolphinscheduler-dist/release-docs/licenses/LICENSE-janino.txt b/dolphinscheduler-dist/release-docs/licenses/LICENSE-janino.txt
new file mode 100644
index 0000000000..ef871e2426
--- /dev/null
+++ b/dolphinscheduler-dist/release-docs/licenses/LICENSE-janino.txt
@@ -0,0 +1,31 @@
+Janino - An embedded Java[TM] compiler
+
+Copyright (c) 2001-2016, Arno Unkrig
+Copyright (c) 2015-2016 TIBCO Software Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following
+ disclaimer in the documentation and/or other materials
+ provided with the distribution.
+ 3. Neither the name of JANINO nor the names of its contributors
+ may be used to endorse or promote products derived from this
+ software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-bin.xml b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-bin.xml
index 33268e3134..85f5a62351 100644
--- a/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-bin.xml
+++ b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-bin.xml
@@ -15,11 +15,9 @@
~ limitations under the License.
-->
-
- bin
+ bin
tar.gz
@@ -28,186 +26,58 @@
- ${basedir}/../dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources
-
- **/*.yaml
- **/*.yml
- **/*.properties
- **/*.xml
- **/*.json
- **/*.ftl
-
- conf
+ ${basedir}/../dolphinscheduler-alert/dolphinscheduler-alert-server/target/alert-server
+ alert-server
- ${basedir}/../dolphinscheduler-api/src/main/resources
-
- **/*.yaml
- **/*.yml
- **/*.properties
- **/*.xml
- **/*.json
-
- conf
+ ${basedir}/../dolphinscheduler-api/target/api-server
+ api-server
- ${basedir}/../dolphinscheduler-common/src/main/resources
-
- **/*.yaml
- **/*.yml
- **/*.properties
- **/*.xml
- **/*.json
-
- conf
+ ${basedir}/../dolphinscheduler-master/target/master-server
+ master-server
- ${basedir}/../dolphinscheduler-dao/src/main/resources
-
- **/*.yaml
- **/*.yml
- **/*.properties
- **/*.xml
- **/*.json
-
-
- org/apache/dolphinscheduler/dao/mapper/*.xml
-
- conf
+ ${basedir}/../dolphinscheduler-worker/target/worker-server
+ worker-server
- ${basedir}/../dolphinscheduler-server/src/main/resources
-
- **/*.yaml
- **/*.yml
- **/*.properties
- **/*.xml
- **/*.json
- config/*.*
-
- conf
+ ${basedir}/../dolphinscheduler-log-server/target/logger-server
+ logger-server
- ${basedir}/../dolphinscheduler-service/src/main/resources
-
- **/*.yaml
- **/*.yml
- **/*.properties
- **/*.xml
- **/*.json
- **/*.yml
-
- conf
+ ${basedir}/../dolphinscheduler-standalone-server/target/standalone-server
+ standalone-server
- ${basedir}/../dolphinscheduler-meter/src/main/resources
-
- **/*.yaml
- **/*.yml
- **/*.properties
- **/*.xml
- **/*.json
- **/*.yml
-
- conf
-
-
-
- ${basedir}/../dolphinscheduler-standalone-server/src/main/resources
-
- **/*.yaml
-
- conf
+ ${basedir}/../dolphinscheduler-tools/target/tools
+ tools
${basedir}/../dolphinscheduler-dist/target/dolphinscheduler-dist-${project.version}
-
- **/*.*
-
.
${basedir}/../dolphinscheduler-ui/dist
-
- **/*.*
-
./ui
-
- ${basedir}/../dolphinscheduler-dao/src/main/resources/sql
-
- **/*
-
- ./sql/sql
-
-
-
- ${basedir}/../script
-
- *.*
-
- ./script
-
-
${basedir}/../script
-
- env/*.*
-
- conf
-
-
-
- ${basedir}/../script
-
- start-all.sh
- stop-all.sh
- dolphinscheduler-daemon.sh
- status-all.sh
-
- ./bin
-
-
-
- ${basedir}/.././
-
- *.sh
- *.py
- DISCLAIMER
-
- .
+ bin
${basedir}/release-docs
- true
-
- **/*
-
.
-
-
-
-
- lib
- true
-
- org.apache.dolphinscheduler:dolphinscheduler-dist
- javax.servlet:servlet-api
- org.eclipse.jetty.aggregate:jetty-all
- org.slf4j:slf4j-log4j12
-
-
-
diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ProjectE2ETest.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ProjectE2ETest.java
index 9615075616..85b5ba0cd9 100644
--- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ProjectE2ETest.java
+++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ProjectE2ETest.java
@@ -53,7 +53,7 @@ class ProjectE2ETest {
@Test
@Order(30)
void testDeleteProject() {
- final var page = new ProjectPage(browser);
+ final ProjectPage page = new ProjectPage(browser);
page.delete(project);
await().untilAsserted(() -> {
diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/TenantE2ETest.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/TenantE2ETest.java
index 9de3edbca3..4938a8b514 100644
--- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/TenantE2ETest.java
+++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/TenantE2ETest.java
@@ -58,7 +58,7 @@ class TenantE2ETest {
@Test
@Order(20)
void testCreateDuplicateTenant() {
- final var page = new TenantPage(browser);
+ final TenantPage page = new TenantPage(browser);
page.create(tenant);
@@ -73,7 +73,7 @@ class TenantE2ETest {
@Test
@Order(30)
void testDeleteTenant() {
- final var page = new TenantPage(browser);
+ final TenantPage page = new TenantPage(browser);
page.delete(tenant);
await().untilAsserted(() -> {
diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/WorkflowE2ETest.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/WorkflowE2ETest.java
index cc51932114..9ef06969fd 100644
--- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/WorkflowE2ETest.java
+++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/WorkflowE2ETest.java
@@ -19,12 +19,10 @@
package org.apache.dolphinscheduler.e2e.cases;
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.awaitility.Awaitility.await;
-
import org.apache.dolphinscheduler.e2e.core.DolphinScheduler;
import org.apache.dolphinscheduler.e2e.pages.LoginPage;
import org.apache.dolphinscheduler.e2e.pages.common.NavBarPage;
+import org.apache.dolphinscheduler.e2e.pages.project.ProjectDetailPage;
import org.apache.dolphinscheduler.e2e.pages.project.ProjectPage;
import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowDefinitionTab;
import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowForm.TaskType;
@@ -34,13 +32,15 @@ import org.apache.dolphinscheduler.e2e.pages.project.workflow.task.ShellTaskForm
import org.apache.dolphinscheduler.e2e.pages.project.workflow.task.SubWorkflowTaskForm;
import org.apache.dolphinscheduler.e2e.pages.security.SecurityPage;
import org.apache.dolphinscheduler.e2e.pages.security.TenantPage;
-
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Order;
import org.junit.jupiter.api.Test;
import org.openqa.selenium.remote.RemoteWebDriver;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.awaitility.Awaitility.await;
+
@DolphinScheduler(composeFiles = "docker/basic/docker-compose.yaml")
class WorkflowE2ETest {
private static final String project = "test-workflow-1";
@@ -81,9 +81,9 @@ class WorkflowE2ETest {
@Test
@Order(1)
void testCreateWorkflow() {
- final var workflow = "test-workflow-1";
+ final String workflow = "test-workflow-1";
- final var workflowDefinitionPage =
+ final WorkflowDefinitionTab workflowDefinitionPage =
new ProjectPage(browser)
.goTo(project)
.goToTab(WorkflowDefinitionTab.class);
@@ -114,9 +114,9 @@ class WorkflowE2ETest {
@Test
@Order(10)
void testCreateSubWorkflow() {
- final var workflow = "test-sub-workflow-1";
+ final String workflow = "test-sub-workflow-1";
- final var workflowDefinitionPage =
+ final WorkflowDefinitionTab workflowDefinitionPage =
new ProjectPage(browser)
.goToNav(ProjectPage.class)
.goTo(project)
@@ -145,9 +145,9 @@ class WorkflowE2ETest {
@Test
@Order(30)
void testRunWorkflow() {
- final var workflow = "test-workflow-1";
+ final String workflow = "test-workflow-1";
- final var projectPage =
+ final ProjectDetailPage projectPage =
new ProjectPage(browser)
.goToNav(ProjectPage.class)
.goTo(project);
diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/project/workflow/WorkflowDefinitionTab.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/project/workflow/WorkflowDefinitionTab.java
index 56f0eb3d04..5d30dcd19b 100644
--- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/project/workflow/WorkflowDefinitionTab.java
+++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/project/workflow/WorkflowDefinitionTab.java
@@ -19,20 +19,18 @@
*/
package org.apache.dolphinscheduler.e2e.pages.project.workflow;
+import lombok.Getter;
import org.apache.dolphinscheduler.e2e.pages.common.NavBarPage;
import org.apache.dolphinscheduler.e2e.pages.project.ProjectDetailPage;
-
-import java.util.List;
-import java.util.function.Supplier;
-import java.util.stream.Collectors;
-
import org.openqa.selenium.By;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.remote.RemoteWebDriver;
import org.openqa.selenium.support.FindBy;
import org.openqa.selenium.support.FindBys;
-import lombok.Getter;
+import java.util.List;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
@Getter
public final class WorkflowDefinitionTab extends NavBarPage implements ProjectDetailPage.Tab {
@@ -94,7 +92,7 @@ public final class WorkflowDefinitionTab extends NavBarPage implements ProjectDe
.filter(WebElement::isDisplayed)
.collect(Collectors.toList());
- for (var buttons = cancelButtons.get();
+ for (List buttons = cancelButtons.get();
!buttons.isEmpty();
buttons = cancelButtons.get()) {
buttons.forEach(WebElement::click);
diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/project/workflow/WorkflowForm.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/project/workflow/WorkflowForm.java
index 95c21b1ba2..b4250829d6 100644
--- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/project/workflow/WorkflowForm.java
+++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/project/workflow/WorkflowForm.java
@@ -55,11 +55,11 @@ public final class WorkflowForm {
@SneakyThrows
@SuppressWarnings("unchecked")
public T addTask(TaskType type) {
- final var task = driver.findElement(By.className("task-item-" + type.name()));
- final var canvas = driver.findElement(By.className("dag-container"));
+ final WebElement task = driver.findElement(By.className("task-item-" + type.name()));
+ final WebElement canvas = driver.findElement(By.className("dag-container"));
- final var js = (JavascriptExecutor) driver;
- final var dragAndDrop = String.join("\n",
+ final JavascriptExecutor js = (JavascriptExecutor) driver;
+ final String dragAndDrop = String.join("\n",
Resources.readLines(Resources.getResource("dragAndDrop.js"), StandardCharsets.UTF_8));
js.executeScript(dragAndDrop, task, canvas);
diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/project/workflow/WorkflowSaveDialog.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/project/workflow/WorkflowSaveDialog.java
index ce08926f90..9c0039782d 100644
--- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/project/workflow/WorkflowSaveDialog.java
+++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/project/workflow/WorkflowSaveDialog.java
@@ -19,9 +19,7 @@
*/
package org.apache.dolphinscheduler.e2e.pages.project.workflow;
-import java.util.List;
-import java.util.stream.Stream;
-
+import lombok.Getter;
import org.openqa.selenium.By;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
@@ -32,7 +30,8 @@ import org.openqa.selenium.support.pagefactory.ByChained;
import org.openqa.selenium.support.ui.ExpectedConditions;
import org.openqa.selenium.support.ui.WebDriverWait;
-import lombok.Getter;
+import java.util.List;
+import java.util.stream.Stream;
@Getter
public final class WorkflowSaveDialog {
@@ -72,7 +71,7 @@ public final class WorkflowSaveDialog {
public WorkflowSaveDialog tenant(String tenant) {
selectTenant().click();
- final var optionsLocator = By.className("option-tenants");
+ final By optionsLocator = By.className("option-tenants");
new WebDriverWait(driver, 10)
.until(ExpectedConditions.visibilityOfElementLocated(optionsLocator));
@@ -91,9 +90,9 @@ public final class WorkflowSaveDialog {
public WorkflowSaveDialog addGlobalParam(String key, String val) {
assert inputParamKey().size() == inputParamVal().size();
- final var len = inputParamKey().size();
+ final int len = inputParamKey().size();
- final var driver = parent().driver();
+ final WebDriver driver = parent().driver();
Stream.concat(
driver.findElements(new ByChained(By.className("user-def-params-model"), By.className("add"))).stream(),
driver.findElements(new ByChained(By.className("user-def-params-model"), By.className("add-dp"))).stream())
diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/project/workflow/task/TaskNodeForm.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/project/workflow/task/TaskNodeForm.java
index 8689a21569..c2e703c1e1 100644
--- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/project/workflow/task/TaskNodeForm.java
+++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/project/workflow/task/TaskNodeForm.java
@@ -19,19 +19,18 @@
*/
package org.apache.dolphinscheduler.e2e.pages.project.workflow.task;
+import lombok.Getter;
import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowForm;
-
-import java.util.List;
-import java.util.stream.Stream;
-
import org.openqa.selenium.By;
+import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.support.FindBy;
import org.openqa.selenium.support.FindBys;
import org.openqa.selenium.support.PageFactory;
import org.openqa.selenium.support.pagefactory.ByChained;
-import lombok.Getter;
+import java.util.List;
+import java.util.stream.Stream;
@Getter
public abstract class TaskNodeForm {
@@ -55,7 +54,7 @@ public abstract class TaskNodeForm {
TaskNodeForm(WorkflowForm parent) {
this.parent = parent;
- final var driver = parent.driver();
+ final WebDriver driver = parent.driver();
PageFactory.initElements(driver, this);
}
@@ -69,9 +68,9 @@ public abstract class TaskNodeForm {
public TaskNodeForm addParam(String key, String val) {
assert inputParamKey().size() == inputParamVal().size();
- final var len = inputParamKey().size();
+ final int len = inputParamKey().size();
- final var driver = parent().driver();
+ final WebDriver driver = parent().driver();
Stream.concat(
driver.findElements(new ByChained(By.className("user-def-params-model"), By.className("add"))).stream(),
driver.findElements(new ByChained(By.className("user-def-params-model"), By.className("add-dp"))).stream())
diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/basic/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/basic/docker-compose.yaml
index 22bd9ecf2b..f12ddd9342 100644
--- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/basic/docker-compose.yaml
+++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/basic/docker-compose.yaml
@@ -19,17 +19,16 @@ version: "2.1"
services:
dolphinscheduler:
- image: apache/dolphinscheduler:ci
- command: [ standalone-server ]
+ image: apache/dolphinscheduler-standalone-server:ci
environment:
- DATABASE_TYPE: h2
+ MASTER_MAX_CPU_LOAD_AVG: 100
WORKER_TENANT_AUTO_CREATE: 'true'
expose:
- 12345
networks:
- e2e
healthcheck:
- test: [ "CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/12345" ]
+ test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ]
interval: 5s
timeout: 60s
retries: 120
diff --git a/dolphinscheduler-e2e/pom.xml b/dolphinscheduler-e2e/pom.xml
index 22f913df95..7be813187c 100644
--- a/dolphinscheduler-e2e/pom.xml
+++ b/dolphinscheduler-e2e/pom.xml
@@ -31,8 +31,8 @@
- 11
- 11
+ 8
+ 8
UTF-8
5.7.2
diff --git a/dolphinscheduler-log-server/pom.xml b/dolphinscheduler-log-server/pom.xml
new file mode 100644
index 0000000000..de58a62753
--- /dev/null
+++ b/dolphinscheduler-log-server/pom.xml
@@ -0,0 +1,89 @@
+
+
+
+
+
+ dolphinscheduler
+ org.apache.dolphinscheduler
+ 2.0.0-SNAPSHOT
+
+ 4.0.0
+
+ dolphinscheduler-log-server
+
+
+
+ org.apache.dolphinscheduler
+ dolphinscheduler-service
+
+
+ org.apache.dolphinscheduler
+ dolphinscheduler-server
+
+
+ org.apache.dolphinscheduler
+ dolphinscheduler-spi
+
+
+
+ org.mockito
+ mockito-core
+ test
+
+
+
+
+
+
+ maven-assembly-plugin
+
+
+ dolphinscheduler-logger-server
+ package
+
+ single
+
+
+ logger-server
+
+ src/main/assembly/dolphinscheduler-log-server.xml
+
+ false
+
+
+
+
+
+
+
+
+
+ docker
+
+
+
+ org.codehaus.mojo
+ exec-maven-plugin
+
+
+
+
+
+
diff --git a/dolphinscheduler-log-server/src/main/assembly/dolphinscheduler-log-server.xml b/dolphinscheduler-log-server/src/main/assembly/dolphinscheduler-log-server.xml
new file mode 100644
index 0000000000..35ac6fffd3
--- /dev/null
+++ b/dolphinscheduler-log-server/src/main/assembly/dolphinscheduler-log-server.xml
@@ -0,0 +1,56 @@
+
+
+
+ dolphinscheduler-logger-server
+
+ dir
+
+ false
+ logger-server
+
+
+ ${basedir}/src/main/bin
+ bin
+ 0755
+ 0755
+
+
+ ${basedir}/../script/env
+ bin
+
+ dolphinscheduler_env.sh
+
+ 0755
+ 0755
+
+
+ ${basedir}/../../dolphinscheduler-common/src/main/resources
+
+ **/*.properties
+
+ conf
+
+
+
+
+ libs
+
+
+
diff --git a/dolphinscheduler-log-server/src/main/bin/start.sh b/dolphinscheduler-log-server/src/main/bin/start.sh
new file mode 100644
index 0000000000..2d6f8fc717
--- /dev/null
+++ b/dolphinscheduler-log-server/src/main/bin/start.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+BIN_DIR=$(dirname $0)
+DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/..; pwd)}
+
+source "$BIN_DIR/dolphinscheduler_env.sh"
+
+JAVA_OPTS=${JAVA_OPTS:-"-server -Xms1g -Xmx1g -Xmn512m -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"}
+
+if [[ "$DOCKER" == "true" ]]; then
+ JAVA_OPTS="${JAVA_OPTS} -XX:-UseContainerSupport"
+fi
+
+java $JAVA_OPTS \
+ -cp "$DOLPHINSCHEDULER_HOME/conf":"$DOLPHINSCHEDULER_HOME/libs/*" \
+ org.apache.dolphinscheduler.server.log.LoggerServer
diff --git a/dolphinscheduler-log-server/src/main/docker/Dockerfile b/dolphinscheduler-log-server/src/main/docker/Dockerfile
new file mode 100644
index 0000000000..de7e05f767
--- /dev/null
+++ b/dolphinscheduler-log-server/src/main/docker/Dockerfile
@@ -0,0 +1,34 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+FROM openjdk:8-jre-slim-buster
+
+ENV DOCKER true
+ENV TZ Asia/Shanghai
+ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler
+
+RUN apt update ; \
+ apt install -y curl ; \
+ rm -rf /var/lib/apt/lists/*
+
+WORKDIR $DOLPHINSCHEDULER_HOME
+
+ADD ./target/logger-server $DOLPHINSCHEDULER_HOME
+
+EXPOSE 50051
+
+CMD [ "/bin/bash", "./bin/start.sh" ]
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java b/dolphinscheduler-log-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java
rename to dolphinscheduler-log-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerServer.java b/dolphinscheduler-log-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerServer.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerServer.java
rename to dolphinscheduler-log-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerServer.java
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessorTest.java b/dolphinscheduler-log-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessorTest.java
similarity index 97%
rename from dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessorTest.java
rename to dolphinscheduler-log-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessorTest.java
index e24539558c..84baed34eb 100644
--- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessorTest.java
+++ b/dolphinscheduler-log-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessorTest.java
@@ -24,7 +24,6 @@ import org.apache.dolphinscheduler.remote.command.CommandType;
import org.apache.dolphinscheduler.remote.command.log.ViewLogRequestCommand;
import org.junit.Test;
-import org.junit.Test.None;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
@@ -37,7 +36,7 @@ import io.netty.channel.Channel;
@PrepareForTest({LoggerUtils.class})
public class LoggerRequestProcessorTest {
- @Test(expected = None.class)
+ @Test
public void testProcessViewWholeLogRequest() {
Channel channel = PowerMockito.mock(Channel.class);
PowerMockito.when(channel.writeAndFlush(Mockito.any(Command.class))).thenReturn(null);
@@ -53,4 +52,4 @@ public class LoggerRequestProcessorTest {
LoggerRequestProcessor loggerRequestProcessor = new LoggerRequestProcessor();
loggerRequestProcessor.process(channel, command);
}
-}
\ No newline at end of file
+}
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerServerTest.java b/dolphinscheduler-log-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerServerTest.java
similarity index 100%
rename from dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerServerTest.java
rename to dolphinscheduler-log-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerServerTest.java
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/MasterLogFilterTest.java b/dolphinscheduler-log-server/src/test/java/org/apache/dolphinscheduler/server/log/MasterLogFilterTest.java
similarity index 100%
rename from dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/MasterLogFilterTest.java
rename to dolphinscheduler-log-server/src/test/java/org/apache/dolphinscheduler/server/log/MasterLogFilterTest.java
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/SensitiveDataConverterTest.java b/dolphinscheduler-log-server/src/test/java/org/apache/dolphinscheduler/server/log/SensitiveDataConverterTest.java
similarity index 100%
rename from dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/SensitiveDataConverterTest.java
rename to dolphinscheduler-log-server/src/test/java/org/apache/dolphinscheduler/server/log/SensitiveDataConverterTest.java
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/TaskLogFilterTest.java b/dolphinscheduler-log-server/src/test/java/org/apache/dolphinscheduler/server/log/TaskLogFilterTest.java
similarity index 100%
rename from dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/TaskLogFilterTest.java
rename to dolphinscheduler-log-server/src/test/java/org/apache/dolphinscheduler/server/log/TaskLogFilterTest.java
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/WorkerLogFilterTest.java b/dolphinscheduler-log-server/src/test/java/org/apache/dolphinscheduler/server/log/WorkerLogFilterTest.java
similarity index 100%
rename from dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/WorkerLogFilterTest.java
rename to dolphinscheduler-log-server/src/test/java/org/apache/dolphinscheduler/server/log/WorkerLogFilterTest.java
diff --git a/dolphinscheduler-master/pom.xml b/dolphinscheduler-master/pom.xml
new file mode 100644
index 0000000000..0799445467
--- /dev/null
+++ b/dolphinscheduler-master/pom.xml
@@ -0,0 +1,135 @@
+
+
+
+
+ dolphinscheduler
+ org.apache.dolphinscheduler
+ 2.0.0-SNAPSHOT
+
+ 4.0.0
+
+ dolphinscheduler-master
+
+
+
+ org.apache.dolphinscheduler
+ dolphinscheduler-common
+
+
+ org.apache.dolphinscheduler
+ dolphinscheduler-meter
+
+
+ org.apache.dolphinscheduler
+ dolphinscheduler-service
+
+
+ org.apache.dolphinscheduler
+ dolphinscheduler-server
+
+
+ org.springframework.boot
+ spring-boot-starter-cache
+
+
+ log4j-api
+ org.apache.logging.log4j
+
+
+ log4j-to-slf4j
+ org.apache.logging.log4j
+
+
+
+
+
+ org.codehaus.janino
+ janino
+
+
+
+ org.mockito
+ mockito-core
+ test
+
+
+ org.springframework.boot
+ spring-boot-starter-test
+ test
+
+
+ org.apache.dolphinscheduler
+ dolphinscheduler-worker
+ test
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+
+
+ *.yaml
+ *.xml
+
+
+
+
+
+ maven-assembly-plugin
+
+
+ dolphinscheduler-master-server
+ package
+
+ single
+
+
+ master-server
+
+ src/main/assembly/dolphinscheduler-master-server.xml
+
+ false
+
+
+
+
+
+
+
+
+
+ docker
+
+
+
+ org.codehaus.mojo
+ exec-maven-plugin
+
+
+
+
+
+
diff --git a/dolphinscheduler-master/src/main/assembly/dolphinscheduler-master-server.xml b/dolphinscheduler-master/src/main/assembly/dolphinscheduler-master-server.xml
new file mode 100644
index 0000000000..760163328e
--- /dev/null
+++ b/dolphinscheduler-master/src/main/assembly/dolphinscheduler-master-server.xml
@@ -0,0 +1,64 @@
+
+
+
+ dolphinscheduler-master-server
+
+ dir
+
+ false
+ master-server
+
+
+ ${basedir}/src/main/resources
+
+ *.yaml
+ *.xml
+
+ conf
+
+
+ ${basedir}/src/main/bin
+ bin
+ 0755
+ 0755
+
+
+ ${basedir}/../script/env
+ bin
+
+ dolphinscheduler_env.sh
+
+ 0755
+ 0755
+
+
+ ${basedir}/../dolphinscheduler-common/src/main/resources
+
+ **/*.properties
+
+ conf
+
+
+
+
+ libs
+
+
+
diff --git a/dolphinscheduler-master/src/main/bin/start.sh b/dolphinscheduler-master/src/main/bin/start.sh
new file mode 100644
index 0000000000..b9d0c8d044
--- /dev/null
+++ b/dolphinscheduler-master/src/main/bin/start.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+BIN_DIR=$(dirname $0)
+DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/..; pwd)}
+
+source "$BIN_DIR/dolphinscheduler_env.sh"
+
+JAVA_OPTS=${JAVA_OPTS:-"-server -Xms4g -Xmx4g -Xmn2g -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"}
+
+if [[ "$DOCKER" == "true" ]]; then
+ JAVA_OPTS="${JAVA_OPTS} -XX:-UseContainerSupport"
+fi
+
+java $JAVA_OPTS \
+ -cp "$DOLPHINSCHEDULER_HOME/conf":"$DOLPHINSCHEDULER_HOME/libs/*" \
+ org.apache.dolphinscheduler.server.master.MasterServer
diff --git a/dolphinscheduler-master/src/main/docker/Dockerfile b/dolphinscheduler-master/src/main/docker/Dockerfile
new file mode 100644
index 0000000000..7ab4f622c3
--- /dev/null
+++ b/dolphinscheduler-master/src/main/docker/Dockerfile
@@ -0,0 +1,34 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+FROM openjdk:8-jre-slim-buster
+
+ENV DOCKER true
+ENV TZ Asia/Shanghai
+ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler
+
+RUN apt update ; \
+ apt install -y curl ; \
+ rm -rf /var/lib/apt/lists/*
+
+WORKDIR $DOLPHINSCHEDULER_HOME
+
+ADD ./target/master-server $DOLPHINSCHEDULER_HOME
+
+EXPOSE 12345
+
+CMD [ "/bin/bash", "./bin/start.sh" ]
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/builder/TaskExecutionContextBuilder.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/builder/TaskExecutionContextBuilder.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/builder/TaskExecutionContextBuilder.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/builder/TaskExecutionContextBuilder.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
similarity index 90%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
index 704b1b0dae..9157ad704b 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
@@ -34,26 +34,21 @@ import org.apache.dolphinscheduler.server.master.registry.MasterRegistryClient;
import org.apache.dolphinscheduler.server.master.runner.EventExecuteService;
import org.apache.dolphinscheduler.server.master.runner.MasterSchedulerService;
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
-
-import javax.annotation.PostConstruct;
-
+import org.quartz.Scheduler;
+import org.quartz.SchedulerException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.builder.SpringApplicationBuilder;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.context.annotation.ComponentScan;
-import org.springframework.context.annotation.FilterType;
import org.springframework.transaction.annotation.EnableTransactionManagement;
-@ComponentScan(value = "org.apache.dolphinscheduler", excludeFilters = {
- @ComponentScan.Filter(type = FilterType.REGEX, pattern = {
- "org.apache.dolphinscheduler.server.worker.*",
- "org.apache.dolphinscheduler.server.monitor.*",
- "org.apache.dolphinscheduler.server.log.*",
- "org.apache.dolphinscheduler.alert.*"
- })
-})
+import javax.annotation.PostConstruct;
+
+@SpringBootApplication
+@ComponentScan("org.apache.dolphinscheduler")
@EnableTransactionManagement
@EnableCaching
public class MasterServer implements IStoppable {
@@ -76,24 +71,28 @@ public class MasterServer implements IStoppable {
@Autowired
private EventExecuteService eventExecuteService;
+ @Autowired
+ private Scheduler scheduler;
+
+ @Autowired
+ private TaskAckProcessor taskAckProcessor;
+
public static void main(String[] args) {
Thread.currentThread().setName(Constants.THREAD_NAME_MASTER_SERVER);
- new SpringApplicationBuilder(MasterServer.class)
- .profiles("master")
- .run(args);
+ SpringApplication.run(MasterServer.class);
}
/**
* run master server
*/
@PostConstruct
- public void run() {
+ public void run() throws SchedulerException {
// init remoting server
NettyServerConfig serverConfig = new NettyServerConfig();
serverConfig.setListenPort(masterConfig.getListenPort());
this.nettyRemotingServer = new NettyRemotingServer(serverConfig);
this.nettyRemotingServer.registerProcessor(CommandType.TASK_EXECUTE_RESPONSE, new TaskResponseProcessor());
- this.nettyRemotingServer.registerProcessor(CommandType.TASK_EXECUTE_ACK, new TaskAckProcessor());
+ this.nettyRemotingServer.registerProcessor(CommandType.TASK_EXECUTE_ACK, taskAckProcessor);
this.nettyRemotingServer.registerProcessor(CommandType.TASK_KILL_RESPONSE, new TaskKillResponseProcessor());
this.nettyRemotingServer.registerProcessor(CommandType.STATE_EVENT_REQUEST, new StateEventProcessor());
this.nettyRemotingServer.registerProcessor(CommandType.TASK_FORCE_STATE_EVENT_REQUEST, new TaskEventProcessor());
@@ -111,6 +110,8 @@ public class MasterServer implements IStoppable {
this.masterSchedulerService.init();
this.masterSchedulerService.start();
+ this.scheduler.start();
+
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
if (Stopper.isRunning()) {
close("shutdownHook");
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/cache/ProcessInstanceExecCacheManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/ProcessInstanceExecCacheManager.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/cache/ProcessInstanceExecCacheManager.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/ProcessInstanceExecCacheManager.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/cache/impl/ProcessInstanceExecCacheManagerImpl.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/impl/ProcessInstanceExecCacheManagerImpl.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/cache/impl/ProcessInstanceExecCacheManagerImpl.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/impl/ProcessInstanceExecCacheManagerImpl.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcher.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcher.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcher.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcher.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/context/ExecutionContext.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/context/ExecutionContext.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/context/ExecutionContext.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/context/ExecutionContext.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/enums/ExecutorType.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/enums/ExecutorType.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/enums/ExecutorType.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/enums/ExecutorType.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/exceptions/ExecuteException.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/exceptions/ExecuteException.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/exceptions/ExecuteException.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/exceptions/ExecuteException.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/AbstractExecutorManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/AbstractExecutorManager.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/AbstractExecutorManager.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/AbstractExecutorManager.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/ExecutorManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/ExecutorManager.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/ExecutorManager.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/ExecutorManager.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManager.java
similarity index 98%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManager.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManager.java
index 03a3672aed..886a7fc64c 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManager.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManager.java
@@ -59,6 +59,9 @@ public class NettyExecutorManager extends AbstractExecutorManager{
@Autowired
private ServerNodeManager serverNodeManager;
+ @Autowired
+ private TaskAckProcessor taskAckProcessor;
+
/**
* netty remote client
*/
@@ -79,7 +82,7 @@ public class NettyExecutorManager extends AbstractExecutorManager{
* register EXECUTE_TASK_ACK command type TaskAckProcessor
*/
this.nettyRemotingClient.registerProcessor(CommandType.TASK_EXECUTE_RESPONSE, new TaskResponseProcessor());
- this.nettyRemotingClient.registerProcessor(CommandType.TASK_EXECUTE_ACK, new TaskAckProcessor());
+ this.nettyRemotingClient.registerProcessor(CommandType.TASK_EXECUTE_ACK, taskAckProcessor);
this.nettyRemotingClient.registerProcessor(CommandType.TASK_KILL_RESPONSE, new TaskKillResponseProcessor());
}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/CommonHostManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/CommonHostManager.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/CommonHostManager.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/CommonHostManager.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/HostManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/HostManager.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/HostManager.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/HostManager.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/HostManagerConfig.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/HostManagerConfig.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/HostManagerConfig.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/HostManagerConfig.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/LowerWeightHostManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/LowerWeightHostManager.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/LowerWeightHostManager.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/LowerWeightHostManager.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/RandomHostManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/RandomHostManager.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/RandomHostManager.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/RandomHostManager.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManager.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManager.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManager.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/AbstractSelector.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/AbstractSelector.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/AbstractSelector.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/AbstractSelector.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostSelector.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostSelector.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostSelector.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostSelector.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWeight.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWeight.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWeight.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWeight.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWorker.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWorker.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWorker.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWorker.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/LowerWeightRoundRobin.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/LowerWeightRoundRobin.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/LowerWeightRoundRobin.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/LowerWeightRoundRobin.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RandomSelector.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RandomSelector.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RandomSelector.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RandomSelector.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelector.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelector.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelector.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelector.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/Selector.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/Selector.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/Selector.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/Selector.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/CacheProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/CacheProcessor.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/CacheProcessor.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/CacheProcessor.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/HostUpdateResponseProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/HostUpdateResponseProcessor.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/HostUpdateResponseProcessor.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/HostUpdateResponseProcessor.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/StateEventProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/StateEventProcessor.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/StateEventProcessor.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/StateEventProcessor.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessor.java
similarity index 90%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessor.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessor.java
index 27b8991541..9c42017b66 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessor.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessor.java
@@ -17,6 +17,8 @@
package org.apache.dolphinscheduler.server.master.processor;
+import com.google.common.base.Preconditions;
+import io.netty.channel.Channel;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.remote.command.Command;
@@ -26,30 +28,21 @@ import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor;
import org.apache.dolphinscheduler.remote.utils.ChannelUtils;
import org.apache.dolphinscheduler.server.master.processor.queue.TaskResponseEvent;
import org.apache.dolphinscheduler.server.master.processor.queue.TaskResponseService;
-import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Preconditions;
-
-import io.netty.channel.Channel;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
/**
* task ack processor
*/
+@Component
public class TaskAckProcessor implements NettyRequestProcessor {
private final Logger logger = LoggerFactory.getLogger(TaskAckProcessor.class);
- /**
- * process service
- */
- private final TaskResponseService taskResponseService;
-
- public TaskAckProcessor() {
- this.taskResponseService = SpringApplicationContext.getBean(TaskResponseService.class);
- }
+ @Autowired
+ private TaskResponseService taskResponseService;
/**
* task ack process
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskEventProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskEventProcessor.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskEventProcessor.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskEventProcessor.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskKillResponseProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskKillResponseProcessor.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskKillResponseProcessor.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskKillResponseProcessor.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessor.java
similarity index 92%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessor.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessor.java
index aa324ebbde..966a07d433 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessor.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessor.java
@@ -33,6 +33,7 @@ import org.slf4j.LoggerFactory;
import com.google.common.base.Preconditions;
import io.netty.channel.Channel;
+import org.springframework.beans.factory.annotation.Autowired;
/**
* task response processor
@@ -41,14 +42,8 @@ public class TaskResponseProcessor implements NettyRequestProcessor {
private final Logger logger = LoggerFactory.getLogger(TaskResponseProcessor.class);
- /**
- * process service
- */
- private final TaskResponseService taskResponseService;
-
- public TaskResponseProcessor() {
- this.taskResponseService = SpringApplicationContext.getBean(TaskResponseService.class);
- }
+ @Autowired
+ private TaskResponseService taskResponseService;
/**
* task final result response
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/StateEventResponseService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/StateEventResponseService.java
similarity index 98%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/StateEventResponseService.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/StateEventResponseService.java
index cb0b93c146..1db91c680e 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/StateEventResponseService.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/StateEventResponseService.java
@@ -108,7 +108,6 @@ public class StateEventResponseService {
StateEvent stateEvent = eventQueue.take();
persist(stateEvent);
} catch (InterruptedException e) {
- logger.warn("persist task error", e);
Thread.currentThread().interrupt();
break;
}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseEvent.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseEvent.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseEvent.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseEvent.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseService.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseService.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseService.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryDataListener.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryDataListener.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryDataListener.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryDataListener.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManager.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManager.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManager.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/EventExecuteService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/EventExecuteService.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/EventExecuteService.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/EventExecuteService.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java
similarity index 98%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java
index 4b48e68cd1..e13de47feb 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java
@@ -30,7 +30,6 @@ import org.apache.dolphinscheduler.remote.config.NettyClientConfig;
import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
import org.apache.dolphinscheduler.server.master.config.MasterConfig;
import org.apache.dolphinscheduler.server.master.dispatch.executor.NettyExecutorManager;
-import org.apache.dolphinscheduler.server.master.registry.MasterRegistryClient;
import org.apache.dolphinscheduler.server.master.registry.ServerNodeManager;
import org.apache.dolphinscheduler.server.master.runner.task.TaskProcessorFactory;
import org.apache.dolphinscheduler.service.alert.ProcessAlertManager;
@@ -70,12 +69,6 @@ public class MasterSchedulerService extends Thread {
@Autowired
private TaskProcessorFactory taskProcessorFactory;
- /**
- * zookeeper master client
- */
- @Autowired
- private MasterRegistryClient masterRegistryClient;
-
/**
* master config
*/
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThread.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThread.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThread.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThread.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/BaseTaskProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/BaseTaskProcessor.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/BaseTaskProcessor.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/BaseTaskProcessor.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessor.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessor.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessor.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/ConditionTaskProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/ConditionTaskProcessor.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/ConditionTaskProcessor.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/ConditionTaskProcessor.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/DependentTaskProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/DependentTaskProcessor.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/DependentTaskProcessor.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/DependentTaskProcessor.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/ITaskProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/ITaskProcessor.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/ITaskProcessor.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/ITaskProcessor.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/SubTaskProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/SubTaskProcessor.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/SubTaskProcessor.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/SubTaskProcessor.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/SwitchTaskProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/SwitchTaskProcessor.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/SwitchTaskProcessor.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/SwitchTaskProcessor.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/TaskAction.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/TaskAction.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/TaskAction.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/TaskAction.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/TaskProcessorFactory.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/TaskProcessorFactory.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/TaskProcessorFactory.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/TaskProcessorFactory.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/DependentExecute.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/utils/DependentExecute.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/DependentExecute.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/utils/DependentExecute.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/SwitchTaskUtils.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/utils/SwitchTaskUtils.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/SwitchTaskUtils.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/utils/SwitchTaskUtils.java
diff --git a/dolphinscheduler-server/src/main/resources/application-master.yaml b/dolphinscheduler-master/src/main/resources/application.yaml
similarity index 51%
rename from dolphinscheduler-server/src/main/resources/application-master.yaml
rename to dolphinscheduler-master/src/main/resources/application.yaml
index 12ed516369..6800496050 100644
--- a/dolphinscheduler-server/src/main/resources/application-master.yaml
+++ b/dolphinscheduler-master/src/main/resources/application.yaml
@@ -15,10 +15,12 @@
# limitations under the License.
#
spring:
+ main:
+ banner-mode: off
application:
name: master-server
cache:
- # default unable cache, you can disable by `type: caffeine`
+ # default enable cache, you can disable by `type: none`
type: none
cache-names:
- tenant
@@ -26,10 +28,58 @@ spring:
- processDefinition
- processTaskRelation
- taskDefinition
- - workerGroup
- - schedule
caffeine:
spec: maximumSize=100,expireAfterWrite=300s,recordStats
+ datasource:
+ driver-class-name: org.postgresql.Driver
+ url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler
+ username: root
+ password: root
+ hikari:
+ connection-test-query: select 1
+ minimum-idle: 5
+ auto-commit: true
+ validation-timeout: 3000
+ pool-name: DolphinScheduler
+ maximum-pool-size: 50
+ connection-timeout: 30000
+ idle-timeout: 600000
+ leak-detection-threshold: 0
+ initialization-fail-timeout: 1
+ quartz:
+ job-store-type: jdbc
+ jdbc:
+ initialize-schema: never
+ properties:
+ org.quartz.threadPool:threadPriority: 5
+ org.quartz.jobStore.isClustered: true
+ org.quartz.jobStore.class: org.quartz.impl.jdbcjobstore.JobStoreTX
+ org.quartz.scheduler.instanceId: AUTO
+ org.quartz.jobStore.tablePrefix: QRTZ_
+ org.quartz.jobStore.acquireTriggersWithinLock: true
+ org.quartz.scheduler.instanceName: DolphinScheduler
+ org.quartz.threadPool.class: org.quartz.simpl.SimpleThreadPool
+ org.quartz.jobStore.useProperties: false
+ org.quartz.threadPool.makeThreadsDaemons: true
+ org.quartz.threadPool.threadCount: 25
+ org.quartz.jobStore.misfireThreshold: 60000
+ org.quartz.scheduler.makeSchedulerThreadDaemon: true
+ org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.PostgreSQLDelegate
+ org.quartz.jobStore.clusterCheckinInterval: 5000
+
+registry:
+ type: zookeeper
+ zookeeper:
+ namespace: dolphinscheduler
+ connect-string: localhost:2181
+ retry-policy:
+ base-sleep-time: 60ms
+ max-sleep: 300ms
+ max-retries: 5
+ session-timeout: 30s
+ connection-timeout: 9s
+ block-until-connected: 600ms
+ digest: ~
master:
listen-port: 5678
@@ -66,3 +116,17 @@ management:
metrics:
tags:
application: ${spring.application.name}
+
+# Override by profile
+
+---
+spring:
+ config:
+ activate:
+ on-profile: mysql
+ datasource:
+ driver-class-name: com.mysql.jdbc.Driver
+ url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8
+ quartz:
+ properties:
+ org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.StdJDBCDelegate
diff --git a/dolphinscheduler-server/src/main/resources/logback-master.xml b/dolphinscheduler-master/src/main/resources/logback-spring.xml
similarity index 94%
rename from dolphinscheduler-server/src/main/resources/logback-master.xml
rename to dolphinscheduler-master/src/main/resources/logback-spring.xml
index d5d2ae3481..35bd657395 100644
--- a/dolphinscheduler-server/src/main/resources/logback-master.xml
+++ b/dolphinscheduler-master/src/main/resources/logback-spring.xml
@@ -16,9 +16,7 @@
~ limitations under the License.
-->
-
-
-
+
@@ -66,8 +64,12 @@
+
+
+
+
+
-
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/ConditionsTaskTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/ConditionsTaskTest.java
similarity index 100%
rename from dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/ConditionsTaskTest.java
rename to dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/ConditionsTaskTest.java
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/DependentTaskTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/DependentTaskTest.java
similarity index 100%
rename from dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/DependentTaskTest.java
rename to dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/DependentTaskTest.java
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterCommandTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/MasterCommandTest.java
similarity index 100%
rename from dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterCommandTest.java
rename to dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/MasterCommandTest.java
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/ParamsTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/ParamsTest.java
similarity index 100%
rename from dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/ParamsTest.java
rename to dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/ParamsTest.java
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/SubProcessTaskTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/SubProcessTaskTest.java
similarity index 100%
rename from dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/SubProcessTaskTest.java
rename to dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/SubProcessTaskTest.java
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/SwitchTaskTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/SwitchTaskTest.java
similarity index 100%
rename from dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/SwitchTaskTest.java
rename to dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/SwitchTaskTest.java
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/WorkflowExecuteThreadTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/WorkflowExecuteThreadTest.java
similarity index 100%
rename from dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/WorkflowExecuteThreadTest.java
rename to dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/WorkflowExecuteThreadTest.java
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/cache/impl/ProcessInstanceExecCacheManagerImplTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/cache/impl/ProcessInstanceExecCacheManagerImplTest.java
similarity index 100%
rename from dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/cache/impl/ProcessInstanceExecCacheManagerImplTest.java
rename to dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/cache/impl/ProcessInstanceExecCacheManagerImplTest.java
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/config/MasterConfigTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/config/MasterConfigTest.java
similarity index 100%
rename from dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/config/MasterConfigTest.java
rename to dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/config/MasterConfigTest.java
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumerTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumerTest.java
similarity index 100%
rename from dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumerTest.java
rename to dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumerTest.java
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ExecutionContextTestUtils.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutionContextTestUtils.java
similarity index 97%
rename from dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ExecutionContextTestUtils.java
rename to dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutionContextTestUtils.java
index 58edd9ac86..597c985b6d 100644
--- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ExecutionContextTestUtils.java
+++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutionContextTestUtils.java
@@ -14,8 +14,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.dolphinscheduler.server.utils;
+package org.apache.dolphinscheduler.server.master.dispatch;
import org.apache.dolphinscheduler.common.enums.CommandType;
import org.apache.dolphinscheduler.common.utils.NetUtils;
@@ -34,8 +34,6 @@ import org.mockito.Mockito;
* for test use only
*/
public class ExecutionContextTestUtils {
-
-
public static ExecutionContext getExecutionContext(int port){
TaskInstance taskInstance = Mockito.mock(TaskInstance.class);
ProcessDefinition processDefinition = Mockito.mock(ProcessDefinition.class);
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcherTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcherTest.java
similarity index 97%
rename from dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcherTest.java
rename to dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcherTest.java
index 619dba84bd..4bfc3ca0a6 100644
--- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcherTest.java
+++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcherTest.java
@@ -21,7 +21,6 @@ import org.apache.dolphinscheduler.remote.NettyRemotingServer;
import org.apache.dolphinscheduler.remote.config.NettyServerConfig;
import org.apache.dolphinscheduler.server.master.dispatch.context.ExecutionContext;
import org.apache.dolphinscheduler.server.master.dispatch.exceptions.ExecuteException;
-import org.apache.dolphinscheduler.server.utils.ExecutionContextTestUtils;
import org.apache.dolphinscheduler.server.worker.config.WorkerConfig;
import org.apache.dolphinscheduler.server.worker.processor.TaskExecuteProcessor;
import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistryClient;
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManagerTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManagerTest.java
similarity index 100%
rename from dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManagerTest.java
rename to dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManagerTest.java
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RefreshResourceTaskTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RefreshResourceTaskTest.java
similarity index 100%
rename from dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RefreshResourceTaskTest.java
rename to dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RefreshResourceTaskTest.java
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java
similarity index 96%
rename from dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java
rename to dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java
index f29f66e993..0707eade28 100644
--- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java
+++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java
@@ -18,9 +18,9 @@
package org.apache.dolphinscheduler.server.master.dispatch.host;
import org.apache.dolphinscheduler.remote.utils.Host;
+import org.apache.dolphinscheduler.server.master.dispatch.ExecutionContextTestUtils;
import org.apache.dolphinscheduler.server.master.dispatch.context.ExecutionContext;
import org.apache.dolphinscheduler.server.master.registry.ServerNodeManager;
-import org.apache.dolphinscheduler.server.utils.ExecutionContextTestUtils;
import org.apache.commons.lang.StringUtils;
@@ -34,7 +34,6 @@ import org.mockito.junit.MockitoJUnitRunner;
import com.google.common.collect.Sets;
-
/**
* round robin host manager test
*/
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWorkerTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWorkerTest.java
similarity index 100%
rename from dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWorkerTest.java
rename to dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWorkerTest.java
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/LowerWeightRoundRobinTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/LowerWeightRoundRobinTest.java
similarity index 100%
rename from dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/LowerWeightRoundRobinTest.java
rename to dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/LowerWeightRoundRobinTest.java
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RandomSelectorTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RandomSelectorTest.java
similarity index 100%
rename from dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RandomSelectorTest.java
rename to dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RandomSelectorTest.java
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelectorTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelectorTest.java
similarity index 100%
rename from dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelectorTest.java
rename to dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelectorTest.java
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/processor/CacheProcessorTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/CacheProcessorTest.java
similarity index 100%
rename from dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/processor/CacheProcessorTest.java
rename to dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/CacheProcessorTest.java
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessorTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessorTest.java
similarity index 100%
rename from dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessorTest.java
rename to dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessorTest.java
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/processor/TaskKillResponseProcessorTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/TaskKillResponseProcessorTest.java
similarity index 100%
rename from dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/processor/TaskKillResponseProcessorTest.java
rename to dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/TaskKillResponseProcessorTest.java
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseServiceTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseServiceTest.java
similarity index 100%
rename from dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseServiceTest.java
rename to dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseServiceTest.java
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClientTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClientTest.java
similarity index 100%
rename from dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClientTest.java
rename to dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClientTest.java
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManagerTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManagerTest.java
similarity index 100%
rename from dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManagerTest.java
rename to dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManagerTest.java
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThreadTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThreadTest.java
similarity index 100%
rename from dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThreadTest.java
rename to dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThreadTest.java
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessorTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessorTest.java
similarity index 100%
rename from dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessorTest.java
rename to dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessorTest.java
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/runner/task/TaskProcessorFactoryTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/task/TaskProcessorFactoryTest.java
similarity index 99%
rename from dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/runner/task/TaskProcessorFactoryTest.java
rename to dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/task/TaskProcessorFactoryTest.java
index 2dd349cfc2..ffef87c8bc 100644
--- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/runner/task/TaskProcessorFactoryTest.java
+++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/task/TaskProcessorFactoryTest.java
@@ -38,4 +38,4 @@ public class TaskProcessorFactoryTest {
Assert.assertNotNull(iTaskProcessor);
}
-}
\ No newline at end of file
+}
diff --git a/dolphinscheduler-meter/pom.xml b/dolphinscheduler-meter/pom.xml
index d1d62d92c1..5ff8e594c3 100644
--- a/dolphinscheduler-meter/pom.xml
+++ b/dolphinscheduler-meter/pom.xml
@@ -34,10 +34,26 @@
org.springframework.boot
spring-boot-starter-actuator
+
+
+ org.springframework.boot
+ spring-boot-starter-logging
+
+
org.springframework.boot
spring-boot-starter-web
+
+
+ org.springframework.boot
+ spring-boot-starter-tomcat
+
+
+ log4j-to-slf4j
+ org.apache.logging.log4j
+
+
org.springframework.boot
diff --git a/dolphinscheduler-python/pom.xml b/dolphinscheduler-python/pom.xml
index ce7968cd1f..7076689473 100644
--- a/dolphinscheduler-python/pom.xml
+++ b/dolphinscheduler-python/pom.xml
@@ -18,43 +18,43 @@
-4.0.0
-
- org.apache.dolphinscheduler
- dolphinscheduler
- 2.0.0-SNAPSHOT
-
-dolphinscheduler-python
-${project.artifactId}
-jar
-
-
-
-
+ 4.0.0
+
org.apache.dolphinscheduler
- dolphinscheduler-api
-
+ dolphinscheduler
+ 2.0.0-SNAPSHOT
+
+ dolphinscheduler-python
+ ${project.artifactId}
+ jar
+
+
+
+
+ org.apache.dolphinscheduler
+ dolphinscheduler-api
+
-
-
- org.springframework.boot
- spring-boot-starter-web
-
-
- org.springframework.boot
- spring-boot-starter-tomcat
-
-
- log4j-to-slf4j
- org.apache.logging.log4j
-
-
-
+
+
+ org.springframework.boot
+ spring-boot-starter-web
+
+
+ org.springframework.boot
+ spring-boot-starter-tomcat
+
+
+ log4j-to-slf4j
+ org.apache.logging.log4j
+
+
+
-
- net.sf.py4j
- py4j
-
+
+ net.sf.py4j
+ py4j
+
-
+
diff --git a/dolphinscheduler-python/src/main/java/org/apache/dolphinscheduler/server/PythonGatewayServer.java b/dolphinscheduler-python/src/main/java/org/apache/dolphinscheduler/server/PythonGatewayServer.java
index 46eb717339..e7bbab8e40 100644
--- a/dolphinscheduler-python/src/main/java/org/apache/dolphinscheduler/server/PythonGatewayServer.java
+++ b/dolphinscheduler-python/src/main/java/org/apache/dolphinscheduler/server/PythonGatewayServer.java
@@ -64,19 +64,10 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.web.servlet.support.SpringBootServletInitializer;
import org.springframework.context.annotation.ComponentScan;
-import org.springframework.context.annotation.FilterType;
import py4j.GatewayServer;
-@ComponentScan(value = "org.apache.dolphinscheduler", excludeFilters = {
- @ComponentScan.Filter(type = FilterType.REGEX, pattern = {
- "org.apache.dolphinscheduler.server.master.*",
- "org.apache.dolphinscheduler.server.worker.*",
- "org.apache.dolphinscheduler.server.monitor.*",
- "org.apache.dolphinscheduler.server.log.*",
- "org.apache.dolphinscheduler.alert.*"
- })
-})
+@ComponentScan(value = "org.apache.dolphinscheduler")
public class PythonGatewayServer extends SpringBootServletInitializer {
private static final Logger LOGGER = LoggerFactory.getLogger(PythonGatewayServer.class);
diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/Registry.java b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/Registry.java
index 6057a7e2e4..51ceae63f1 100644
--- a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/Registry.java
+++ b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/Registry.java
@@ -24,8 +24,6 @@ import java.util.Collection;
import java.util.Map;
public interface Registry extends Closeable {
- void start(Map config);
-
boolean subscribe(String path, SubscribeListener listener);
void unsubscribe(String path);
diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/RegistryFactory.java b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/RegistryFactory.java
deleted file mode 100644
index 6903e95bbd..0000000000
--- a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/RegistryFactory.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to Apache Software Foundation (ASF) under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Apache Software Foundation (ASF) licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.dolphinscheduler.registry.api;
-
-public interface RegistryFactory {
- String name();
-
- Registry create();
-}
diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/RegistryFactoryLoader.java b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/RegistryFactoryLoader.java
deleted file mode 100644
index 45d4596e28..0000000000
--- a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/RegistryFactoryLoader.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to Apache Software Foundation (ASF) under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Apache Software Foundation (ASF) licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.dolphinscheduler.registry.api;
-
-import static java.util.stream.Collectors.toMap;
-
-import java.util.Map;
-import java.util.ServiceLoader;
-import java.util.function.Function;
-import java.util.stream.StreamSupport;
-
-public final class RegistryFactoryLoader {
- public static Map load() {
- final ServiceLoader factories = ServiceLoader.load(RegistryFactory.class);
- return StreamSupport.stream(factories.spliterator(), false)
- .collect(toMap(RegistryFactory::name, Function.identity()));
- }
-}
diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/RegistryProperties.java b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/RegistryProperties.java
new file mode 100644
index 0000000000..dbe9ac1046
--- /dev/null
+++ b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/RegistryProperties.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.dolphinscheduler.registry.api;
+
+import java.time.Duration;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+@ConfigurationProperties(prefix = "registry")
+public class RegistryProperties {
+ private Type type;
+ private ZookeeperProperties zookeeper = new ZookeeperProperties();
+
+ public Type getType() {
+ return type;
+ }
+
+ public void setType(Type type) {
+ this.type = type;
+ }
+
+ public ZookeeperProperties getZookeeper() {
+ return zookeeper;
+ }
+
+ public void setZookeeper(ZookeeperProperties zookeeper) {
+ this.zookeeper = zookeeper;
+ }
+
+ public enum Type {
+ ZOOKEEPER
+ }
+
+ public static final class ZookeeperProperties {
+ private String namespace;
+ private String connectString;
+ private RetryPolicy retryPolicy = new RetryPolicy();
+ private String digest;
+ private Duration sessionTimeout = Duration.ofSeconds(30);
+ private Duration connectionTimeout = Duration.ofSeconds(9);
+ private Duration blockUntilConnected = Duration.ofMillis(600);
+
+ public String getNamespace() {
+ return namespace;
+ }
+
+ public void setNamespace(String namespace) {
+ this.namespace = namespace;
+ }
+
+ public String getConnectString() {
+ return connectString;
+ }
+
+ public void setConnectString(String connectString) {
+ this.connectString = connectString;
+ }
+
+ public RetryPolicy getRetryPolicy() {
+ return retryPolicy;
+ }
+
+ public void setRetryPolicy(RetryPolicy retryPolicy) {
+ this.retryPolicy = retryPolicy;
+ }
+
+ public String getDigest() {
+ return digest;
+ }
+
+ public void setDigest(String digest) {
+ this.digest = digest;
+ }
+
+ public Duration getSessionTimeout() {
+ return sessionTimeout;
+ }
+
+ public void setSessionTimeout(Duration sessionTimeout) {
+ this.sessionTimeout = sessionTimeout;
+ }
+
+ public Duration getConnectionTimeout() {
+ return connectionTimeout;
+ }
+
+ public void setConnectionTimeout(Duration connectionTimeout) {
+ this.connectionTimeout = connectionTimeout;
+ }
+
+ public Duration getBlockUntilConnected() {
+ return blockUntilConnected;
+ }
+
+ public void setBlockUntilConnected(Duration blockUntilConnected) {
+ this.blockUntilConnected = blockUntilConnected;
+ }
+
+ public static final class RetryPolicy {
+ private Duration baseSleepTime = Duration.ofMillis(60);
+ private int maxRetries;
+ private Duration maxSleep = Duration.ofMillis(300);
+
+ public Duration getBaseSleepTime() {
+ return baseSleepTime;
+ }
+
+ public void setBaseSleepTime(Duration baseSleepTime) {
+ this.baseSleepTime = baseSleepTime;
+ }
+
+ public int getMaxRetries() {
+ return maxRetries;
+ }
+
+ public void setMaxRetries(int maxRetries) {
+ this.maxRetries = maxRetries;
+ }
+
+ public Duration getMaxSleep() {
+ return maxSleep;
+ }
+
+ public void setMaxSleep(Duration maxSleep) {
+ this.maxSleep = maxSleep;
+ }
+ }
+ }
+
+}
diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/pom.xml b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/pom.xml
index 732237e875..74a942c155 100644
--- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/pom.xml
+++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/pom.xml
@@ -42,6 +42,10 @@
org.apache.curator
curator-framework
+
+ org.apache.curator
+ curator-client
+
org.apache.curator
diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperConfiguration.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperConfiguration.java
deleted file mode 100644
index ffe1f0edbd..0000000000
--- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperConfiguration.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.plugin.registry.zookeeper;
-
-import java.util.function.Function;
-
-public enum ZookeeperConfiguration {
-
- NAME_SPACE("namespace", "dolphinscheduler", value -> value),
- SERVERS("servers", null, value -> value),
-
- /**
- * Initial amount of time to wait between retries
- */
- BASE_SLEEP_TIME("base.sleep.time.ms", 60, Integer::valueOf),
- MAX_SLEEP_TIME("max.sleep.ms", 300, Integer::valueOf),
- DIGEST("digest", null, value -> value),
-
- MAX_RETRIES("max.retries", 5, Integer::valueOf),
-
-
- SESSION_TIMEOUT_MS("session.timeout.ms", 30000, Integer::valueOf),
- CONNECTION_TIMEOUT_MS("connection.timeout.ms", 7500, Integer::valueOf),
-
- BLOCK_UNTIL_CONNECTED_WAIT_MS("block.until.connected.wait", 600, Integer::valueOf),
- ;
- private final String name;
-
- public String getName() {
- return name;
- }
-
- private final Object defaultValue;
-
- private final Function converter;
-
- ZookeeperConfiguration(String name, T defaultValue, Function converter) {
- this.name = name;
- this.defaultValue = defaultValue;
- this.converter = (Function) converter;
- }
-
- public T getParameterValue(String param) {
- Object value = param != null ? converter.apply(param) : defaultValue;
- return (T) value;
- }
-
-}
diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistry.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistry.java
index 89cb28006d..1719fb5386 100644
--- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistry.java
+++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistry.java
@@ -17,24 +17,16 @@
package org.apache.dolphinscheduler.plugin.registry.zookeeper;
-import static org.apache.dolphinscheduler.plugin.registry.zookeeper.ZookeeperConfiguration.BASE_SLEEP_TIME;
-import static org.apache.dolphinscheduler.plugin.registry.zookeeper.ZookeeperConfiguration.BLOCK_UNTIL_CONNECTED_WAIT_MS;
-import static org.apache.dolphinscheduler.plugin.registry.zookeeper.ZookeeperConfiguration.CONNECTION_TIMEOUT_MS;
-import static org.apache.dolphinscheduler.plugin.registry.zookeeper.ZookeeperConfiguration.DIGEST;
-import static org.apache.dolphinscheduler.plugin.registry.zookeeper.ZookeeperConfiguration.MAX_RETRIES;
-import static org.apache.dolphinscheduler.plugin.registry.zookeeper.ZookeeperConfiguration.NAME_SPACE;
-import static org.apache.dolphinscheduler.plugin.registry.zookeeper.ZookeeperConfiguration.SERVERS;
-import static org.apache.dolphinscheduler.plugin.registry.zookeeper.ZookeeperConfiguration.SESSION_TIMEOUT_MS;
-
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import org.apache.dolphinscheduler.registry.api.ConnectionListener;
import org.apache.dolphinscheduler.registry.api.Event;
import org.apache.dolphinscheduler.registry.api.Registry;
import org.apache.dolphinscheduler.registry.api.RegistryException;
+import org.apache.dolphinscheduler.registry.api.RegistryProperties;
+import org.apache.dolphinscheduler.registry.api.RegistryProperties.ZookeeperProperties;
import org.apache.dolphinscheduler.registry.api.SubscribeListener;
-import org.apache.curator.RetryPolicy;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.framework.api.ACLProvider;
@@ -56,25 +48,48 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
+import javax.annotation.PostConstruct;
+
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Component;
+
import com.google.common.base.Strings;
+@Component
+@ConditionalOnProperty(prefix = "registry", name = "type", havingValue = "zookeeper")
public final class ZookeeperRegistry implements Registry {
-
- private CuratorFramework client;
+ private final ZookeeperProperties properties;
+ private final CuratorFramework client;
private final Map treeCacheMap = new ConcurrentHashMap<>();
private static final ThreadLocal> threadLocalLockMap = new ThreadLocal<>();
- private static RetryPolicy buildRetryPolicy(Map registerData) {
- int baseSleepTimeMs = BASE_SLEEP_TIME.getParameterValue(registerData.get(BASE_SLEEP_TIME.getName()));
- int maxRetries = MAX_RETRIES.getParameterValue(registerData.get(MAX_RETRIES.getName()));
- int maxSleepMs = baseSleepTimeMs * maxRetries;
- return new ExponentialBackoffRetry(baseSleepTimeMs, maxRetries, maxSleepMs);
+ public ZookeeperRegistry(RegistryProperties registryProperties) {
+ properties = registryProperties.getZookeeper();
+
+ final ExponentialBackoffRetry retryPolicy = new ExponentialBackoffRetry(
+ (int) properties.getRetryPolicy().getBaseSleepTime().toMillis(),
+ properties.getRetryPolicy().getMaxRetries(),
+ (int) properties.getRetryPolicy().getMaxSleep().toMillis());
+
+ CuratorFrameworkFactory.Builder builder =
+ CuratorFrameworkFactory.builder()
+ .connectString(properties.getConnectString())
+ .retryPolicy(retryPolicy)
+ .namespace(properties.getNamespace())
+ .sessionTimeoutMs((int) properties.getSessionTimeout().toMillis())
+ .connectionTimeoutMs((int) properties.getConnectionTimeout().toMillis());
+
+ final String digest = properties.getDigest();
+ if (!Strings.isNullOrEmpty(digest)) {
+ buildDigest(builder, digest);
+ }
+ client = builder.build();
}
- private static void buildDigest(CuratorFrameworkFactory.Builder builder, String digest) {
- builder.authorization(DIGEST.getName(), digest.getBytes(StandardCharsets.UTF_8))
+ private void buildDigest(CuratorFrameworkFactory.Builder builder, String digest) {
+ builder.authorization("digest", digest.getBytes(StandardCharsets.UTF_8))
.aclProvider(new ACLProvider() {
@Override
public List getDefaultAcl() {
@@ -88,31 +103,16 @@ public final class ZookeeperRegistry implements Registry {
});
}
- @Override
- public void start(Map config) {
- CuratorFrameworkFactory.Builder builder =
- CuratorFrameworkFactory.builder()
- .connectString(SERVERS.getParameterValue(config.get(SERVERS.getName())))
- .retryPolicy(buildRetryPolicy(config))
- .namespace(NAME_SPACE.getParameterValue(config.get(NAME_SPACE.getName())))
- .sessionTimeoutMs(SESSION_TIMEOUT_MS.getParameterValue(config.get(SESSION_TIMEOUT_MS.getName())))
- .connectionTimeoutMs(CONNECTION_TIMEOUT_MS.getParameterValue(config.get(CONNECTION_TIMEOUT_MS.getName())));
-
- String digest = DIGEST.getParameterValue(config.get(DIGEST.getName()));
- if (!Strings.isNullOrEmpty(digest)) {
- buildDigest(builder, digest);
- }
- client = builder.build();
-
+ @PostConstruct
+ public void start() {
client.start();
try {
- if (!client.blockUntilConnected(BLOCK_UNTIL_CONNECTED_WAIT_MS.getParameterValue(config.get(BLOCK_UNTIL_CONNECTED_WAIT_MS.getName())), MILLISECONDS)) {
+ if (!client.blockUntilConnected((int) properties.getBlockUntilConnected().toMillis(), MILLISECONDS)) {
client.close();
- throw new RegistryException("zookeeper connect timeout");
+ throw new RegistryException("zookeeper connect timeout: " + properties.getConnectString());
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
- throw new RegistryException("zookeeper connect error", e);
}
}
diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/test/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryTest.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/test/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryTest.java
index 8442c02c70..312153addc 100644
--- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/test/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryTest.java
+++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/test/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryTest.java
@@ -18,6 +18,7 @@
package org.apache.dolphinscheduler.plugin.registry.zookeeper;
import org.apache.dolphinscheduler.registry.api.Event;
+import org.apache.dolphinscheduler.registry.api.RegistryProperties;
import org.apache.dolphinscheduler.registry.api.SubscribeListener;
import org.apache.curator.test.TestingServer;
@@ -25,9 +26,7 @@ import org.apache.curator.test.TestingServer;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
import java.util.concurrent.CountDownLatch;
import org.junit.After;
@@ -43,14 +42,16 @@ public class ZookeeperRegistryTest {
TestingServer server;
- ZookeeperRegistry registry = new ZookeeperRegistry();
+ ZookeeperRegistry registry;
@Before
public void before() throws Exception {
server = new TestingServer(true);
- Map registryConfig = new HashMap<>();
- registryConfig.put(ZookeeperConfiguration.SERVERS.getName(), server.getConnectString());
- registry.start(registryConfig);
+
+ RegistryProperties p = new RegistryProperties();
+ p.getZookeeper().setConnectString(server.getConnectString());
+ registry = new ZookeeperRegistry(p);
+ registry.start();
registry.put("/sub", "", false);
}
diff --git a/dolphinscheduler-registry/pom.xml b/dolphinscheduler-registry/pom.xml
index 4b51019a00..133aca5260 100644
--- a/dolphinscheduler-registry/pom.xml
+++ b/dolphinscheduler-registry/pom.xml
@@ -27,6 +27,24 @@
dolphinscheduler-registry
pom
+
+
+ org.springframework
+ spring-context
+ provided
+
+
+ org.springframework.boot
+ spring-boot-autoconfigure
+ provided
+
+
+ javax.annotation
+ javax.annotation-api
+ provided
+
+
+
dolphinscheduler-registry-api
dolphinscheduler-registry-plugins
diff --git a/dolphinscheduler-remote/pom.xml b/dolphinscheduler-remote/pom.xml
index 788a7c41bd..8136f7dc03 100644
--- a/dolphinscheduler-remote/pom.xml
+++ b/dolphinscheduler-remote/pom.xml
@@ -66,12 +66,6 @@
com.google.guava
guava
-
-
- com.google.code.findbugs
- jsr305
-
-
org.springframework
diff --git a/dolphinscheduler-server/pom.xml b/dolphinscheduler-server/pom.xml
index 47ff4f6bcb..cf525d6d1a 100644
--- a/dolphinscheduler-server/pom.xml
+++ b/dolphinscheduler-server/pom.xml
@@ -32,108 +32,18 @@
org.apache.dolphinscheduler
- dolphinscheduler-service
-
-
- org.apache.dolphinscheduler
- dolphinscheduler-meter
-
-
- org.apache.dolphinscheduler
- dolphinscheduler-spi
-
-
- org.apache.dolphinscheduler
- dolphinscheduler-task-datax
-
-
- org.apache.dolphinscheduler
- dolphinscheduler-task-flink
-
-
- org.apache.dolphinscheduler
- dolphinscheduler-task-http
-
-
- org.apache.dolphinscheduler
- dolphinscheduler-task-mr
-
-
- org.apache.dolphinscheduler
- dolphinscheduler-task-pigeon
-
-
- org.apache.dolphinscheduler
- dolphinscheduler-task-procedure
-
-
- org.apache.dolphinscheduler
- dolphinscheduler-task-python
+ dolphinscheduler-common
org.apache.dolphinscheduler
- dolphinscheduler-task-shell
-
-
- org.apache.dolphinscheduler
- dolphinscheduler-task-spark
-
-
- org.apache.dolphinscheduler
- dolphinscheduler-task-sql
-
-
- org.apache.dolphinscheduler
- dolphinscheduler-task-sqoop
+ dolphinscheduler-service
-
- org.apache.httpcomponents
- httpclient
-
-
- org.apache.httpcomponents
- httpcore
-
-
- com.google.guava
- guava
-
-
- com.google.code.findbugs
- jsr305
-
-
-
org.mockito
mockito-core
test
-
- org.springframework
- spring-test
- test
-
-
- org.springframework.boot
- spring-boot-test
- test
-
-
- org.springframework.boot
- spring-boot-starter-cache
-
-
- log4j-api
- org.apache.logging.log4j
-
-
- log4j-to-slf4j
- org.apache.logging.log4j
-
-
-
@@ -144,8 +54,6 @@
config/
- *.yaml
- *.xml
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/SensitiveDataConverter.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/SensitiveDataConverter.java
index 8413fed576..ee37a871d0 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/SensitiveDataConverter.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/SensitiveDataConverter.java
@@ -17,8 +17,6 @@
package org.apache.dolphinscheduler.server.log;
-import static org.apache.dolphinscheduler.common.Constants.STAR;
-
import org.apache.dolphinscheduler.common.Constants;
import org.apache.commons.lang.StringUtils;
@@ -80,7 +78,7 @@ public class SensitiveDataConverter extends MessageConverter {
String password = matcher.group();
- String maskPassword = StringUtils.repeat(STAR, StringUtils.length(password));
+ String maskPassword = StringUtils.repeat(Constants.STAR, StringUtils.length(password));
matcher.appendReplacement(sb, maskPassword);
}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/TaskLogFilter.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/TaskLogFilter.java
index 9c47fb901f..ec93b85c76 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/TaskLogFilter.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/TaskLogFilter.java
@@ -16,8 +16,6 @@
*/
package org.apache.dolphinscheduler.server.log;
-import static org.apache.dolphinscheduler.common.utils.LoggerUtils.TASK_APPID_LOG_FORMAT;
-
import org.apache.dolphinscheduler.common.utils.LoggerUtils;
import ch.qos.logback.classic.Level;
@@ -47,10 +45,10 @@ public class TaskLogFilter extends Filter {
@Override
public FilterReply decide(ILoggingEvent event) {
if (event.getThreadName().startsWith(LoggerUtils.TASK_LOGGER_THREAD_NAME)
- || event.getLoggerName().startsWith(" - " + TASK_APPID_LOG_FORMAT)
+ || event.getLoggerName().startsWith(" - " + LoggerUtils.TASK_APPID_LOG_FORMAT)
|| event.getLevel().isGreaterOrEqual(level)) {
return FilterReply.ACCEPT;
}
return FilterReply.DENY;
}
-}
\ No newline at end of file
+}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/AbstractMonitor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/AbstractMonitor.java
deleted file mode 100644
index 2640d8d663..0000000000
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/AbstractMonitor.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.dolphinscheduler.server.monitor;
-
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * abstract server monitor and auto restart server
- */
-@Component
-public abstract class AbstractMonitor implements Monitor {
-
- private static final Logger logger = LoggerFactory.getLogger(AbstractMonitor.class);
-
-
- @Autowired
- private RunConfig runConfig;
-
- /**
- * monitor server and restart
- */
- @Override
- public void monitor(String masterPath,String workerPath,Integer port,String installPath) {
- try {
- restartServer(masterPath,port,installPath);
- restartServer(workerPath,port,installPath);
- }catch (Exception e){
- logger.error("server start up error",e);
- }
- }
-
- private void restartServer(String path,Integer port,String installPath) throws Exception{
-
- String type = path.split("/")[2];
- String serverName = null;
- String nodes = null;
- if ("masters".equals(type)){
- serverName = "master-server";
- nodes = runConfig.getMasters();
- }else if ("workers".equals(type)){
- serverName = "worker-server";
- nodes = runConfig.getWorkers();
- }
-
- Map activeNodeMap = getActiveNodesByPath(path);
-
- Set needRestartServer = getNeedRestartServer(getRunConfigServer(nodes),
- activeNodeMap.keySet());
-
- for (String node : needRestartServer){
- // os.system('ssh -p ' + ssh_port + ' ' + self.get_ip_by_hostname(master) + ' sh ' + install_path + '/bin/dolphinscheduler-daemon.sh start master-server')
- String runCmd = "ssh -p " + port + " " + node + " sh " + installPath + "/bin/dolphinscheduler-daemon.sh start " + serverName;
- Runtime.getRuntime().exec(runCmd);
- }
- }
-
- /**
- * get need restart server
- * @param deployedNodes deployedNodes
- * @param activeNodes activeNodes
- * @return need restart server
- */
- private Set getNeedRestartServer(Set deployedNodes,Set activeNodes){
- if (CollectionUtils.isEmpty(activeNodes)){
- return deployedNodes;
- }
-
- Set result = new HashSet<>();
-
- result.addAll(deployedNodes);
- result.removeAll(activeNodes);
-
- return result;
- }
-
- /**
- * run config masters/workers
- * @return master set/worker set
- */
- private Set getRunConfigServer(String nodes){
- Set nodeSet = new HashSet();
-
-
- if (StringUtils.isEmpty(nodes)){
- return null;
- }
-
- String[] nodeArr = nodes.split(",");
-
- for (String node : nodeArr){
- nodeSet.add(node);
- }
-
- return nodeSet;
- }
-
- /**
- * get active nodes by path
- * @param path path
- * @return active nodes
- */
- protected abstract Map getActiveNodesByPath(String path);
-}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/Monitor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/Monitor.java
deleted file mode 100644
index 8d7bf0bb89..0000000000
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/Monitor.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.dolphinscheduler.server.monitor;
-
-/**
- * server monitor and auto restart server
- */
-public interface Monitor {
-
- /**
- * monitor server and restart
- *
- * @param masterPath masterPath
- * @param workerPath workerPath
- * @param port port
- * @param installPath installPath
- */
- void monitor(String masterPath, String workerPath, Integer port, String installPath);
-}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/MonitorServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/MonitorServer.java
deleted file mode 100644
index a1f43add6e..0000000000
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/MonitorServer.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.dolphinscheduler.server.monitor;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.CommandLineRunner;
-import org.springframework.boot.WebApplicationType;
-import org.springframework.boot.builder.SpringApplicationBuilder;
-import org.springframework.context.annotation.ComponentScan;
-
-/**
- * monitor server
- */
-@ComponentScan("org.apache.dolphinscheduler")
-public class MonitorServer implements CommandLineRunner {
-
- private static final Integer ARGS_LENGTH = 4;
-
- private static final Logger logger = LoggerFactory.getLogger(MonitorServer.class);
-
- /**
- * monitor
- */
- @Autowired
- private Monitor monitor;
-
-
-
- public static void main(String[] args) throws Exception{
-
- new SpringApplicationBuilder(MonitorServer.class).web(WebApplicationType.NONE).run(args);
- }
-
- @Override
- public void run(String... args) throws Exception {
- if (args.length != ARGS_LENGTH){
- logger.error("Usage: ");
- return;
- }
-
- String masterPath = args[0];
- String workerPath = args[1];
- Integer port = Integer.parseInt(args[2]);
- String installPath = args[3];
- monitor.monitor(masterPath,workerPath,port,installPath);
- }
-}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/RegistryMonitorImpl.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/RegistryMonitorImpl.java
deleted file mode 100644
index 9e3bb0c65b..0000000000
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/RegistryMonitorImpl.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.monitor;
-
-import org.apache.dolphinscheduler.service.registry.RegistryClient;
-
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-
-@Component
-public class RegistryMonitorImpl extends AbstractMonitor {
-
- @Autowired
- private RegistryClient registryClient;
-
- @Override
- protected Map getActiveNodesByPath(String path) {
-
- Map maps = new HashMap<>();
-
- Collection childrenList = registryClient.getChildrenKeys(path);
-
- if (childrenList == null) {
- return maps;
- }
-
- for (String child : childrenList) {
- maps.put(child.split("_")[0], child);
- }
-
- return maps;
- }
-}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/RunConfig.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/RunConfig.java
deleted file mode 100644
index 6aac184fce..0000000000
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/RunConfig.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.dolphinscheduler.server.monitor;
-
-import org.apache.dolphinscheduler.common.Constants;
-
-import org.apache.commons.lang.StringUtils;
-
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.context.annotation.PropertySource;
-import org.springframework.stereotype.Component;
-
-/**
- * zookeeper conf
- */
-@Component
-@PropertySource("classpath:config/install_config.conf")
-public class RunConfig {
-
- //zk connect config
- @Value("${masters}")
- private String masters;
-
- @Value("${workers}")
- private String workers;
-
- @Value("${alertServer}")
- private String alertServer;
-
- @Value("${apiServers}")
- private String apiServers;
-
- @Value("${sshPort}")
- private String sshPort;
-
- public String getMasters() {
- return masters;
- }
-
- public void setMasters(String masters) {
- this.masters = masters;
- }
-
- public String getWorkers() {
- StringBuilder sb = new StringBuilder(50);
- if(StringUtils.isNotBlank(workers)){
- String[] workersArr = workers.trim().split(Constants.COMMA);
- for (String workerGroup : workersArr) {
- sb.append(workerGroup.split(Constants.COLON)[0]).append(Constants.COMMA);
- }
- }
- if( sb.length() > 0) {
- sb.deleteCharAt(sb.length() - 1);
- }
-
- return sb.toString();
- }
-
- public void setWorkers(String workers) {
- this.workers = workers;
- }
-
- public String getAlertServer() {
- return alertServer;
- }
-
- public void setAlertServer(String alertServer) {
- this.alertServer = alertServer;
- }
-
- public String getApiServers() {
- return apiServers;
- }
-
- public void setApiServers(String apiServers) {
- this.apiServers = apiServers;
- }
-
- public String getSshPort() {
- return sshPort;
- }
-
- public void setSshPort(String sshPort) {
- this.sshPort = sshPort;
- }
-}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/registry/HeartBeatTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/registry/HeartBeatTask.java
index 67fd07a892..17b4df14a9 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/registry/HeartBeatTask.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/registry/HeartBeatTask.java
@@ -18,7 +18,6 @@
package org.apache.dolphinscheduler.server.registry;
import org.apache.dolphinscheduler.common.utils.HeartBeat;
-import org.apache.dolphinscheduler.server.worker.runner.WorkerManagerThread;
import org.apache.dolphinscheduler.service.registry.RegistryClient;
import java.util.Set;
@@ -35,7 +34,7 @@ public class HeartBeatTask implements Runnable {
private final Set heartBeatPaths;
private final RegistryClient registryClient;
- private WorkerManagerThread workerManagerThread;
+ private int workerWaitingTaskCount;
private final String serverType;
private final HeartBeat heartBeat;
@@ -59,11 +58,11 @@ public class HeartBeatTask implements Runnable {
String serverType,
RegistryClient registryClient,
int workerThreadCount,
- WorkerManagerThread workerManagerThread
+ int workerWaitingTaskCount
) {
this.heartBeatPaths = heartBeatPaths;
this.registryClient = registryClient;
- this.workerManagerThread = workerManagerThread;
+ this.workerWaitingTaskCount = workerWaitingTaskCount;
this.serverType = serverType;
this.heartBeat = new HeartBeat(startupTime, maxCpuloadAvg, reservedMemory, hostWeight, workerThreadCount);
}
@@ -83,10 +82,8 @@ public class HeartBeatTask implements Runnable {
}
}
- if (workerManagerThread != null) {
- // update waiting task count
- heartBeat.setWorkerWaitingTaskCount(workerManagerThread.getThreadPoolQueueSize());
- }
+ // update waiting task count
+ heartBeat.setWorkerWaitingTaskCount(workerWaitingTaskCount);
for (String heartBeatPath : heartBeatPaths) {
registryClient.persistEphemeral(heartBeatPath, heartBeat.encodeHeartBeat());
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/TaskLogDiscriminatorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/TaskLogDiscriminatorTest.java
index b9a9353ae6..65b92e6379 100644
--- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/TaskLogDiscriminatorTest.java
+++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/TaskLogDiscriminatorTest.java
@@ -150,4 +150,4 @@ public class TaskLogDiscriminatorTest {
public void setLogBase() {
taskLogDiscriminator.setLogBase("logs");
}
-}
\ No newline at end of file
+}
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/registry/DependencyConfig.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/registry/DependencyConfig.java
deleted file mode 100644
index 8d1faa80a9..0000000000
--- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/registry/DependencyConfig.java
+++ /dev/null
@@ -1,189 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.registry;
-
-import org.apache.dolphinscheduler.dao.AlertDao;
-import org.apache.dolphinscheduler.dao.mapper.AlertGroupMapper;
-import org.apache.dolphinscheduler.dao.mapper.AlertMapper;
-import org.apache.dolphinscheduler.dao.mapper.AlertPluginInstanceMapper;
-import org.apache.dolphinscheduler.dao.mapper.CommandMapper;
-import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper;
-import org.apache.dolphinscheduler.dao.mapper.ErrorCommandMapper;
-import org.apache.dolphinscheduler.dao.mapper.PluginDefineMapper;
-import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper;
-import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapMapper;
-import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
-import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
-import org.apache.dolphinscheduler.dao.mapper.ResourceMapper;
-import org.apache.dolphinscheduler.dao.mapper.ResourceUserMapper;
-import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper;
-import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper;
-import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
-import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper;
-import org.apache.dolphinscheduler.dao.mapper.UserMapper;
-import org.apache.dolphinscheduler.server.master.config.MasterConfig;
-import org.apache.dolphinscheduler.server.master.dispatch.host.HostManager;
-import org.apache.dolphinscheduler.server.master.dispatch.host.RandomHostManager;
-import org.apache.dolphinscheduler.server.master.processor.queue.TaskResponseService;
-import org.apache.dolphinscheduler.server.worker.config.WorkerConfig;
-import org.apache.dolphinscheduler.server.worker.processor.TaskCallbackService;
-import org.apache.dolphinscheduler.service.process.ProcessService;
-import org.apache.dolphinscheduler.service.queue.TaskPriorityQueue;
-import org.apache.dolphinscheduler.service.queue.TaskPriorityQueueImpl;
-
-import org.mockito.Mockito;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-
-/**
- * dependency config
- */
-@Configuration
-public class DependencyConfig {
-
- @Bean
- public AlertDao alertDao() {
- return new AlertDao();
- }
-
- @Bean
- public AlertMapper alertMapper() {
- return Mockito.mock(AlertMapper.class);
- }
-
- @Bean
- public ProcessService processService() {
- return Mockito.mock(ProcessService.class);
- }
-
- @Bean
- public MasterConfig masterConfig() {
- return Mockito.mock(MasterConfig.class);
- }
-
- @Bean
- public WorkerConfig workerConfig() {
- return Mockito.mock(WorkerConfig.class);
- }
-
- @Bean
- public UserMapper userMapper() {
- return Mockito.mock(UserMapper.class);
- }
-
- @Bean
- public ProcessDefinitionMapper processDefineMapper() {
- return Mockito.mock(ProcessDefinitionMapper.class);
- }
-
- @Bean
- public ProcessInstanceMapper processInstanceMapper() {
- return Mockito.mock(ProcessInstanceMapper.class);
- }
-
- @Bean
- public DataSourceMapper dataSourceMapper() {
- return Mockito.mock(DataSourceMapper.class);
- }
-
- @Bean
- public ProcessInstanceMapMapper processInstanceMapMapper() {
- return Mockito.mock(ProcessInstanceMapMapper.class);
- }
-
- @Bean
- public TaskInstanceMapper taskInstanceMapper() {
- return Mockito.mock(TaskInstanceMapper.class);
- }
-
- @Bean
- public CommandMapper commandMapper() {
- return Mockito.mock(CommandMapper.class);
- }
-
- @Bean
- public ScheduleMapper scheduleMapper() {
- return Mockito.mock(ScheduleMapper.class);
- }
-
- @Bean
- public UdfFuncMapper udfFuncMapper() {
- return Mockito.mock(UdfFuncMapper.class);
- }
-
- @Bean
- public ResourceMapper resourceMapper() {
- return Mockito.mock(ResourceMapper.class);
- }
-
- @Bean
- public ResourceUserMapper resourceUserMapper() {
- return Mockito.mock(ResourceUserMapper.class);
- }
-
- @Bean
- public ErrorCommandMapper errorCommandMapper() {
- return Mockito.mock(ErrorCommandMapper.class);
- }
-
- @Bean
- public TenantMapper tenantMapper() {
- return Mockito.mock(TenantMapper.class);
- }
-
- @Bean
- public ProjectMapper projectMapper() {
- return Mockito.mock(ProjectMapper.class);
- }
-
- @Bean
- public TaskCallbackService taskCallbackService() {
- return Mockito.mock(TaskCallbackService.class);
- }
-
- @Bean
- public HostManager hostManager() {
- return new RandomHostManager();
- }
-
- @Bean
- public TaskResponseService taskResponseService() {
- return Mockito.mock(TaskResponseService.class);
- }
-
- @Bean
- public TaskPriorityQueue taskPriorityQueue() {
- return new TaskPriorityQueueImpl();
- }
-
- @Bean
- public AlertPluginInstanceMapper alertPluginInstanceMapper() {
- return Mockito.mock(AlertPluginInstanceMapper.class);
- }
-
- @Bean
- public AlertGroupMapper alertGroupMapper() {
- return Mockito.mock(AlertGroupMapper.class);
- }
-
- @Bean
- public PluginDefineMapper pluginDefineMapper() {
- return Mockito.mock(PluginDefineMapper.class);
- }
-
-}
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/registry/ZookeeperRegistryCenterTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/registry/ZookeeperRegistryCenterTest.java
deleted file mode 100644
index 1e73c7d76a..0000000000
--- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/registry/ZookeeperRegistryCenterTest.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.registry;
-
-import org.junit.Ignore;
-import org.junit.runner.RunWith;
-import org.mockito.junit.MockitoJUnitRunner;
-
-/**
- * zookeeper registry center test
- */
-@RunWith(MockitoJUnitRunner.class)
-@Ignore
-public class ZookeeperRegistryCenterTest {
-/*
- @InjectMocks
- private RegistryCenter registryCenter;
-
- @Mock
- protected RegisterOperator registerOperator;
-
- @Mock
- private ZookeeperConfig zookeeperConfig;
-
- private static final String DS_ROOT = "/dolphinscheduler";
-
- @Test
- public void testGetDeadZNodeParentPath() {
- ZookeeperConfig zookeeperConfig = new ZookeeperConfig();
- zookeeperConfig.setDsRoot(DS_ROOT);
- Mockito.when(registerOperator.getZookeeperConfig()).thenReturn(zookeeperConfig);
-
- String deadZNodeParentPath = registryCenter.getDeadZNodeParentPath();
-
- // Assert.assertEquals(deadZNodeParentPath, DS_ROOT + Constants.ZOOKEEPER_DOLPHINSCHEDULER_DEAD_SERVERS);
-
- }*/
-
-}
\ No newline at end of file
diff --git a/dolphinscheduler-service/pom.xml b/dolphinscheduler-service/pom.xml
index 8fe9d1f576..e3194b0a56 100644
--- a/dolphinscheduler-service/pom.xml
+++ b/dolphinscheduler-service/pom.xml
@@ -50,6 +50,12 @@
org.springframework.boot
spring-boot-starter-quartz
+
+
+ org.springframework.boot
+ spring-boot-starter
+
+
org.quartz-scheduler
@@ -65,7 +71,7 @@
com.zaxxer
- HikariCP-java6
+ HikariCP-java7
@@ -85,19 +91,4 @@
caffeine
-
-
-
-
- org.apache.maven.plugins
- maven-jar-plugin
-
-
- *.yaml
- *.properties
-
-
-
-
-
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/SchedulerConfiguration.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/SchedulerConfiguration.java
deleted file mode 100644
index d2c1ffd223..0000000000
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/SchedulerConfiguration.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-package org.apache.dolphinscheduler.service.quartz;
-
-import java.util.Properties;
-
-import javax.sql.DataSource;
-
-import org.quartz.impl.jdbcjobstore.PostgreSQLDelegate;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.core.io.ClassPathResource;
-import org.springframework.scheduling.quartz.SchedulerFactoryBean;
-import org.springframework.scheduling.quartz.SpringBeanJobFactory;
-
-@Configuration
-public class SchedulerConfiguration {
- @Value("${spring.datasource.driver-class-name}")
- private String dataSourceDriverClass;
-
- @Bean
- public SchedulerFactoryBean schedulerFactory(DataSource dataSource) {
- SchedulerFactoryBean schedulerFactory = new SchedulerFactoryBean();
- schedulerFactory.setConfigLocation(new ClassPathResource("quartz.properties"));
- if (dataSourceDriverClass.equals(org.postgresql.Driver.class.getName())) {
- final Properties overrideProperties = new Properties();
- overrideProperties.setProperty("org.quartz.jobStore.driverDelegateClass", PostgreSQLDelegate.class.getName());
- schedulerFactory.setQuartzProperties(overrideProperties);
- }
- schedulerFactory.setDataSource(dataSource);
- schedulerFactory.setJobFactory(new SpringBeanJobFactory());
- return schedulerFactory;
- }
-}
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/registry/RegistryClient.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/registry/RegistryClient.java
index f384678681..2a7e7a1514 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/registry/RegistryClient.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/registry/RegistryClient.java
@@ -37,12 +37,9 @@ import org.apache.dolphinscheduler.common.enums.NodeType;
import org.apache.dolphinscheduler.common.model.Server;
import org.apache.dolphinscheduler.common.utils.HeartBeat;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
-import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.registry.api.ConnectionListener;
import org.apache.dolphinscheduler.registry.api.Registry;
import org.apache.dolphinscheduler.registry.api.RegistryException;
-import org.apache.dolphinscheduler.registry.api.RegistryFactory;
-import org.apache.dolphinscheduler.registry.api.RegistryFactoryLoader;
import org.apache.dolphinscheduler.registry.api.SubscribeListener;
import java.io.IOException;
@@ -53,7 +50,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Collectors;
import javax.annotation.PostConstruct;
@@ -69,16 +65,16 @@ public class RegistryClient {
private static final Logger logger = LoggerFactory.getLogger(RegistryClient.class);
private static final String EMPTY = "";
- private static final String REGISTRY_PREFIX = "registry";
- private static final String REGISTRY_PLUGIN_NAME = "plugin.name";
- private static final String REGISTRY_CONFIG_FILE_PATH = "/registry.properties";
- private final AtomicBoolean isStarted = new AtomicBoolean(false);
- private Registry registry;
private IStoppable stoppable;
+ private final Registry registry;
+
+ public RegistryClient(Registry registry) {
+ this.registry = registry;
+ }
+
@PostConstruct
public void afterConstruct() {
- start();
initNodes();
}
@@ -204,9 +200,7 @@ public class RegistryClient {
}
public void close() throws IOException {
- if (isStarted.compareAndSet(true, false) && registry != null) {
- registry.close();
- }
+ registry.close();
}
public void persistEphemeral(String key, String value) {
@@ -274,24 +268,6 @@ public class RegistryClient {
}
}
- private void start() {
- if (isStarted.compareAndSet(false, true)) {
- PropertyUtils.loadPropertyFile(REGISTRY_CONFIG_FILE_PATH);
- final Map registryConfig = PropertyUtils.getPropertiesByPrefix(REGISTRY_PREFIX);
-
- if (null == registryConfig || registryConfig.isEmpty()) {
- throw new RegistryException("registry config param is null");
- }
- final String pluginName = registryConfig.get(REGISTRY_PLUGIN_NAME);
- final Map factories = RegistryFactoryLoader.load();
- if (!factories.containsKey(pluginName)) {
- throw new RegistryException("No such registry plugin: " + pluginName);
- }
- registry = factories.get(pluginName).create();
- registry.start(registryConfig);
- }
- }
-
private void initNodes() {
registry.put(REGISTRY_DOLPHINSCHEDULER_MASTERS, EMPTY, false);
registry.put(REGISTRY_DOLPHINSCHEDULER_WORKERS, EMPTY, false);
diff --git a/dolphinscheduler-service/src/main/resources/quartz.properties b/dolphinscheduler-service/src/main/resources/quartz.properties
deleted file mode 100644
index 52d535dde3..0000000000
--- a/dolphinscheduler-service/src/main/resources/quartz.properties
+++ /dev/null
@@ -1,32 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-org.quartz.threadPool.threadPriority=5
-org.quartz.jobStore.isClustered=true
-org.quartz.jobStore.class=org.quartz.impl.jdbcjobstore.JobStoreTX
-org.quartz.scheduler.instanceId=AUTO
-org.quartz.jobStore.tablePrefix=QRTZ_
-org.quartz.jobStore.acquireTriggersWithinLock=true
-org.quartz.scheduler.instanceName=DolphinScheduler
-org.quartz.threadPool.class=org.quartz.simpl.SimpleThreadPool
-org.quartz.jobStore.useProperties=false
-org.quartz.threadPool.makeThreadsDaemons=true
-org.quartz.threadPool.threadCount=25
-org.quartz.jobStore.misfireThreshold=60000
-org.quartz.scheduler.makeSchedulerThreadDaemon=true
-org.quartz.jobStore.driverDelegateClass=org.quartz.impl.jdbcjobstore.StdJDBCDelegate
-org.quartz.jobStore.clusterCheckinInterval=5000
diff --git a/dolphinscheduler-service/src/main/resources/registry.properties b/dolphinscheduler-service/src/main/resources/registry.properties
deleted file mode 100644
index f534a65ed8..0000000000
--- a/dolphinscheduler-service/src/main/resources/registry.properties
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-registry.plugin.name=zookeeper
-registry.servers=127.0.0.1:2181
diff --git a/dolphinscheduler-spi/pom.xml b/dolphinscheduler-spi/pom.xml
index 36b6b25fe5..fb562f2c39 100644
--- a/dolphinscheduler-spi/pom.xml
+++ b/dolphinscheduler-spi/pom.xml
@@ -53,12 +53,6 @@
com.google.guava
guava
-
-
- com.google.code.findbugs
- jsr305
-
-
diff --git a/dolphinscheduler-standalone-server/pom.xml b/dolphinscheduler-standalone-server/pom.xml
index c27e9cefa7..6f854d5fc9 100644
--- a/dolphinscheduler-standalone-server/pom.xml
+++ b/dolphinscheduler-standalone-server/pom.xml
@@ -30,7 +30,11 @@
org.apache.dolphinscheduler
- dolphinscheduler-server
+ dolphinscheduler-master
+
+
+ org.apache.dolphinscheduler
+ dolphinscheduler-worker
org.apache.dolphinscheduler
@@ -38,7 +42,7 @@
org.apache.dolphinscheduler
- dolphinscheduler-meter
+ dolphinscheduler-alert-server
@@ -52,10 +56,6 @@
-
- org.apache.dolphinscheduler
- dolphinscheduler-alert-server
-
org.apache.dolphinscheduler
dolphinscheduler-python
@@ -70,9 +70,43 @@
*.yaml
+ *.xml
+
+ maven-assembly-plugin
+
+
+ dolphinscheduler-standalone-server
+ package
+
+ single
+
+
+ standalone-server
+
+ src/main/assembly/dolphinscheduler-standalone-server.xml
+
+ false
+
+
+
+
+
+
+
+ docker
+
+
+
+ org.codehaus.mojo
+ exec-maven-plugin
+
+
+
+
+
diff --git a/dolphinscheduler-standalone-server/src/main/assembly/dolphinscheduler-standalone-server.xml b/dolphinscheduler-standalone-server/src/main/assembly/dolphinscheduler-standalone-server.xml
new file mode 100644
index 0000000000..8d767eec0c
--- /dev/null
+++ b/dolphinscheduler-standalone-server/src/main/assembly/dolphinscheduler-standalone-server.xml
@@ -0,0 +1,107 @@
+
+
+
+ dolphinscheduler-standalone-server
+
+ dir
+
+ false
+ standalone-server
+
+
+ ${basedir}/../dolphinscheduler-alert/dolphinscheduler-alert-server/target/alert-server
+ .
+
+ libs
+
+
+
+ ${basedir}/../dolphinscheduler-api/target/api-server
+
+ libs
+
+ .
+
+
+ ${basedir}/../dolphinscheduler-master/target/master-server
+
+ libs
+
+ .
+
+
+ ${basedir}/../dolphinscheduler-worker/target/worker-server
+
+ libs
+
+ .
+
+
+ ${basedir}/../dolphinscheduler-log-server/target/logger-server
+
+ libs
+
+ .
+
+
+
+ ${basedir}/src/main/resources
+
+ *.yaml
+ *.xml
+
+ conf
+
+
+
+ ${basedir}/src/main/bin
+ bin
+ 0755
+ 0755
+
+
+ ${basedir}/../script/env
+ bin
+
+ dolphinscheduler_env.sh
+
+ 0755
+ 0755
+
+
+
+ ${basedir}/../dolphinscheduler-dao/src/main/resources
+
+ sql/**/*
+
+ conf
+
+
+ ${basedir}/../dolphinscheduler-ui/dist
+ ./ui
+
+
+
+
+
+ libs
+
+
+
diff --git a/dolphinscheduler-standalone-server/src/main/bin/start.sh b/dolphinscheduler-standalone-server/src/main/bin/start.sh
new file mode 100644
index 0000000000..d8c76115ce
--- /dev/null
+++ b/dolphinscheduler-standalone-server/src/main/bin/start.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+BIN_DIR=$(dirname $0)
+DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/..; pwd)}
+
+source "$BIN_DIR/dolphinscheduler_env.sh"
+
+JAVA_OPTS=${JAVA_OPTS:-"-server -Xms1g -Xmx1g -Xmn512m -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"}
+
+if [[ "$DOCKER" == "true" ]]; then
+ JAVA_OPTS="${JAVA_OPTS} -XX:-UseContainerSupport"
+fi
+
+java $JAVA_OPTS \
+ -cp "$DOLPHINSCHEDULER_HOME/conf":"$DOLPHINSCHEDULER_HOME/libs/*" \
+ org.apache.dolphinscheduler.StandaloneServer
diff --git a/dolphinscheduler-standalone-server/src/main/docker/Dockerfile b/dolphinscheduler-standalone-server/src/main/docker/Dockerfile
new file mode 100644
index 0000000000..0a59d255db
--- /dev/null
+++ b/dolphinscheduler-standalone-server/src/main/docker/Dockerfile
@@ -0,0 +1,34 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+FROM openjdk:8-jre-slim-buster
+
+ENV DOCKER true
+ENV TZ Asia/Shanghai
+ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler
+
+RUN apt update ; \
+ apt install -y curl ; \
+ rm -rf /var/lib/apt/lists/*
+
+WORKDIR $DOLPHINSCHEDULER_HOME
+
+ADD ./target/standalone-server $DOLPHINSCHEDULER_HOME
+
+EXPOSE 12345
+
+CMD [ "/bin/bash", "./bin/start.sh" ]
diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryFactory.java b/dolphinscheduler-standalone-server/src/main/java/org/apache/dolphinscheduler/StandaloneServer.java
similarity index 60%
rename from dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryFactory.java
rename to dolphinscheduler-standalone-server/src/main/java/org/apache/dolphinscheduler/StandaloneServer.java
index 949df21bee..13e1ea006e 100644
--- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryFactory.java
+++ b/dolphinscheduler-standalone-server/src/main/java/org/apache/dolphinscheduler/StandaloneServer.java
@@ -15,23 +15,19 @@
* limitations under the License.
*/
-package org.apache.dolphinscheduler.plugin.registry.zookeeper;
+package org.apache.dolphinscheduler;
-import org.apache.dolphinscheduler.registry.api.Registry;
-import org.apache.dolphinscheduler.registry.api.RegistryFactory;
+import org.apache.curator.test.TestingServer;
-import com.google.auto.service.AutoService;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
-@AutoService(RegistryFactory.class)
-public final class ZookeeperRegistryFactory implements RegistryFactory {
+@SpringBootApplication
+public class StandaloneServer {
+ public static void main(String[] args) throws Exception {
+ final TestingServer server = new TestingServer(true);
+ System.setProperty("registry.zookeeper.connect-string", server.getConnectString());
- @Override
- public String name() {
- return "zookeeper";
- }
-
- @Override
- public Registry create() {
- return new ZookeeperRegistry();
+ SpringApplication.run(StandaloneServer.class, args);
}
}
diff --git a/dolphinscheduler-standalone-server/src/main/java/org/apache/dolphinscheduler/server/StandaloneServer.java b/dolphinscheduler-standalone-server/src/main/java/org/apache/dolphinscheduler/server/StandaloneServer.java
deleted file mode 100644
index 830cd42edf..0000000000
--- a/dolphinscheduler-standalone-server/src/main/java/org/apache/dolphinscheduler/server/StandaloneServer.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server;
-
-import org.apache.dolphinscheduler.alert.AlertServer;
-import org.apache.dolphinscheduler.api.ApiApplicationServer;
-import org.apache.dolphinscheduler.server.master.MasterServer;
-import org.apache.dolphinscheduler.server.worker.WorkerServer;
-
-import org.apache.curator.test.TestingServer;
-
-import org.springframework.boot.builder.SpringApplicationBuilder;
-
-public class StandaloneServer {
- public static void main(String[] args) throws Exception {
- final TestingServer server = new TestingServer(true);
- System.setProperty("registry.servers", server.getConnectString());
-
- new SpringApplicationBuilder(
- ApiApplicationServer.class,
- MasterServer.class,
- WorkerServer.class,
- AlertServer.class,
- PythonGatewayServer.class
- ).profiles("master", "worker", "api", "alert", "h2", "standalone").run(args);
- }
-}
diff --git a/dolphinscheduler-standalone-server/src/main/resources/application.yaml b/dolphinscheduler-standalone-server/src/main/resources/application.yaml
new file mode 100644
index 0000000000..4aacdbab10
--- /dev/null
+++ b/dolphinscheduler-standalone-server/src/main/resources/application.yaml
@@ -0,0 +1,171 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+spring:
+ application:
+ name: standalone-server
+ main:
+ banner-mode: off
+ cache:
+ # default enable cache, you can disable by `type: none`
+ type: none
+ cache-names:
+ - tenant
+ - user
+ - processDefinition
+ - processTaskRelation
+ - taskDefinition
+ caffeine:
+ spec: maximumSize=100,expireAfterWrite=300s,recordStats
+ datasource:
+ driver-class-name: org.h2.Driver
+ url: jdbc:h2:mem:dolphinscheduler;MODE=MySQL;DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=true;INIT=runscript from 'classpath:sql/dolphinscheduler_h2.sql'
+ username: sa
+ password: ""
+ hikari:
+ connection-test-query: select 1
+ minimum-idle: 5
+ auto-commit: true
+ validation-timeout: 3000
+ pool-name: DolphinScheduler
+ maximum-pool-size: 50
+ connection-timeout: 30000
+ idle-timeout: 600000
+ leak-detection-threshold: 0
+ initialization-fail-timeout: 1
+ quartz:
+ job-store-type: jdbc
+ jdbc:
+ initialize-schema: never
+ properties:
+ org.quartz.threadPool:threadPriority: 5
+ org.quartz.jobStore.isClustered: true
+ org.quartz.jobStore.class: org.quartz.impl.jdbcjobstore.JobStoreTX
+ org.quartz.scheduler.instanceId: AUTO
+ org.quartz.jobStore.tablePrefix: QRTZ_
+ org.quartz.jobStore.acquireTriggersWithinLock: true
+ org.quartz.scheduler.instanceName: DolphinScheduler
+ org.quartz.threadPool.class: org.quartz.simpl.SimpleThreadPool
+ org.quartz.jobStore.useProperties: false
+ org.quartz.threadPool.makeThreadsDaemons: true
+ org.quartz.threadPool.threadCount: 25
+ org.quartz.jobStore.misfireThreshold: 60000
+ org.quartz.scheduler.makeSchedulerThreadDaemon: true
+ org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.StdJDBCDelegate
+ org.quartz.jobStore.clusterCheckinInterval: 5000
+ jackson:
+ time-zone: GMT+8
+ servlet:
+ multipart:
+ max-file-size: 1024MB
+ max-request-size: 1024MB
+ messages:
+ basename: i18n/messages
+
+registry:
+ type: zookeeper
+ zookeeper:
+ namespace: dolphinscheduler
+ connect-string: localhost:2181
+ retry-policy:
+ base-sleep-time: 60ms
+ max-sleep: 300ms
+ max-retries: 5
+ session-timeout: 30s
+ connection-timeout: 9s
+ block-until-connected: 600ms
+ digest: ~
+
+master:
+ listen-port: 5678
+ # master fetch command num
+ fetch-command-num: 10
+ # master prepare execute thread number to limit handle commands in parallel
+ pre-exec-threads: 10
+ # master execute thread number to limit process instances in parallel
+ exec-threads: 100
+ # master dispatch task number per batch
+ dispatch-task-number: 3
+ # master host selector to select a suitable worker, default value: LowerWeight. Optional values include random, round_robin, lower_weight
+ host-selector: lower_weight
+ # master heartbeat interval, the unit is second
+ heartbeat-interval: 10
+ # master commit task retry times
+ task-commit-retry-times: 5
+ # master commit task interval, the unit is millisecond
+ task-commit-interval: 1000
+ state-wheel-interval: 5
+ # master max cpuload avg, only higher than the system cpu load average, master server can schedule. default value -1: the number of cpu cores * 2
+ max-cpu-load-avg: -1
+ # master reserved memory, only lower than system available memory, master server can schedule. default value 0.3, the unit is G
+ reserved-memory: 0.3
+
+worker:
+ # worker listener port
+ listen-port: 1234
+ # worker execute thread number to limit task instances in parallel
+ exec-threads: 100
+ # worker heartbeat interval, the unit is second
+ heartbeat-interval: 10
+ # worker host weight to dispatch tasks, default value 100
+ host-weight: 100
+ # worker tenant auto create
+ tenant-auto-create: true
+ # worker max cpuload avg, only higher than the system cpu load average, worker server can be dispatched tasks. default value -1: the number of cpu cores * 2
+ max-cpu-load-avg: -1
+ # worker reserved memory, only lower than system available memory, worker server can be dispatched tasks. default value 0.3, the unit is G
+ reserved-memory: 0.3
+ # default worker groups separated by comma, like 'worker.groups=default,test'
+ groups:
+ - default
+ # alert server listen host
+ alert-listen-host: localhost
+ alert-listen-port: 50052
+
+alert:
+ port: 50052
+
+server:
+ port: 12345
+ servlet:
+ session:
+ timeout: 120m
+ context-path: /dolphinscheduler/
+ compression:
+ enabled: true
+ mime-types: text/html,text/xml,text/plain,text/css,text/javascript,application/javascript,application/json,application/xml
+ jetty:
+ max-http-form-post-size: 5000000
+
+management:
+ endpoints:
+ web:
+ exposure:
+ include: '*'
+ metrics:
+ tags:
+ application: ${spring.application.name}
+
+# Override by profile
+---
+spring:
+ config:
+ activate:
+ on-profile: postgresql
+ quartz:
+ properties:
+ org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.PostgreSQLDelegate
diff --git a/docker/build/conf/dolphinscheduler/logback/logback-alert.xml b/dolphinscheduler-standalone-server/src/main/resources/logback-spring.xml
similarity index 56%
rename from docker/build/conf/dolphinscheduler/logback/logback-alert.xml
rename to dolphinscheduler-standalone-server/src/main/resources/logback-spring.xml
index 1718947dd1..75694da73a 100644
--- a/docker/build/conf/dolphinscheduler/logback/logback-alert.xml
+++ b/dolphinscheduler-standalone-server/src/main/resources/logback-spring.xml
@@ -16,10 +16,7 @@
~ limitations under the License.
-->
-
-
-
-
+
@@ -29,24 +26,7 @@
-
- ${log.base}/dolphinscheduler-alert.log
-
- ${log.base}/dolphinscheduler-alert.%d{yyyy-MM-dd_HH}.%i.log
- 20
- 64MB
-
-
-
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
-
- UTF-8
-
-
-
-
-
-
\ No newline at end of file
+
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/pom.xml
index 1c169977fe..97af883bf3 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/pom.xml
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/pom.xml
@@ -98,10 +98,6 @@
com.google.code.gson
gson
-
- com.google.code.findbugs
- jsr305
-
io.dropwizard.metrics
metrics-core
@@ -114,10 +110,6 @@
org.apache.avro
avro
-
- org.apache.commons
- commons-compress
-
org.apache.curator
curator-client
diff --git a/dolphinscheduler-tools/pom.xml b/dolphinscheduler-tools/pom.xml
new file mode 100644
index 0000000000..6e4a65c84b
--- /dev/null
+++ b/dolphinscheduler-tools/pom.xml
@@ -0,0 +1,86 @@
+
+
+
+
+ dolphinscheduler
+ org.apache.dolphinscheduler
+ 2.0.0-SNAPSHOT
+
+ 4.0.0
+
+ dolphinscheduler-tools
+
+
+
+ org.apache.dolphinscheduler
+ dolphinscheduler-dao
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+
+
+ *.yaml
+
+
+
+
+ maven-assembly-plugin
+
+
+ dolphinscheduler-tools
+ package
+
+ single
+
+
+ tools
+
+ src/main/assembly/dolphinscheduler-tools.xml
+
+ false
+
+
+
+
+
+
+
+
+
+ docker
+
+
+
+ org.codehaus.mojo
+ exec-maven-plugin
+
+
+
+
+
+
diff --git a/dolphinscheduler-tools/src/main/assembly/dolphinscheduler-tools.xml b/dolphinscheduler-tools/src/main/assembly/dolphinscheduler-tools.xml
new file mode 100644
index 0000000000..bb822ba9b0
--- /dev/null
+++ b/dolphinscheduler-tools/src/main/assembly/dolphinscheduler-tools.xml
@@ -0,0 +1,70 @@
+
+
+
+ dolphinscheduler-tools
+
+ dir
+
+ false
+ tools
+
+
+ ${basedir}/src/main/resources
+
+ *.yaml
+
+ conf
+
+
+ ${basedir}/src/main/bin
+ bin
+ 0755
+ 0755
+
+
+ ${basedir}/../script/env
+ bin
+
+ dolphinscheduler_env.sh
+
+ 0755
+ 0755
+
+
+ ${basedir}/../dolphinscheduler-dao/src/main/resources
+
+ sql/**/*
+
+ conf
+
+
+ ${basedir}/../dolphinscheduler-common/src/main/resources
+
+ **/*.properties
+
+ conf
+
+
+
+
+ libs
+
+
+
diff --git a/docker/build/hooks/push b/dolphinscheduler-tools/src/main/bin/create-schema.sh
similarity index 73%
rename from docker/build/hooks/push
rename to dolphinscheduler-tools/src/main/bin/create-schema.sh
index 41a25c54fe..efb12c9cdd 100755
--- a/docker/build/hooks/push
+++ b/dolphinscheduler-tools/src/main/bin/create-schema.sh
@@ -16,9 +16,11 @@
# limitations under the License.
#
-echo "------ push start -------"
-printenv
+BIN_DIR=$(dirname $0)
+DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/..; pwd)}
-docker push $DOCKER_REPO:${VERSION}
+source "$BIN_DIR/dolphinscheduler_env.sh"
-echo "------ push end -------"
+java $JAVA_OPTS \
+ -cp "$DOLPHINSCHEDULER_HOME/conf":"$DOLPHINSCHEDULER_HOME/libs/*" \
+ org.apache.dolphinscheduler.tools.datasource.CreateDolphinScheduler
diff --git a/docker/build/checkpoint.sh b/dolphinscheduler-tools/src/main/bin/upgrade-schema.sh
similarity index 73%
rename from docker/build/checkpoint.sh
rename to dolphinscheduler-tools/src/main/bin/upgrade-schema.sh
index cd2774f9ce..691d71dc18 100755
--- a/docker/build/checkpoint.sh
+++ b/dolphinscheduler-tools/src/main/bin/upgrade-schema.sh
@@ -16,12 +16,11 @@
# limitations under the License.
#
-set -e
+BIN_DIR=$(dirname $0)
+DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/..; pwd)}
-if [ "$(ps -ef | grep java | grep -c $1)" -eq 0 ]; then
- echo "[ERROR] $1 process not exits."
- exit 1
-else
- echo "[INFO] $1 process exits."
- exit 0
-fi
+source "$BIN_DIR/dolphinscheduler_env.sh"
+
+java $JAVA_OPTS \
+ -cp "$DOLPHINSCHEDULER_HOME/conf":"$DOLPHINSCHEDULER_HOME/libs/*" \
+ org.apache.dolphinscheduler.tools.datasource.UpgradeDolphinScheduler
diff --git a/dolphinscheduler-tools/src/main/docker/Dockerfile b/dolphinscheduler-tools/src/main/docker/Dockerfile
new file mode 100644
index 0000000000..5c2fb45e45
--- /dev/null
+++ b/dolphinscheduler-tools/src/main/docker/Dockerfile
@@ -0,0 +1,32 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+FROM openjdk:8-jre-slim-buster
+
+ENV DOCKER true
+ENV TZ Asia/Shanghai
+ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler
+
+RUN apt update ; \
+ apt install -y curl ; \
+ rm -rf /var/lib/apt/lists/*
+
+WORKDIR $DOLPHINSCHEDULER_HOME
+
+ADD ./target/tools $DOLPHINSCHEDULER_HOME
+
+ENTRYPOINT [ "/bin/bash" ]
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/shell/CreateDolphinScheduler.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/CreateDolphinScheduler.java
similarity index 68%
rename from dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/shell/CreateDolphinScheduler.java
rename to dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/CreateDolphinScheduler.java
index d1ac2e231a..d204f7a32e 100644
--- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/shell/CreateDolphinScheduler.java
+++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/CreateDolphinScheduler.java
@@ -15,33 +15,22 @@
* limitations under the License.
*/
-package org.apache.dolphinscheduler.dao.upgrade.shell;
-
-import org.apache.dolphinscheduler.dao.upgrade.DolphinSchedulerManager;
+package org.apache.dolphinscheduler.tools.datasource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.CommandLineRunner;
-import org.springframework.boot.WebApplicationType;
-import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
-import org.springframework.boot.autoconfigure.quartz.QuartzAutoConfiguration;
-import org.springframework.boot.builder.SpringApplicationBuilder;
-import org.springframework.context.annotation.ComponentScan;
-import org.springframework.context.annotation.Profile;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.stereotype.Component;
-@ComponentScan(value = "org.apache.dolphinscheduler.dao")
-@EnableAutoConfiguration(exclude = {QuartzAutoConfiguration.class})
+@SpringBootApplication
public class CreateDolphinScheduler {
public static void main(String[] args) {
- new SpringApplicationBuilder(CreateDolphinScheduler.class)
- .profiles("shell-create", "shell-cli")
- .web(WebApplicationType.NONE)
- .run(args);
+ SpringApplication.run(CreateDolphinScheduler.class, args);
}
@Component
- @Profile("shell-create")
static class CreateRunner implements CommandLineRunner {
private static final Logger logger = LoggerFactory.getLogger(CreateRunner.class);
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/DolphinSchedulerManager.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/DolphinSchedulerManager.java
similarity index 96%
rename from dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/DolphinSchedulerManager.java
rename to dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/DolphinSchedulerManager.java
index 5c3af46cee..2788da8d4e 100644
--- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/DolphinSchedulerManager.java
+++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/DolphinSchedulerManager.java
@@ -15,9 +15,11 @@
* limitations under the License.
*/
-package org.apache.dolphinscheduler.dao.upgrade;
+package org.apache.dolphinscheduler.tools.datasource;
+import org.apache.dolphinscheduler.dao.upgrade.SchemaUtils;
import org.apache.dolphinscheduler.spi.enums.DbType;
+import org.apache.dolphinscheduler.tools.datasource.dao.UpgradeDao;
import java.io.IOException;
import java.sql.Connection;
@@ -27,11 +29,9 @@ import javax.sql.DataSource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Service;
@Service
-@Profile("shell-cli")
public class DolphinSchedulerManager {
private static final Logger logger = LoggerFactory.getLogger(DolphinSchedulerManager.class);
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/shell/InitDolphinScheduler.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/InitDolphinScheduler.java
similarity index 63%
rename from dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/shell/InitDolphinScheduler.java
rename to dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/InitDolphinScheduler.java
index 261b98895d..18eb310654 100644
--- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/shell/InitDolphinScheduler.java
+++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/InitDolphinScheduler.java
@@ -14,33 +14,23 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.dolphinscheduler.dao.upgrade.shell;
-import org.apache.dolphinscheduler.dao.upgrade.DolphinSchedulerManager;
+package org.apache.dolphinscheduler.tools.datasource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.CommandLineRunner;
-import org.springframework.boot.WebApplicationType;
-import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
-import org.springframework.boot.autoconfigure.quartz.QuartzAutoConfiguration;
-import org.springframework.boot.builder.SpringApplicationBuilder;
-import org.springframework.context.annotation.ComponentScan;
-import org.springframework.context.annotation.Profile;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.stereotype.Component;
-@ComponentScan(value = "org.apache.dolphinscheduler.dao")
-@EnableAutoConfiguration(exclude = {QuartzAutoConfiguration.class})
+@SpringBootApplication
public class InitDolphinScheduler {
public static void main(String[] args) {
- new SpringApplicationBuilder(InitDolphinScheduler.class)
- .profiles("shell-init", "shell-cli")
- .web(WebApplicationType.NONE)
- .run(args);
+ SpringApplication.run(InitDolphinScheduler.class, args);
}
@Component
- @Profile("shell-init")
static class InitRunner implements CommandLineRunner {
private static final Logger logger = LoggerFactory.getLogger(InitRunner.class);
@@ -51,7 +41,7 @@ public class InitDolphinScheduler {
}
@Override
- public void run(String... args) throws Exception {
+ public void run(String... args) {
dolphinSchedulerManager.initDolphinScheduler();
logger.info("init DolphinScheduler finished");
}
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/shell/UpgradeDolphinScheduler.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/UpgradeDolphinScheduler.java
similarity index 66%
rename from dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/shell/UpgradeDolphinScheduler.java
rename to dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/UpgradeDolphinScheduler.java
index 427f2f5973..d09acc6692 100644
--- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/shell/UpgradeDolphinScheduler.java
+++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/UpgradeDolphinScheduler.java
@@ -14,33 +14,23 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.dolphinscheduler.dao.upgrade.shell;
-import org.apache.dolphinscheduler.dao.upgrade.DolphinSchedulerManager;
+package org.apache.dolphinscheduler.tools.datasource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.CommandLineRunner;
-import org.springframework.boot.WebApplicationType;
-import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
-import org.springframework.boot.autoconfigure.quartz.QuartzAutoConfiguration;
-import org.springframework.boot.builder.SpringApplicationBuilder;
-import org.springframework.context.annotation.ComponentScan;
-import org.springframework.context.annotation.Profile;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.stereotype.Component;
-@ComponentScan(value = "org.apache.dolphinscheduler.dao")
-@EnableAutoConfiguration(exclude = {QuartzAutoConfiguration.class})
+@SpringBootApplication
public class UpgradeDolphinScheduler {
public static void main(String[] args) {
- new SpringApplicationBuilder(UpgradeDolphinScheduler.class)
- .profiles("shell-upgrade", "shell-cli")
- .web(WebApplicationType.NONE)
- .run(args);
+ SpringApplication.run(UpgradeDolphinScheduler.class, args);
}
@Component
- @Profile("shell-upgrade")
static class UpgradeRunner implements CommandLineRunner {
private static final Logger logger = LoggerFactory.getLogger(UpgradeRunner.class);
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/MysqlUpgradeDao.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/MysqlUpgradeDao.java
similarity index 94%
rename from dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/MysqlUpgradeDao.java
rename to dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/MysqlUpgradeDao.java
index b83809d2ac..1c8358f4fe 100644
--- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/MysqlUpgradeDao.java
+++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/MysqlUpgradeDao.java
@@ -14,7 +14,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.dolphinscheduler.dao.upgrade;
+
+package org.apache.dolphinscheduler.tools.datasource.dao;
import org.apache.dolphinscheduler.common.utils.ConnectionUtils;
import org.apache.dolphinscheduler.spi.enums.DbType;
@@ -27,11 +28,9 @@ import javax.sql.DataSource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Service;
@Service
-@Profile("shell-cli")
public class MysqlUpgradeDao extends UpgradeDao {
public static final Logger logger = LoggerFactory.getLogger(MysqlUpgradeDao.class);
@@ -45,7 +44,7 @@ public class MysqlUpgradeDao extends UpgradeDao {
}
@Override
- protected DbType getDbType() {
+ public DbType getDbType() {
return DbType.MYSQL;
}
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/PostgresqlUpgradeDao.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/PostgresqlUpgradeDao.java
similarity index 95%
rename from dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/PostgresqlUpgradeDao.java
rename to dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/PostgresqlUpgradeDao.java
index 97da39a894..c406a37100 100644
--- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/PostgresqlUpgradeDao.java
+++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/PostgresqlUpgradeDao.java
@@ -14,7 +14,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.dolphinscheduler.dao.upgrade;
+
+package org.apache.dolphinscheduler.tools.datasource.dao;
import org.apache.dolphinscheduler.common.utils.ConnectionUtils;
import org.apache.dolphinscheduler.spi.enums.DbType;
@@ -28,11 +29,9 @@ import javax.sql.DataSource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Service;
@Service
-@Profile("shell-cli")
public class PostgresqlUpgradeDao extends UpgradeDao {
public static final Logger logger = LoggerFactory.getLogger(PostgresqlUpgradeDao.class);
@@ -46,7 +45,7 @@ public class PostgresqlUpgradeDao extends UpgradeDao {
}
@Override
- protected DbType getDbType() {
+ public DbType getDbType() {
return DbType.POSTGRESQL;
}
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ResourceDao.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ResourceDao.java
similarity index 94%
rename from dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ResourceDao.java
rename to dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ResourceDao.java
index c9ff149306..4cd45a29e6 100644
--- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ResourceDao.java
+++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ResourceDao.java
@@ -15,12 +15,10 @@
* limitations under the License.
*/
-package org.apache.dolphinscheduler.dao.upgrade;
+package org.apache.dolphinscheduler.tools.datasource.dao;
import org.apache.dolphinscheduler.common.utils.ConnectionUtils;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.dolphinscheduler.dao.upgrade.ProcessDefinitionDao;
import java.sql.Connection;
import java.sql.PreparedStatement;
@@ -28,6 +26,9 @@ import java.sql.ResultSet;
import java.util.HashMap;
import java.util.Map;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
/**
* resource dao
*/
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/UpgradeDao.java
similarity index 98%
rename from dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java
rename to dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/UpgradeDao.java
index d2ce1049e6..b5d2106127 100644
--- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java
+++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/UpgradeDao.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.dolphinscheduler.dao.upgrade;
+package org.apache.dolphinscheduler.tools.datasource.dao;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ConditionType;
@@ -33,6 +33,12 @@ import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog;
import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelationLog;
import org.apache.dolphinscheduler.dao.entity.TaskDefinitionLog;
+import org.apache.dolphinscheduler.dao.upgrade.JsonSplitDao;
+import org.apache.dolphinscheduler.dao.upgrade.ProcessDefinitionDao;
+import org.apache.dolphinscheduler.dao.upgrade.ProjectDao;
+import org.apache.dolphinscheduler.dao.upgrade.ScheduleDao;
+import org.apache.dolphinscheduler.dao.upgrade.SchemaUtils;
+import org.apache.dolphinscheduler.dao.upgrade.WorkerGroupDao;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.apache.commons.collections.CollectionUtils;
@@ -80,7 +86,7 @@ public abstract class UpgradeDao {
protected abstract String initSqlPath();
- protected abstract DbType getDbType();
+ public abstract DbType getDbType();
public void initSchema() {
// Execute the dolphinscheduler DDL, it cannot be rolled back
diff --git a/dolphinscheduler-tools/src/main/resources/application.yaml b/dolphinscheduler-tools/src/main/resources/application.yaml
new file mode 100644
index 0000000000..2c26aed0f3
--- /dev/null
+++ b/dolphinscheduler-tools/src/main/resources/application.yaml
@@ -0,0 +1,46 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+spring:
+ main:
+ banner-mode: off
+ datasource:
+ driver-class-name: org.postgresql.Driver
+ url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler
+ username: root
+ password: root
+ hikari:
+ connection-test-query: select 1
+ minimum-idle: 5
+ auto-commit: true
+ validation-timeout: 3000
+ pool-name: DolphinScheduler
+ maximum-pool-size: 50
+ connection-timeout: 30000
+ idle-timeout: 600000
+ leak-detection-threshold: 0
+ initialization-fail-timeout: 1
+
+# Override by profile
+
+---
+spring:
+ config:
+ activate:
+ on-profile: mysql
+ datasource:
+ driver-class-name: com.mysql.jdbc.Driver
+ url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8
diff --git a/dolphinscheduler-ui/pom.xml b/dolphinscheduler-ui/pom.xml
index 8ffb2d4c68..84f6b9aca3 100644
--- a/dolphinscheduler-ui/pom.xml
+++ b/dolphinscheduler-ui/pom.xml
@@ -33,6 +33,7 @@
6.14.11
src
+
release
@@ -145,5 +146,4 @@
-
diff --git a/dolphinscheduler-worker/pom.xml b/dolphinscheduler-worker/pom.xml
new file mode 100644
index 0000000000..7cce8b257f
--- /dev/null
+++ b/dolphinscheduler-worker/pom.xml
@@ -0,0 +1,178 @@
+
+
+
+
+ dolphinscheduler
+ org.apache.dolphinscheduler
+ 2.0.0-SNAPSHOT
+
+ 4.0.0
+
+ dolphinscheduler-worker
+
+
+
+ org.apache.dolphinscheduler
+ dolphinscheduler-service
+
+
+ org.apache.dolphinscheduler
+ dolphinscheduler-spi
+
+
+ org.apache.dolphinscheduler
+ dolphinscheduler-common
+
+
+ org.apache.dolphinscheduler
+ dolphinscheduler-server
+
+
+ org.apache.dolphinscheduler
+ dolphinscheduler-meter
+
+
+ org.apache.dolphinscheduler
+ dolphinscheduler-task-datax
+
+
+ org.apache.dolphinscheduler
+ dolphinscheduler-task-flink
+
+
+ org.apache.dolphinscheduler
+ dolphinscheduler-task-http
+
+
+ org.apache.dolphinscheduler
+ dolphinscheduler-task-mr
+
+
+ org.apache.dolphinscheduler
+ dolphinscheduler-task-pigeon
+
+
+ org.apache.dolphinscheduler
+ dolphinscheduler-task-procedure
+
+
+ org.apache.dolphinscheduler
+ dolphinscheduler-task-python
+
+
+ org.apache.dolphinscheduler
+ dolphinscheduler-task-shell
+
+
+ org.apache.dolphinscheduler
+ dolphinscheduler-task-spark
+
+
+ org.apache.dolphinscheduler
+ dolphinscheduler-task-sql
+
+
+ org.apache.dolphinscheduler
+ dolphinscheduler-task-sqoop
+
+
+
+ org.springframework.boot
+ spring-boot-starter
+
+
+ org.springframework.boot
+ spring-boot-starter-tomcat
+
+
+ log4j-to-slf4j
+ org.apache.logging.log4j
+
+
+
+
+
+ org.codehaus.janino
+ janino
+
+
+
+ org.mockito
+ mockito-core
+ test
+
+
+ org.springframework.boot
+ spring-boot-starter-test
+ test
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+
+
+ *.yaml
+ *.xml
+
+
+
+
+ maven-assembly-plugin
+
+
+ dolphinscheduler-worker-server
+ package
+
+ single
+
+
+ worker-server
+
+ src/main/assembly/dolphinscheduler-worker-server.xml
+
+ false
+
+
+
+
+
+
+
+
+
+ docker
+
+
+
+ org.codehaus.mojo
+ exec-maven-plugin
+
+
+
+
+
+
diff --git a/dolphinscheduler-worker/src/main/assembly/dolphinscheduler-worker-server.xml b/dolphinscheduler-worker/src/main/assembly/dolphinscheduler-worker-server.xml
new file mode 100644
index 0000000000..d1e03f8e02
--- /dev/null
+++ b/dolphinscheduler-worker/src/main/assembly/dolphinscheduler-worker-server.xml
@@ -0,0 +1,64 @@
+
+
+
+ dolphinscheduler-worker-server
+
+ dir
+
+ false
+ worker-server
+
+
+ ${basedir}/src/main/resources
+
+ *.yaml
+ *.xml
+
+ conf
+
+
+ ${basedir}/src/main/bin
+ bin
+ 0755
+ 0755
+
+
+ ${basedir}/../script/env
+ bin
+
+ dolphinscheduler_env.sh
+
+ 0755
+ 0755
+
+
+ ${basedir}/../dolphinscheduler-common/src/main/resources
+
+ **/*.properties
+
+ conf
+
+
+
+
+ libs
+
+
+
diff --git a/dolphinscheduler-worker/src/main/bin/start.sh b/dolphinscheduler-worker/src/main/bin/start.sh
new file mode 100644
index 0000000000..0f21c64f80
--- /dev/null
+++ b/dolphinscheduler-worker/src/main/bin/start.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+BIN_DIR=$(dirname $0)
+DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/..; pwd)}
+
+JAVA_OPTS=${JAVA_OPTS:-"-server -Xms4g -Xmx4g -Xmn2g -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"}
+
+if [[ "$DOCKER" == "true" ]]; then
+ JAVA_OPTS="${JAVA_OPTS} -XX:-UseContainerSupport"
+fi
+
+java $JAVA_OPTS \
+ -cp "$DOLPHINSCHEDULER_HOME/conf":"$DOLPHINSCHEDULER_HOME/libs/*" \
+ org.apache.dolphinscheduler.server.worker.WorkerServer
diff --git a/dolphinscheduler-worker/src/main/docker/Dockerfile b/dolphinscheduler-worker/src/main/docker/Dockerfile
new file mode 100644
index 0000000000..56fe987774
--- /dev/null
+++ b/dolphinscheduler-worker/src/main/docker/Dockerfile
@@ -0,0 +1,34 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+FROM openjdk:8-jre-slim-buster
+
+ENV DOCKER true
+ENV TZ Asia/Shanghai
+ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler
+
+RUN apt update ; \
+ apt install -y curl ; \
+ rm -rf /var/lib/apt/lists/*
+
+WORKDIR $DOLPHINSCHEDULER_HOME
+
+ADD ./target/worker-server $DOLPHINSCHEDULER_HOME
+
+EXPOSE 1235
+
+CMD [ "/bin/bash", "./bin/start.sh" ]
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java
similarity index 89%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java
rename to dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java
index 765724f072..ad70991d6c 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java
@@ -21,7 +21,6 @@ import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.IStoppable;
import org.apache.dolphinscheduler.common.enums.NodeType;
import org.apache.dolphinscheduler.common.thread.Stopper;
-import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.remote.NettyRemotingServer;
import org.apache.dolphinscheduler.remote.command.CommandType;
import org.apache.dolphinscheduler.remote.config.NettyServerConfig;
@@ -45,23 +44,14 @@ import javax.annotation.PostConstruct;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.builder.SpringApplicationBuilder;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.ComponentScan;
-import org.springframework.context.annotation.FilterType;
import org.springframework.transaction.annotation.EnableTransactionManagement;
-/**
- * worker server
- */
-@ComponentScan(value = "org.apache.dolphinscheduler", excludeFilters = {
- @ComponentScan.Filter(type = FilterType.REGEX, pattern = {
- "org.apache.dolphinscheduler.server.master.*",
- "org.apache.dolphinscheduler.server.monitor.*",
- "org.apache.dolphinscheduler.server.log.*",
- "org.apache.dolphinscheduler.alert.*"
- })
-})
+@SpringBootApplication
@EnableTransactionManagement
+@ComponentScan("org.apache.dolphinscheduler")
public class WorkerServer implements IStoppable {
/**
@@ -114,9 +104,7 @@ public class WorkerServer implements IStoppable {
*/
public static void main(String[] args) {
Thread.currentThread().setName(Constants.THREAD_NAME_WORKER_SERVER);
- new SpringApplicationBuilder(WorkerServer.class)
- .profiles("worker")
- .run(args);
+ SpringApplication.run(WorkerServer.class);
}
/**
@@ -125,7 +113,8 @@ public class WorkerServer implements IStoppable {
@PostConstruct
public void run() {
// alert-server client registry
- alertClientService = new AlertClientService(workerConfig.getAlertListenHost(), PropertyUtils.getInt(Constants.ALERT_RPC_PORT, 50052));
+ alertClientService = new AlertClientService(workerConfig.getAlertListenHost(),
+ workerConfig.getAlertListenPort());
// init remoting server
NettyServerConfig serverConfig = new NettyServerConfig();
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/cache/ResponceCache.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/cache/ResponceCache.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/cache/ResponceCache.java
rename to dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/cache/ResponceCache.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java
similarity index 93%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java
rename to dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java
index 4dca41234d..5192e27d61 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java
@@ -36,6 +36,7 @@ public class WorkerConfig {
private double reservedMemory;
private Set groups;
private String alertListenHost;
+ private int alertListenPort;
public int getListenPort() {
return listenPort;
@@ -108,4 +109,12 @@ public class WorkerConfig {
public void setAlertListenHost(String alertListenHost) {
this.alertListenHost = alertListenHost;
}
+
+ public int getAlertListenPort() {
+ return alertListenPort;
+ }
+
+ public void setAlertListenPort(final int alertListenPort) {
+ this.alertListenPort = alertListenPort;
+ }
}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/plugin/TaskPluginManager.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/plugin/TaskPluginManager.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/plugin/TaskPluginManager.java
rename to dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/plugin/TaskPluginManager.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/DBTaskAckProcessor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/DBTaskAckProcessor.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/DBTaskAckProcessor.java
rename to dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/DBTaskAckProcessor.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/DBTaskResponseProcessor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/DBTaskResponseProcessor.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/DBTaskResponseProcessor.java
rename to dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/DBTaskResponseProcessor.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/HostUpdateProcessor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/HostUpdateProcessor.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/HostUpdateProcessor.java
rename to dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/HostUpdateProcessor.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackService.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackService.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackService.java
rename to dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackService.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java
rename to dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java
rename to dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryClient.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryClient.java
similarity index 99%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryClient.java
rename to dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryClient.java
index ca1a6e3b48..037db2a328 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryClient.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryClient.java
@@ -112,7 +112,7 @@ public class WorkerRegistryClient {
Constants.WORKER_TYPE,
registryClient,
workerConfig.getExecThreads(),
- workerManagerThread
+ workerManagerThread.getThreadPoolQueueSize()
);
this.heartBeatExecutor.scheduleAtFixedRate(heartBeatTask, workerHeartbeatInterval, workerHeartbeatInterval, TimeUnit.SECONDS);
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/RetryReportTaskStatusThread.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/RetryReportTaskStatusThread.java
similarity index 92%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/RetryReportTaskStatusThread.java
rename to dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/RetryReportTaskStatusThread.java
index b2d00317a5..394eb79cde 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/RetryReportTaskStatusThread.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/RetryReportTaskStatusThread.java
@@ -26,6 +26,7 @@ import org.apache.dolphinscheduler.server.worker.processor.TaskCallbackService;
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.Map;
@@ -43,10 +44,8 @@ public class RetryReportTaskStatusThread implements Runnable {
*/
private static long RETRY_REPORT_TASK_STATUS_INTERVAL = 5 * 60 * 1000L;
- /**
- * task callback service
- */
- private final TaskCallbackService taskCallbackService;
+ @Autowired
+ private TaskCallbackService taskCallbackService;
public void start(){
Thread thread = new Thread(this,"RetryReportTaskStatusThread");
@@ -54,10 +53,6 @@ public class RetryReportTaskStatusThread implements Runnable {
thread.start();
}
- public RetryReportTaskStatusThread(){
- this.taskCallbackService = SpringApplicationContext.getBean(TaskCallbackService.class);
- }
-
/**
* retry ack/response
*/
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java
rename to dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerManagerThread.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerManagerThread.java
similarity index 100%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerManagerThread.java
rename to dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerManagerThread.java
diff --git a/dolphinscheduler-server/src/main/resources/application-worker.yaml b/dolphinscheduler-worker/src/main/resources/application.yaml
similarity index 69%
rename from dolphinscheduler-server/src/main/resources/application-worker.yaml
rename to dolphinscheduler-worker/src/main/resources/application.yaml
index d30f86d862..9d5d0ff67a 100644
--- a/dolphinscheduler-server/src/main/resources/application-worker.yaml
+++ b/dolphinscheduler-worker/src/main/resources/application.yaml
@@ -15,8 +15,40 @@
# limitations under the License.
#
spring:
+ main:
+ banner-mode: off
application:
name: worker-server
+ datasource:
+ driver-class-name: org.postgresql.Driver
+ url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler
+ username: root
+ password: root
+ hikari:
+ connection-test-query: select 1
+ minimum-idle: 5
+ auto-commit: true
+ validation-timeout: 3000
+ pool-name: DolphinScheduler
+ maximum-pool-size: 50
+ connection-timeout: 30000
+ idle-timeout: 600000
+ leak-detection-threshold: 0
+ initialization-fail-timeout: 1
+
+registry:
+ type: zookeeper
+ zookeeper:
+ namespace: dolphinscheduler
+ connect-string: localhost:2181
+ retry-policy:
+ base-sleep-time: 60ms
+ max-sleep: 300ms
+ max-retries: 5
+ session-timeout: 30s
+ connection-timeout: 9s
+ block-until-connected: 600ms
+ digest: ~
worker:
# worker listener port
@@ -38,6 +70,7 @@ worker:
- default
# alert server listen host
alert-listen-host: localhost
+ alert-listen-port: 50052
server:
port: 1235
diff --git a/dolphinscheduler-server/src/main/resources/logback-worker.xml b/dolphinscheduler-worker/src/main/resources/logback-spring.xml
similarity index 94%
rename from dolphinscheduler-server/src/main/resources/logback-worker.xml
rename to dolphinscheduler-worker/src/main/resources/logback-spring.xml
index 1c45037e41..773fd4365a 100644
--- a/dolphinscheduler-server/src/main/resources/logback-worker.xml
+++ b/dolphinscheduler-worker/src/main/resources/logback-spring.xml
@@ -16,10 +16,9 @@
~ limitations under the License.
-->
-
-
-
+
+
@@ -66,8 +65,12 @@
+
+
+
+
+
-
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackServiceTestConfig.java b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/worker/processor/TaskCallbackServiceTestConfig.java
similarity index 100%
rename from dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackServiceTestConfig.java
rename to dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/worker/processor/TaskCallbackServiceTestConfig.java
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessorTest.java b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/worker/processor/TaskExecuteProcessorTest.java
similarity index 100%
rename from dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessorTest.java
rename to dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/worker/processor/TaskExecuteProcessorTest.java
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryClientTest.java b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/worker/registry/WorkerRegistryClientTest.java
similarity index 100%
rename from dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryClientTest.java
rename to dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/worker/registry/WorkerRegistryClientTest.java
diff --git a/pom.xml b/pom.xml
index fd48f53498..2bb5971dfb 100644
--- a/pom.xml
+++ b/pom.xml
@@ -58,7 +58,7 @@
1.8
1.2.3
2.7.3
- 2.3.0
+ 2.3.2
2.10.5
3.2.0
2.0.1
@@ -96,7 +96,7 @@
2.12.0
1.6
3.3
- 3.1.0
+ 3.3.0
2.5.3
2.10.3
2.4
@@ -122,6 +122,16 @@
4.1.53.Final
3.2.0
2.0.9
+ 3.0.0
+ 1.19
+ 3.1.1
+ 2.5.1
+ 3.0.0
+ 3.1.6
+
+ apache
+ ${project.name}
+ ${project.version}
@@ -221,6 +231,16 @@
dolphinscheduler-server
${project.version}
+
+ org.apache.dolphinscheduler
+ dolphinscheduler-master
+ ${project.version}
+
+
+ org.apache.dolphinscheduler
+ dolphinscheduler-worker
+ ${project.version}
+
org.apache.dolphinscheduler
dolphinscheduler-standalone-server
@@ -443,6 +463,11 @@
dolphinscheduler-task-sqoop
${project.version}
+
+ org.apache.dolphinscheduler
+ dolphinscheduler-ui
+ ${project.version}
+
org.apache.curator
@@ -809,6 +834,33 @@
py4j
${py4j.version}
+
+
+ org.codehaus.janino
+ janino
+ ${janino.version}
+
+
+
+ com.google.code.findbugs
+ jsr305
+ ${jsr305.version}
+
+
+ org.apache.commons
+ commons-compress
+ ${commons-compress.version}
+
+
+ org.apache.commons
+ commons-math3
+ ${commons-math3.version}
+
+
+ com.google.errorprone
+ error_prone_annotations
+ ${error_prone_annotations.version}
+
@@ -866,12 +918,6 @@
-
- org.apache.maven.plugins
- maven-source-plugin
- ${maven-source-plugin.version}
-
-
org.apache.maven.plugins
maven-dependency-plugin
@@ -882,6 +928,61 @@
maven-jar-plugin
${maven-jar-plugin.version}
+
+ org.codehaus.mojo
+ exec-maven-plugin
+ ${exec-maven-plugin.version}
+
+
+ docker-build
+ package
+
+ exec
+
+
+
+ 1
+
+ docker
+ ${project.basedir}
+
+ build
+ -t
+ ${docker.hub}/${docker.repo}:${docker.tag}
+ -t
+ ${docker.hub}/${docker.repo}:latest
+ .
+ --file=src/main/docker/Dockerfile
+
+
+
+
+ docker-push
+ deploy
+
+ exec
+
+
+
+ 1
+
+ docker
+ ${project.basedir}
+
+ buildx
+ build
+ --push
+ -t
+ ${docker.hub}/${docker.repo}:${docker.tag}
+ -t
+ ${docker.hub}/${docker.repo}:latest
+ .
+ --file=src/main/docker/Dockerfile
+
+
+
+
+
@@ -891,20 +992,6 @@
provisio-maven-plugin
true
-
- org.apache.maven.plugins
- maven-source-plugin
-
-
- attach-sources
- verify
-
- jar-no-fork
-
-
-
-
-
org.apache.maven.plugins
maven-javadoc-plugin
@@ -1072,6 +1159,18 @@
+
+ maven-source-plugin
+ ${maven-source-plugin.version}
+
+
+ attach-sources
+
+ jar
+
+
+
+
@@ -1143,5 +1242,9 @@
dolphinscheduler-datasource-plugin
dolphinscheduler-python
dolphinscheduler-meter
+ dolphinscheduler-master
+ dolphinscheduler-worker
+ dolphinscheduler-log-server
+ dolphinscheduler-tools
diff --git a/script/create-dolphinscheduler.sh b/script/create-dolphinscheduler.sh
deleted file mode 100755
index 8591277ad1..0000000000
--- a/script/create-dolphinscheduler.sh
+++ /dev/null
@@ -1,41 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-BIN_DIR=`dirname $0`
-BIN_DIR=`cd "$BIN_DIR"; pwd`
-DOLPHINSCHEDULER_HOME=$BIN_DIR/..
-
-export JAVA_HOME=$JAVA_HOME
-
-export DATABASE_TYPE=${DATABASE_TYPE:-"h2"}
-export SPRING_PROFILES_ACTIVE=${SPRING_PROFILES_ACTIVE:-"default"}
-export SPRING_PROFILES_ACTIVE="${SPRING_PROFILES_ACTIVE},${DATABASE_TYPE}"
-
-export DOLPHINSCHEDULER_CONF_DIR=$DOLPHINSCHEDULER_HOME/conf
-export DOLPHINSCHEDULER_LIB_JARS=$DOLPHINSCHEDULER_HOME/lib/*
-export DOLPHINSCHEDULER_SQL_DIR=$DOLPHINSCHEDULER_HOME/sql
-
-export DOLPHINSCHEDULER_OPTS="-server -Xms64m -Xmx64m -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=64m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70"
-export STOP_TIMEOUT=5
-
-CLASS=org.apache.dolphinscheduler.dao.upgrade.shell.CreateDolphinScheduler
-
-exec_command="$DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_SQL_DIR:$DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS"
-
-cd $DOLPHINSCHEDULER_HOME
-$JAVA_HOME/bin/java $exec_command
diff --git a/script/dolphinscheduler-daemon.sh b/script/dolphinscheduler-daemon.sh
index 64d6b8327c..a75a9c94b3 100755
--- a/script/dolphinscheduler-daemon.sh
+++ b/script/dolphinscheduler-daemon.sh
@@ -35,16 +35,11 @@ BIN_DIR=`dirname $0`
BIN_DIR=`cd "$BIN_DIR"; pwd`
DOLPHINSCHEDULER_HOME=`cd "$BIN_DIR/.."; pwd`
-source /etc/profile
-source "${DOLPHINSCHEDULER_HOME}/conf/env/dolphinscheduler_env.sh"
+source "${DOLPHINSCHEDULER_HOME}/bin/env/dolphinscheduler_env.sh"
export HOSTNAME=`hostname`
-export DOLPHINSCHEDULER_PID_DIR=$DOLPHINSCHEDULER_HOME/pid
-export DOLPHINSCHEDULER_LOG_DIR=$DOLPHINSCHEDULER_HOME/logs
-export DOLPHINSCHEDULER_CONF_DIR=$DOLPHINSCHEDULER_HOME/conf
-export DOLPHINSCHEDULER_SQL_DIR=$DOLPHINSCHEDULER_HOME/sql
-export DOLPHINSCHEDULER_LIB_JARS=$DOLPHINSCHEDULER_HOME/lib/*
+export DOLPHINSCHEDULER_LOG_DIR=$DOLPHINSCHEDULER_HOME/$command/logs
export STOP_TIMEOUT=5
@@ -52,47 +47,23 @@ if [ ! -d "$DOLPHINSCHEDULER_LOG_DIR" ]; then
mkdir $DOLPHINSCHEDULER_LOG_DIR
fi
-log=$DOLPHINSCHEDULER_LOG_DIR/dolphinscheduler-$command-$HOSTNAME.out
-pid=$DOLPHINSCHEDULER_PID_DIR/dolphinscheduler-$command.pid
+log=$DOLPHINSCHEDULER_HOME/$command-$HOSTNAME.out
+pid=$DOLPHINSCHEDULER_HOME/$command/pid
-cd $DOLPHINSCHEDULER_HOME
-
-export DOLPHINSCHEDULER_OPTS="-server -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=128m -Xss512k -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 -XX:+PrintGCDetails -Xloggc:$DOLPHINSCHEDULER_LOG_DIR/gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof -XshowSettings:vm $DOLPHINSCHEDULER_OPTS"
-
-export DATABASE_TYPE=${DATABASE_TYPE:-"h2"}
-export SPRING_PROFILES_ACTIVE=${SPRING_PROFILES_ACTIVE:-"default"}
+cd $DOLPHINSCHEDULER_HOME/$command
if [ "$command" = "api-server" ]; then
- LOG_FILE="-Dlogging.config=classpath:logback-api.xml"
- CLASS=org.apache.dolphinscheduler.api.ApiApplicationServer
- HEAP_OPTS="-Xms1g -Xmx1g -Xmn512m"
- export DOLPHINSCHEDULER_OPTS="$HEAP_OPTS $DOLPHINSCHEDULER_OPTS $API_SERVER_OPTS"
- export SPRING_PROFILES_ACTIVE="${SPRING_PROFILES_ACTIVE},api,${DATABASE_TYPE}"
+ :
elif [ "$command" = "master-server" ]; then
- LOG_FILE="-Dlogging.config=classpath:logback-master.xml"
- CLASS=org.apache.dolphinscheduler.server.master.MasterServer
- HEAP_OPTS="-Xms4g -Xmx4g -Xmn2g"
- export DOLPHINSCHEDULER_OPTS="$HEAP_OPTS $DOLPHINSCHEDULER_OPTS $MASTER_SERVER_OPTS"
- export SPRING_PROFILES_ACTIVE="${SPRING_PROFILES_ACTIVE},master,${DATABASE_TYPE}"
+ :
elif [ "$command" = "worker-server" ]; then
- LOG_FILE="-Dlogging.config=classpath:logback-worker.xml"
- CLASS=org.apache.dolphinscheduler.server.worker.WorkerServer
- HEAP_OPTS="-Xms2g -Xmx2g -Xmn1g"
- export DOLPHINSCHEDULER_OPTS="$HEAP_OPTS $DOLPHINSCHEDULER_OPTS $WORKER_SERVER_OPTS"
- export SPRING_PROFILES_ACTIVE="${SPRING_PROFILES_ACTIVE},worker,${DATABASE_TYPE}"
+ :
elif [ "$command" = "alert-server" ]; then
- LOG_FILE="-Dlogback.configurationFile=conf/logback-alert.xml"
- CLASS=org.apache.dolphinscheduler.alert.AlertServer
- HEAP_OPTS="-Xms1g -Xmx1g -Xmn512m"
- export DOLPHINSCHEDULER_OPTS="$HEAP_OPTS $DOLPHINSCHEDULER_OPTS $ALERT_SERVER_OPTS"
- export SPRING_PROFILES_ACTIVE="${SPRING_PROFILES_ACTIVE},alert,${DATABASE_TYPE}"
+ :
elif [ "$command" = "logger-server" ]; then
- CLASS=org.apache.dolphinscheduler.server.log.LoggerServer
- HEAP_OPTS="-Xms1g -Xmx1g -Xmn512m"
- export DOLPHINSCHEDULER_OPTS="$HEAP_OPTS $DOLPHINSCHEDULER_OPTS $LOGGER_SERVER_OPTS"
+ :
elif [ "$command" = "standalone-server" ]; then
- CLASS=org.apache.dolphinscheduler.server.StandaloneServer
- export SPRING_PROFILES_ACTIVE="${SPRING_PROFILES_ACTIVE},standalone,${DATABASE_TYPE}"
+ :
else
echo "Error: No command named '$command' was found."
exit 1
@@ -100,40 +71,21 @@ fi
case $startStop in
(start)
- if [ "$DOCKER" = "true" ]; then
- echo start $command in docker
- export DOLPHINSCHEDULER_OPTS="$DOLPHINSCHEDULER_OPTS -XX:-UseContainerSupport"
- exec_command="$LOG_FILE $DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_SQL_DIR:$DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS"
- $JAVA_HOME/bin/java $exec_command
- else
- [ -w "$DOLPHINSCHEDULER_PID_DIR" ] || mkdir -p "$DOLPHINSCHEDULER_PID_DIR"
-
- if [ -f $pid ]; then
- if kill -0 `cat $pid` > /dev/null 2>&1; then
- echo $command running as process `cat $pid`. Stop it first.
- exit 1
- fi
- fi
-
- echo starting $command, logging to $log
- exec_command="$LOG_FILE $DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_SQL_DIR:$DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS"
- echo "nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 &"
- nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 &
- echo $! > $pid
- fi
+ echo starting $command, logging to $DOLPHINSCHEDULER_LOG_DIR
+ nohup "$DOLPHINSCHEDULER_HOME/$command/bin/start.sh" > $log 2>&1 &
+ echo $! > $pid
;;
(stop)
-
if [ -f $pid ]; then
TARGET_PID=`cat $pid`
if kill -0 $TARGET_PID > /dev/null 2>&1; then
echo stopping $command
- kill $TARGET_PID
+ pkill -P $TARGET_PID
sleep $STOP_TIMEOUT
if kill -0 $TARGET_PID > /dev/null 2>&1; then
echo "$command did not stop gracefully after $STOP_TIMEOUT seconds: killing with kill -9"
- kill -9 $TARGET_PID
+ pkill -P -9 $TARGET_PID
fi
else
echo no $command to stop
diff --git a/script/env/dolphinscheduler_env.sh b/script/env/dolphinscheduler_env.sh
index 1798a5d311..f0fdbaa46e 100755
--- a/script/env/dolphinscheduler_env.sh
+++ b/script/env/dolphinscheduler_env.sh
@@ -15,14 +15,26 @@
# limitations under the License.
#
-export HADOOP_HOME=/opt/soft/hadoop
-export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop
-export SPARK_HOME1=/opt/soft/spark1
-export SPARK_HOME2=/opt/soft/spark2
-export PYTHON_HOME=/opt/soft/python
+export HADOOP_HOME=${HADOOP_HOME:-/opt/soft/hadoop}
+export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/opt/soft/hadoop/etc/hadoop}
+export SPARK_HOME1=${SPARK_HOME1:-/opt/soft/spark1}
+export SPARK_HOME2=${SPARK_HOME2:-/opt/soft/spark2}
+export PYTHON_HOME=${PYTHON_HOME:-/opt/soft/python}
export JAVA_HOME=${JAVA_HOME:-/opt/soft/java}
-export HIVE_HOME=/opt/soft/hive
-export FLINK_HOME=/opt/soft/flink
-export DATAX_HOME=/opt/soft/datax
+export HIVE_HOME=${HIVE_HOME:-/opt/soft/hive}
+export FLINK_HOME=${FLINK_HOME:-/opt/soft/flink}
+export DATAX_HOME=${DATAX_HOME:-/opt/soft/datax}
export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH
+
+export DATABASE=${DATABASE:-postgresql}
+export SPRING_PROFILES_ACTIVE=${DATABASE}
+export SPRING_DATASOURCE_URL
+export SPRING_DATASOURCE_USERNAME
+export SPRING_DATASOURCE_PASSWORD
+export SPRING_CACHE_TYPE=none
+
+export MASTER_FETCH_COMMAND_NUM=10
+
+export REGISTRY_TYPE=zookeeper
+export REGISTRY_ZOOKEEPER_CONNECT_STRING=localhost:2181
diff --git a/dolphinscheduler-server/src/main/resources/config/install_config.conf b/script/env/install_env.sh
old mode 100755
new mode 100644
similarity index 83%
rename from dolphinscheduler-server/src/main/resources/config/install_config.conf
rename to script/env/install_env.sh
index 50adf3c621..8b68c0ea00
--- a/dolphinscheduler-server/src/main/resources/config/install_config.conf
+++ b/script/env/install_env.sh
@@ -22,48 +22,40 @@
# including master, worker, api, alert. If you want to deploy in pseudo-distributed
# mode, just write a pseudo-distributed hostname
# Example for hostnames: ips="ds1,ds2,ds3,ds4,ds5", Example for IPs: ips="192.168.8.1,192.168.8.2,192.168.8.3,192.168.8.4,192.168.8.5"
-ips="ds1,ds2,ds3,ds4,ds5"
+ips=${ips:-"ds1,ds2,ds3,ds4,ds5"}
# Port of SSH protocol, default value is 22. For now we only support same port in all `ips` machine
# modify it if you use different ssh port
-sshPort="22"
+sshPort=${sshPort:-"22"}
# A comma separated list of machine hostname or IP would be installed Master server, it
# must be a subset of configuration `ips`.
# Example for hostnames: masters="ds1,ds2", Example for IPs: masters="192.168.8.1,192.168.8.2"
-masters="ds1,ds2"
+masters=${masters:-"ds1,ds2"}
# A comma separated list of machine : or :.All hostname or IP must be a
# subset of configuration `ips`, And workerGroup have default value as `default`, but we recommend you declare behind the hosts
# Example for hostnames: workers="ds1:default,ds2:default,ds3:default", Example for IPs: workers="192.168.8.1:default,192.168.8.2:default,192.168.8.3:default"
-workers="ds1:default,ds2:default,ds3:default,ds4:default,ds5:default"
+workers=${workers:-"ds1:default,ds2:default,ds3:default,ds4:default,ds5:default"}
# A comma separated list of machine hostname or IP would be installed Alert server, it
# must be a subset of configuration `ips`.
# Example for hostname: alertServer="ds3", Example for IP: alertServer="192.168.8.3"
-alertServer="ds3"
+alertServer=${alertServer:-"ds3"}
# A comma separated list of machine hostname or IP would be installed API server, it
# must be a subset of configuration `ips`.
# Example for hostname: apiServers="ds1", Example for IP: apiServers="192.168.8.1"
-apiServers="ds1"
+apiServers=${apiServers:-"ds1"}
# The directory to install DolphinScheduler for all machine we config above. It will automatically be created by `install.sh` script if not exists.
# Do not set this configuration same as the current path (pwd)
-installPath="/data1_1T/dolphinscheduler"
+installPath=${installPath:-"/tmp/dolphinscheduler"}
# The user to deploy DolphinScheduler for all machine we config above. For now user must create by yourself before running `install.sh`
# script. The user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled than the root directory needs
# to be created by this user
-deployUser="dolphinscheduler"
-
-# ---------------------------------------------------------
-# Database
-# NOTICE: If database value has special characters, such as `.*[]^${}\+?|()@#&`, Please add prefix `\` for escaping.
-# ---------------------------------------------------------
-# The type for the metadata database
-# Supported values: ``postgresql``, ``mysql``.
-dbtype="mysql"
+deployUser=${deployUser:-"dolphinscheduler"}
# The root of zookeeper, for now DolphinScheduler default registry server is zookeeper.
-zkRoot="/dolphinscheduler"
+zkRoot=${zkRoot:-"/dolphinscheduler"}
diff --git a/install.sh b/script/install.sh
old mode 100755
new mode 100644
similarity index 80%
rename from install.sh
rename to script/install.sh
index dac369b0d2..390ea7302b
--- a/install.sh
+++ b/script/install.sh
@@ -19,9 +19,9 @@
workDir=`dirname $0`
workDir=`cd ${workDir};pwd`
-source ${workDir}/conf/config/install_config.conf
+source ${workDir}/env/install_env.sh
+source ${workDir}/env/dolphinscheduler_env.sh
-# 1.create directory
echo "1.create directory"
if [ ! -d $installPath ];then
@@ -29,9 +29,8 @@ if [ ! -d $installPath ];then
sudo chown -R $deployUser:$deployUser $installPath
fi
-# 2.scp resources
echo "2.scp resources"
-sh ${workDir}/script/scp-hosts.sh
+sh ${workDir}/scp-hosts.sh
if [ $? -eq 0 ];then
echo 'scp copy completed'
else
@@ -39,14 +38,11 @@ else
exit 1
fi
-# 3.stop server
echo "3.stop server"
-sh ${workDir}/script/stop-all.sh
+sh ${workDir}/stop-all.sh
-# 4.delete zk node
echo "4.delete zk node"
-sh ${workDir}/script/remove-zk-node.sh $zkRoot
+sh ${workDir}/remove-zk-node.sh $zkRoot
-# 5.startup
echo "5.startup"
-sh ${workDir}/script/start-all.sh
+sh ${workDir}/start-all.sh
diff --git a/script/monitor-server.sh b/script/monitor-server.sh
deleted file mode 100755
index 05d46048aa..0000000000
--- a/script/monitor-server.sh
+++ /dev/null
@@ -1,52 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-print_usage(){
- printf $"USAGE:$0 masterPath workerPath port installPath\n"
- exit 1
-}
-
-if [ $# -ne 4 ];then
- print_usage
-fi
-
-masterPath=$1
-workerPath=$2
-port=$3
-installPath=$4
-
-
-BIN_DIR=`dirname $0`
-BIN_DIR=`cd "$BIN_DIR"; pwd`
-DOLPHINSCHEDULER_HOME=$BIN_DIR/..
-
-export JAVA_HOME=$JAVA_HOME
-
-
-export DOLPHINSCHEDULER_CONF_DIR=$DOLPHINSCHEDULER_HOME/conf
-export DOLPHINSCHEDULER_LIB_JARS=$DOLPHINSCHEDULER_HOME/lib/*
-
-export DOLPHINSCHEDULER_OPTS="-server -Xmx1g -Xms1g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70"
-export STOP_TIMEOUT=5
-
-CLASS=org.apache.dolphinscheduler.server.monitor.MonitorServer
-
-exec_command="$DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS $masterPath $workerPath $port $installPath"
-
-cd $DOLPHINSCHEDULER_HOME
-$JAVA_HOME/bin/java $exec_command
diff --git a/script/remove-zk-node.sh b/script/remove-zk-node.sh
index 2cb8a2b284..5abffeddb2 100755
--- a/script/remove-zk-node.sh
+++ b/script/remove-zk-node.sh
@@ -31,12 +31,11 @@ BIN_DIR=`dirname $0`
BIN_DIR=`cd "$BIN_DIR"; pwd`
DOLPHINSCHEDULER_HOME=$BIN_DIR/..
-source ${BIN_DIR}/../conf/config/install_config.conf
-source ${BIN_DIR}/../conf/env/dolphinscheduler_env.sh
+source ${workDir}/env/install_env.sh
+source ${workDir}/dolphinscheduler_env.sh
export JAVA_HOME=$JAVA_HOME
-
export DOLPHINSCHEDULER_CONF_DIR=$DOLPHINSCHEDULER_HOME/conf
export DOLPHINSCHEDULER_LIB_JARS=$DOLPHINSCHEDULER_HOME/lib/*
diff --git a/script/scp-hosts.sh b/script/scp-hosts.sh
index 8def4d1bae..4f742f9822 100755
--- a/script/scp-hosts.sh
+++ b/script/scp-hosts.sh
@@ -18,7 +18,8 @@
workDir=`dirname $0`
workDir=`cd ${workDir};pwd`
-source $workDir/../conf/config/install_config.conf
+
+source ${workDir}/env/install_env.sh
txt=""
if [[ "$OSTYPE" == "darwin"* ]]; then
@@ -26,7 +27,7 @@ if [[ "$OSTYPE" == "darwin"* ]]; then
txt="''"
fi
-declare -A workersGroupMap=()
+workersGroupMap=()
workersGroup=(${workers//,/ })
for workerGroup in ${workersGroup[@]}
@@ -37,7 +38,6 @@ do
workersGroupMap+=([$worker]=$groupsName)
done
-
hostsArr=(${ips//,/ })
for host in ${hostsArr[@]}
do
@@ -49,11 +49,11 @@ do
echo "scp dirs to $host/$installPath starting"
ssh -p $sshPort $host "cd $installPath/; rm -rf bin/ conf/ lib/ script/ sql/ ui/"
- for dsDir in bin conf lib script sql ui install.sh
+ for dsDir in bin master-server worker-server alert-server api-server logger-server ui
do
# if worker in workersGroupMap
- if [[ "${workersGroupMap[${host}]}" ]] && [[ "${dsDir}" == "conf" ]]; then
- sed -i ${txt} "s@^#\?worker.groups=.*@worker.groups=${workersGroupMap[${host}]}@g" ${dsDir}/worker.properties
+ if [[ "${workersGroupMap[${host}]}" ]]; then
+ echo "export WORKER_GROUPS_0=${workersGroupMap[${host}]}" >> worker-server/bin/dolphinscheduler_env.sh
fi
echo "start to scp $dsDir to $host/$installPath"
diff --git a/script/start-all.sh b/script/start-all.sh
index d0c0ab8188..7deb94f8fe 100755
--- a/script/start-all.sh
+++ b/script/start-all.sh
@@ -18,9 +18,10 @@
workDir=`dirname $0`
workDir=`cd ${workDir};pwd`
-source $workDir/../conf/config/install_config.conf
-declare -A workersGroupMap=()
+source ${workDir}/env/install_env.sh
+
+workersGroupMap=()
workersGroup=(${workers//,/ })
for workerGroup in ${workersGroup[@]}
diff --git a/script/status-all.sh b/script/status-all.sh
index 555a2c1ead..2fa51d3f16 100755
--- a/script/status-all.sh
+++ b/script/status-all.sh
@@ -18,9 +18,10 @@
workDir=`dirname $0`
workDir=`cd ${workDir};pwd`
-source $workDir/../conf/config/install_config.conf
-# install_config.conf info
+source ${workDir}/env/install_env.sh
+
+# install_env.sh info
echo -e '\n'
echo "====================== dolphinscheduler server config ============================="
echo -e "1.dolphinscheduler server node config hosts:[ \033[1;32m ${ips} \033[0m ]"
@@ -36,7 +37,7 @@ firstColumn="node server state"
echo $firstColumn
echo -e '\n'
-declare -A workersGroupMap=()
+workersGroupMap=()
workersGroup=(${workers//,/ })
for workerGroup in ${workersGroup[@]}
diff --git a/script/stop-all.sh b/script/stop-all.sh
index 20175e9a51..2630e23d99 100755
--- a/script/stop-all.sh
+++ b/script/stop-all.sh
@@ -19,7 +19,7 @@
workDir=`dirname $0`
workDir=`cd ${workDir};pwd`
-source $workDir/../conf/config/install_config.conf
+source ${workDir}/env/install_env.sh
declare -A workersGroupMap=()
@@ -55,7 +55,3 @@ do
echo "$apiServer worker server is stopping"
ssh -p $sshPort $apiServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop api-server;"
done
-
-# query server status
-echo "query server status"
-cd $installPath/; sh bin/status-all.sh
diff --git a/script/upgrade-dolphinscheduler.sh b/script/upgrade-dolphinscheduler.sh
deleted file mode 100755
index a8ef4e7307..0000000000
--- a/script/upgrade-dolphinscheduler.sh
+++ /dev/null
@@ -1,41 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-BIN_DIR=`dirname $0`
-BIN_DIR=`cd "$BIN_DIR"; pwd`
-DOLPHINSCHEDULER_HOME=$BIN_DIR/..
-
-export JAVA_HOME=$JAVA_HOME
-
-export DATABASE_TYPE=${DATABASE_TYPE:-"h2"}
-export SPRING_PROFILES_ACTIVE=${SPRING_PROFILES_ACTIVE:-"default"}
-export SPRING_PROFILES_ACTIVE="${SPRING_PROFILES_ACTIVE},${DATABASE_TYPE}"
-
-export DOLPHINSCHEDULER_CONF_DIR=$DOLPHINSCHEDULER_HOME/conf
-export DOLPHINSCHEDULER_LIB_JARS=$DOLPHINSCHEDULER_HOME/lib/*
-export DOLPHINSCHEDULER_SQL_DIR=$DOLPHINSCHEDULER_HOME/sql
-
-export DOLPHINSCHEDULER_OPTS="-server -Xms64m -Xmx64m -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=64m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70"
-export STOP_TIMEOUT=5
-
-CLASS=org.apache.dolphinscheduler.dao.upgrade.shell.UpgradeDolphinScheduler
-
-exec_command="$DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_SQL_DIR:$DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS"
-
-cd $DOLPHINSCHEDULER_HOME
-$JAVA_HOME/bin/java $exec_command
diff --git a/tools/dependencies/check-LICENSE.sh b/tools/dependencies/check-LICENSE.sh
index 924d6d7133..81f2169b4d 100755
--- a/tools/dependencies/check-LICENSE.sh
+++ b/tools/dependencies/check-LICENSE.sh
@@ -25,10 +25,10 @@ tar -zxf dolphinscheduler-dist/target/apache-dolphinscheduler*-bin.tar.gz --stri
# licenses
echo '=== Self modules: ' && ./mvnw --batch-mode --quiet -Dexec.executable='echo' -Dexec.args='${project.artifactId}-${project.version}.jar' exec:exec | tee self-modules.txt
-echo '=== Distributed dependencies: ' && find dist/lib -name "*.jar" -exec basename {} \; | uniq | sort | tee all-dependencies.txt
+echo '=== Distributed dependencies: ' && find dist -name "*.jar" -exec basename {} \; | sort | uniq | tee all-dependencies.txt
# Exclude all self modules(jars) to generate all third-party dependencies
-echo '=== Third party dependencies: ' && grep -vf self-modules.txt all-dependencies.txt | uniq | sort | tee third-party-dependencies.txt
+echo '=== Third party dependencies: ' && grep -vf self-modules.txt all-dependencies.txt | sort | uniq | tee third-party-dependencies.txt
# 1. Compare the third-party dependencies with known dependencies, expect that all third-party dependencies are KNOWN
# and the exit code of the command is 0, otherwise we should add its license to LICENSE file and add the dependency to
@@ -36,4 +36,4 @@ echo '=== Third party dependencies: ' && grep -vf self-modules.txt all-dependenc
# command in target OS is different from what we used to sort the file `known-dependencies.txt`, i.e. "sort the two file
# using the same command (and default arguments)"
-diff -w -B -U0 <(sort < tools/dependencies/known-dependencies.txt) <(sort < third-party-dependencies.txt)
\ No newline at end of file
+diff -w -B -U0 <(sort < tools/dependencies/known-dependencies.txt) <(sort < third-party-dependencies.txt)
diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt
index c76dda886c..973f3cb1a4 100755
--- a/tools/dependencies/known-dependencies.txt
+++ b/tools/dependencies/known-dependencies.txt
@@ -26,7 +26,8 @@ commons-cli-1.2.jar
commons-codec-1.11.jar
commons-collections-3.2.2.jar
commons-collections4-4.1.jar
-commons-compress-1.4.1.jar
+commons-compiler-3.1.6.jar
+commons-compress-1.19.jar
commons-configuration-1.10.jar
commons-daemon-1.0.13.jar
commons-beanutils-1.9.4.jar
@@ -39,6 +40,8 @@ commons-math3-3.1.1.jar
commons-net-3.1.jar
commons-pool-1.6.jar
cron-utils-9.1.3.jar
+error_prone_annotations-2.5.1.jar
+janino-3.1.6.jar
javax.el-3.0.0.jar
commons-lang3-3.12.0.jar
curator-client-4.3.0.jar
@@ -50,7 +53,6 @@ datanucleus-api-jdo-4.2.1.jar
datanucleus-core-4.1.6.jar
datanucleus-rdbms-4.1.7.jar
derby-10.14.2.0.jar
-error_prone_annotations-2.1.3.jar
druid-1.2.4.jar
gson-2.8.8.jar
guava-24.1-jre.jar
@@ -134,7 +136,7 @@ jetty-xml-9.4.44.v20210927.jar
jline-0.9.94.jar
jna-4.5.2.jar
jna-platform-4.5.2.jar
-joda-time-2.5.jar
+joda-time-2.10.13.jar
jpam-1.1.jar
jsch-0.1.42.jar
jsp-api-2.1.jar
@@ -178,8 +180,8 @@ protostuff-runtime-1.7.2.jar
protostuff-api-1.7.2.jar
protostuff-collectionschema-1.7.2.jar
py4j-0.10.9.jar
-quartz-2.3.0.jar
-quartz-jobs-2.3.0.jar
+quartz-2.3.2.jar
+quartz-jobs-2.3.2.jar
reflections-0.9.12.jar
simpleclient-0.10.0.jar
simpleclient_common-0.10.0.jar
@@ -233,6 +235,5 @@ xercesImpl-2.9.1.jar
xml-apis-1.3.04.jar
xmlbeans-3.1.0.jar
xmlenc-0.52.jar
-xz-1.0.jar
zookeeper-3.4.14.jar
Java-WebSocket-1.5.1.jar