Browse Source

[Improvement-16311] Polish installation docs (#16312)

* polish installation docs

* fix cluster test error
dev
xiangzihao 4 months ago committed by GitHub
parent
commit
34cd747597
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
  1. 3
      .github/workflows/cluster-test/mysql_with_mysql_registry/Dockerfile
  2. 5
      .github/workflows/cluster-test/mysql_with_mysql_registry/deploy.sh
  3. 2
      .github/workflows/cluster-test/mysql_with_mysql_registry/dolphinscheduler_env.sh
  4. 5
      .github/workflows/cluster-test/mysql_with_mysql_registry/running_test.sh
  5. 3
      .github/workflows/cluster-test/mysql_with_mysql_registry/start-job.sh
  6. 3
      .github/workflows/cluster-test/mysql_with_zookeeper_registry/Dockerfile
  7. 5
      .github/workflows/cluster-test/mysql_with_zookeeper_registry/deploy.sh
  8. 61
      .github/workflows/cluster-test/mysql_with_zookeeper_registry/install_env.sh
  9. 5
      .github/workflows/cluster-test/mysql_with_zookeeper_registry/running_test.sh
  10. 3
      .github/workflows/cluster-test/mysql_with_zookeeper_registry/start-job.sh
  11. 3
      .github/workflows/cluster-test/postgresql_with_postgresql_registry/Dockerfile
  12. 5
      .github/workflows/cluster-test/postgresql_with_postgresql_registry/deploy.sh
  13. 5
      .github/workflows/cluster-test/postgresql_with_postgresql_registry/running_test.sh
  14. 3
      .github/workflows/cluster-test/postgresql_with_postgresql_registry/start-job.sh
  15. 3
      .github/workflows/cluster-test/postgresql_with_zookeeper_registry/Dockerfile
  16. 6
      .github/workflows/cluster-test/postgresql_with_zookeeper_registry/deploy.sh
  17. 5
      .github/workflows/cluster-test/postgresql_with_zookeeper_registry/running_test.sh
  18. 3
      .github/workflows/cluster-test/postgresql_with_zookeeper_registry/start-job.sh
  19. 7
      docs/docs/en/architecture/configuration.md
  20. 2
      docs/docs/en/faq.md
  21. 5
      docs/docs/en/guide/installation/cluster.md
  22. 68
      docs/docs/en/guide/installation/pseudo-cluster.md
  23. 2
      docs/docs/en/guide/installation/standalone.md
  24. 7
      docs/docs/zh/architecture/configuration.md
  25. 2
      docs/docs/zh/faq.md
  26. 5
      docs/docs/zh/guide/installation/cluster.md
  27. 59
      docs/docs/zh/guide/installation/pseudo-cluster.md
  28. 2
      docs/docs/zh/guide/installation/standalone.md
  29. 7
      dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/bin/start.sh
  30. 7
      dolphinscheduler-api/src/main/bin/start.sh
  31. 3
      dolphinscheduler-dist/src/main/assembly/dolphinscheduler-bin.xml
  32. 7
      dolphinscheduler-master/src/main/bin/start.sh
  33. 6
      dolphinscheduler-standalone-server/src/main/assembly/dolphinscheduler-standalone-server.xml
  34. 7
      dolphinscheduler-standalone-server/src/main/bin/start.sh
  35. 43
      dolphinscheduler-standalone-server/src/main/dist-bin/start.sh
  36. 9
      dolphinscheduler-worker/src/main/bin/start.sh
  37. 1
      script/dolphinscheduler-daemon.sh
  38. 50
      script/remove-zk-node.sh
  39. 62
      script/scp-hosts.sh
  40. 58
      script/start-all.sh
  41. 73
      script/status-all.sh
  42. 53
      script/stop-all.sh

3
.github/workflows/cluster-test/mysql_with_mysql_registry/Dockerfile

@ -27,9 +27,6 @@ RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinschedule
ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin
#Setting install.sh
COPY .github/workflows/cluster-test/mysql_with_mysql_registry/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh
#Setting dolphinscheduler_env.sh
COPY .github/workflows/cluster-test/mysql_with_mysql_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh

5
.github/workflows/cluster-test/mysql_with_mysql_registry/deploy.sh

@ -37,7 +37,10 @@ service ssh start
/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/upgrade-schema.sh
#Start Cluster
/bin/bash $DOLPHINSCHEDULER_HOME/bin/start-all.sh
/bin/bash $DOLPHINSCHEDULER_HOME/bin/dolphinscheduler-daemon.sh start master-server
/bin/bash $DOLPHINSCHEDULER_HOME/bin/dolphinscheduler-daemon.sh start worker-server
/bin/bash $DOLPHINSCHEDULER_HOME/bin/dolphinscheduler-daemon.sh start api-server
/bin/bash $DOLPHINSCHEDULER_HOME/bin/dolphinscheduler-daemon.sh start alert-server
#Keep running
tail -f /dev/null

2
.github/workflows/cluster-test/mysql_with_mysql_registry/dolphinscheduler_env.sh

@ -30,7 +30,7 @@ export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none}
export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC}
# Registry center configuration, determines the type and link of the registry center
export REGISTRY_TYPE=${REGISTRY_TYPE:-jdbc}
export REGISTRY_TYPE=jdbc
export REGISTRY_HIKARI_CONFIG_JDBC_URL="jdbc:mysql://mysql:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8&useSSL=false"
export REGISTRY_HIKARI_CONFIG_USERNAME=root
export REGISTRY_HIKARI_CONFIG_PASSWORD=123456

5
.github/workflows/cluster-test/mysql_with_mysql_registry/running_test.sh

@ -71,7 +71,10 @@ do
done
#Stop Cluster
docker exec -u root ds bash -c "/root/apache-dolphinscheduler-*-SNAPSHOT-bin/bin/stop-all.sh"
docker exec -u root ds bash -c "/root/apache-dolphinscheduler-*-SNAPSHOT-bin/bin/dolphinscheduler-daemon.sh stop master-server"
docker exec -u root ds bash -c "/root/apache-dolphinscheduler-*-SNAPSHOT-bin/bin/dolphinscheduler-daemon.sh stop worker-server"
docker exec -u root ds bash -c "/root/apache-dolphinscheduler-*-SNAPSHOT-bin/bin/dolphinscheduler-daemon.sh stop alert-server"
docker exec -u root ds bash -c "/root/apache-dolphinscheduler-*-SNAPSHOT-bin/bin/dolphinscheduler-daemon.sh stop api-server"
#Cluster stop health check
sleep 5

3
.github/workflows/cluster-test/mysql_with_mysql_registry/start-job.sh

@ -30,4 +30,5 @@ docker-compose -f .github/workflows/cluster-test/mysql_with_mysql_registry/docke
/bin/bash .github/workflows/cluster-test/mysql_with_mysql_registry/running_test.sh
#Cleanup
docker rm -f $(docker ps -aq)
docker-compose -f .github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-cluster.yaml down -v --remove-orphans
docker-compose -f .github/workflows/cluster-test/mysql_with_mysql_registry/docker-compose-base.yaml down -v --remove-orphans

3
.github/workflows/cluster-test/mysql_with_zookeeper_registry/Dockerfile

@ -27,9 +27,6 @@ RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinschedule
ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin
#Setting install.sh
COPY .github/workflows/cluster-test/mysql_with_zookeeper_registry/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh
#Setting dolphinscheduler_env.sh
COPY .github/workflows/cluster-test/mysql_with_zookeeper_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh

5
.github/workflows/cluster-test/mysql_with_zookeeper_registry/deploy.sh

@ -37,7 +37,10 @@ service ssh start
/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/upgrade-schema.sh
#Start Cluster
/bin/bash $DOLPHINSCHEDULER_HOME/bin/start-all.sh
/bin/bash $DOLPHINSCHEDULER_HOME/bin/dolphinscheduler-daemon.sh start master-server
/bin/bash $DOLPHINSCHEDULER_HOME/bin/dolphinscheduler-daemon.sh start worker-server
/bin/bash $DOLPHINSCHEDULER_HOME/bin/dolphinscheduler-daemon.sh start api-server
/bin/bash $DOLPHINSCHEDULER_HOME/bin/dolphinscheduler-daemon.sh start alert-server
#Keep running
tail -f /dev/null

61
.github/workflows/cluster-test/mysql_with_zookeeper_registry/install_env.sh

@ -1,61 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# ---------------------------------------------------------
# INSTALL MACHINE
# ---------------------------------------------------------
# A comma separated list of machine hostname or IP would be installed DolphinScheduler,
# including master, worker, api, alert. If you want to deploy in pseudo-distributed
# mode, just write a pseudo-distributed hostname
# Example for hostnames: ips="ds1,ds2,ds3,ds4,ds5", Example for IPs: ips="192.168.8.1,192.168.8.2,192.168.8.3,192.168.8.4,192.168.8.5"
ips=${ips:-"localhost"}
# Port of SSH protocol, default value is 22. For now we only support same port in all `ips` machine
# modify it if you use different ssh port
sshPort=${sshPort:-"22"}
# A comma separated list of machine hostname or IP would be installed Master server, it
# must be a subset of configuration `ips`.
# Example for hostnames: masters="ds1,ds2", Example for IPs: masters="192.168.8.1,192.168.8.2"
masters=${masters:-"localhost"}
# A comma separated list of machine <hostname>:<workerGroup> or <IP>:<workerGroup>.All hostname or IP must be a
# subset of configuration `ips`, And workerGroup have default value as `default`, but we recommend you declare behind the hosts
# Example for hostnames: workers="ds1:default,ds2:default,ds3:default", Example for IPs: workers="192.168.8.1:default,192.168.8.2:default,192.168.8.3:default"
workers=${workers:-"localhost:default"}
# A comma separated list of machine hostname or IP would be installed Alert server, it
# must be a subset of configuration `ips`.
# Example for hostname: alertServer="ds3", Example for IP: alertServer="192.168.8.3"
alertServer=${alertServer:-"localhost"}
# A comma separated list of machine hostname or IP would be installed API server, it
# must be a subset of configuration `ips`.
# Example for hostname: apiServers="ds1", Example for IP: apiServers="192.168.8.1"
apiServers=${apiServers:-"localhost"}
# The directory to install DolphinScheduler for all machine we config above. It will automatically be created by `install.sh` script if not exists.
# Do not set this configuration same as the current path (pwd)
installPath=${installPath:-"/root/apache-dolphinscheduler-*-SNAPSHOT-bin"}
# The user to deploy DolphinScheduler for all machine we config above. For now user must create by yourself before running `install.sh`
# script. The user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled than the root directory needs
# to be created by this user
deployUser=${deployUser:-"dolphinscheduler"}
# The root of zookeeper, for now DolphinScheduler default registry server is zookeeper.
zkRoot=${zkRoot:-"/dolphinscheduler"}

5
.github/workflows/cluster-test/mysql_with_zookeeper_registry/running_test.sh

@ -71,7 +71,10 @@ do
done
#Stop Cluster
docker exec -u root ds bash -c "/root/apache-dolphinscheduler-*-SNAPSHOT-bin/bin/stop-all.sh"
docker exec -u root ds bash -c "/root/apache-dolphinscheduler-*-SNAPSHOT-bin/bin/dolphinscheduler-daemon.sh stop master-server"
docker exec -u root ds bash -c "/root/apache-dolphinscheduler-*-SNAPSHOT-bin/bin/dolphinscheduler-daemon.sh stop worker-server"
docker exec -u root ds bash -c "/root/apache-dolphinscheduler-*-SNAPSHOT-bin/bin/dolphinscheduler-daemon.sh stop alert-server"
docker exec -u root ds bash -c "/root/apache-dolphinscheduler-*-SNAPSHOT-bin/bin/dolphinscheduler-daemon.sh stop api-server"
#Cluster stop health check
sleep 5

3
.github/workflows/cluster-test/mysql_with_zookeeper_registry/start-job.sh

@ -30,4 +30,5 @@ docker-compose -f .github/workflows/cluster-test/mysql_with_zookeeper_registry/d
/bin/bash .github/workflows/cluster-test/mysql_with_zookeeper_registry/running_test.sh
#Cleanup
docker rm -f $(docker ps -aq)
docker-compose -f .github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-cluster.yaml down -v --remove-orphans
docker-compose -f .github/workflows/cluster-test/mysql_with_zookeeper_registry/docker-compose-base.yaml down -v --remove-orphans

3
.github/workflows/cluster-test/postgresql_with_postgresql_registry/Dockerfile

@ -27,9 +27,6 @@ RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinschedule
ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin
#Setting install.sh
COPY .github/workflows/cluster-test/postgresql_with_postgresql_registry/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh
#Setting dolphinscheduler_env.sh
COPY .github/workflows/cluster-test/postgresql_with_postgresql_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh

5
.github/workflows/cluster-test/postgresql_with_postgresql_registry/deploy.sh

@ -34,7 +34,10 @@ service ssh start
/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/upgrade-schema.sh
#Start Cluster
/bin/bash $DOLPHINSCHEDULER_HOME/bin/start-all.sh
/bin/bash $DOLPHINSCHEDULER_HOME/bin/dolphinscheduler-daemon.sh start master-server
/bin/bash $DOLPHINSCHEDULER_HOME/bin/dolphinscheduler-daemon.sh start worker-server
/bin/bash $DOLPHINSCHEDULER_HOME/bin/dolphinscheduler-daemon.sh start api-server
/bin/bash $DOLPHINSCHEDULER_HOME/bin/dolphinscheduler-daemon.sh start alert-server
#Keep running
tail -f /dev/null

5
.github/workflows/cluster-test/postgresql_with_postgresql_registry/running_test.sh

@ -71,7 +71,10 @@ do
done
#Stop Cluster
docker exec -u root ds bash -c "/root/apache-dolphinscheduler-*-SNAPSHOT-bin/bin/stop-all.sh"
docker exec -u root ds bash -c "/root/apache-dolphinscheduler-*-SNAPSHOT-bin/bin/dolphinscheduler-daemon.sh stop master-server"
docker exec -u root ds bash -c "/root/apache-dolphinscheduler-*-SNAPSHOT-bin/bin/dolphinscheduler-daemon.sh stop worker-server"
docker exec -u root ds bash -c "/root/apache-dolphinscheduler-*-SNAPSHOT-bin/bin/dolphinscheduler-daemon.sh stop alert-server"
docker exec -u root ds bash -c "/root/apache-dolphinscheduler-*-SNAPSHOT-bin/bin/dolphinscheduler-daemon.sh stop api-server"
#Cluster stop health check
sleep 5

3
.github/workflows/cluster-test/postgresql_with_postgresql_registry/start-job.sh

@ -30,4 +30,5 @@ docker-compose -f .github/workflows/cluster-test/postgresql_with_postgresql_regi
/bin/bash .github/workflows/cluster-test/postgresql_with_postgresql_registry/running_test.sh
#Cleanup
docker rm -f $(docker ps -aq)
docker-compose -f .github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-cluster.yaml down -v --remove-orphans
docker-compose -f .github/workflows/cluster-test/postgresql_with_postgresql_registry/docker-compose-base.yaml down -v --remove-orphans

3
.github/workflows/cluster-test/postgresql_with_zookeeper_registry/Dockerfile

@ -27,9 +27,6 @@ RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinschedule
ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin
#Setting install.sh
COPY .github/workflows/cluster-test/postgresql_with_zookeeper_registry/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh
#Setting dolphinscheduler_env.sh
COPY .github/workflows/cluster-test/postgresql_with_zookeeper_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh

6
.github/workflows/cluster-test/postgresql_with_zookeeper_registry/deploy.sh

@ -34,7 +34,11 @@ service ssh start
/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/upgrade-schema.sh
#Start Cluster
/bin/bash $DOLPHINSCHEDULER_HOME/bin/start-all.sh
/bin/bash $DOLPHINSCHEDULER_HOME/bin/dolphinscheduler-daemon.sh start master-server
/bin/bash $DOLPHINSCHEDULER_HOME/bin/dolphinscheduler-daemon.sh start worker-server
/bin/bash $DOLPHINSCHEDULER_HOME/bin/dolphinscheduler-daemon.sh start api-server
/bin/bash $DOLPHINSCHEDULER_HOME/bin/dolphinscheduler-daemon.sh start alert-server
#Keep running
tail -f /dev/null

5
.github/workflows/cluster-test/postgresql_with_zookeeper_registry/running_test.sh

@ -71,7 +71,10 @@ do
done
#Stop Cluster
docker exec -u root ds bash -c "/root/apache-dolphinscheduler-*-SNAPSHOT-bin/bin/stop-all.sh"
docker exec -u root ds bash -c "/root/apache-dolphinscheduler-*-SNAPSHOT-bin/bin/dolphinscheduler-daemon.sh stop master-server"
docker exec -u root ds bash -c "/root/apache-dolphinscheduler-*-SNAPSHOT-bin/bin/dolphinscheduler-daemon.sh stop worker-server"
docker exec -u root ds bash -c "/root/apache-dolphinscheduler-*-SNAPSHOT-bin/bin/dolphinscheduler-daemon.sh stop alert-server"
docker exec -u root ds bash -c "/root/apache-dolphinscheduler-*-SNAPSHOT-bin/bin/dolphinscheduler-daemon.sh stop api-server"
#Cluster stop health check
sleep 5

3
.github/workflows/cluster-test/postgresql_with_zookeeper_registry/start-job.sh

@ -30,4 +30,5 @@ docker-compose -f .github/workflows/cluster-test/postgresql_with_zookeeper_regis
/bin/bash .github/workflows/cluster-test/postgresql_with_zookeeper_registry/running_test.sh
#Cleanup
docker rm -f $(docker ps -aq)
docker-compose -f .github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-cluster.yaml down -v --remove-orphans
docker-compose -f .github/workflows/cluster-test/postgresql_with_zookeeper_registry/docker-compose-base.yaml down -v --remove-orphans

7
docs/docs/en/architecture/configuration.md

@ -21,13 +21,6 @@ The directory structure of DolphinScheduler is as follows:
   ├── dolphinscheduler-daemon.sh script to start or shut down DolphinScheduler application
   ├── env directory of scripts to load environment variables
     ├── dolphinscheduler_env.sh script to export environment variables [eg: JAVA_HOME,HADOOP_HOME, HIVE_HOME ...] when you start or stop service using script `dolphinscheduler-daemon.sh`
     └── install_env.sh script to export environment variables for DolphinScheduler installation when you use scripts `install.sh` `start-all.sh` `stop-all.sh` `status-all.sh`
   ├── install.sh script to auto-setup services when you deploy DolphinScheduler in `psuedo-cluster` mode or `cluster` mode
   ├── remove-zk-node.sh script to cleanup ZooKeeper caches
   ├── scp-hosts.sh script to copy installation files to target hosts
   ├── start-all.sh script to start all services when you deploy DolphinScheduler in `psuedo-cluster` mode or `cluster` mode
   ├── status-all.sh script to check the status of all services when you deploy DolphinScheduler in `psuedo-cluster` mode or `cluster` mode
   └── stop-all.sh script to shut down all services when you deploy DolphinScheduler in `psuedo-cluster` mode or `cluster` mode
├── alert-server directory of DolphinScheduler alert-server commands, configurations scripts and libs
   ├── bin

2
docs/docs/en/faq.md

@ -506,7 +506,7 @@ A: 1, Where is the executed server? Specify one worker to run the task, you can
## Q : What’s the best deploy mode you suggest in production env
A: 1, I suggest you use 3 nodes for stability if you don't have too many tasks to run. And deploy Master/Worker server on different nodes is better. If you only have one node, you of course only can deploy them together! By the way, how many machines you need is determined by your business. The DolphinScheduler system itself does not use too many resources. Test more, and you'll find the right way to use a few machines.
A: 1, I suggest you use 3 nodes for stability if you don't have too many tasks to run. And deploy Master/Worker/Api server on different nodes is better. If you only have one node, you of course only can deploy them together! By the way, how many machines you need is determined by your business. The DolphinScheduler system itself does not use too many resources. Test more, and you'll find the right way to use a few machines.
---

5
docs/docs/en/guide/installation/cluster.md

@ -10,7 +10,10 @@ Cluster deployment uses the same scripts and configuration files as [pseudo-clus
### Prerequisites and DolphinScheduler Startup Environment Preparations
Configure all the configurations refer to [pseudo-cluster deployment](pseudo-cluster.md) on every machine, except sections `Prerequisites`, `Start ZooKeeper` and `Initialize the Database` of the `DolphinScheduler Startup Environment`.
Distribute the installation package to each server of each cluster and perform all the steps in [pseudo-cluster deployment](pseudo-cluster.md) on each machine.
> **_NOTICE:_** Make sure that the configuration files on each machine are consistent, otherwise the cluster will not work properly.
> **_NOTICE:_** Each service is stateless and independent of each other, so you can deploy multiple services on each machine, but you need to pay attention to port conflicts.
### Modify Configuration

68
docs/docs/en/guide/installation/pseudo-cluster.md

@ -8,10 +8,10 @@ If you are a new hand and want to experience DolphinScheduler functions, we reco
Pseudo-cluster deployment of DolphinScheduler requires external software support:
- JDK:download [JDK][jdk] (1.8+), install and configure environment variable `JAVA_HOME` and append `bin` dir (included in `JAVA_HOME`) to `PATH` variable. You can skip this step if it already exists in your environment.
- JDK:download [JDK][jdk] (1.8 or 11), install and configure environment variable `JAVA_HOME` and append `bin` dir (included in `JAVA_HOME`) to `PATH` variable. You can skip this step if it already exists in your environment.
- Binary package: Download the DolphinScheduler binary package at [download page](https://dolphinscheduler.apache.org/en-us/download)
- Database: [PostgreSQL](https://www.postgresql.org/download/) (8.2.15+) or [MySQL](https://dev.mysql.com/downloads/mysql/) (5.7+), you can choose one of the two, such as MySQL requires JDBC Driver 8.0.16
- Registry Center: [ZooKeeper](https://zookeeper.apache.org/releases.html) (3.8.0+),[download link][zookeeper]
- Database: [PostgreSQL](https://www.postgresql.org/download/) (8.2.15+) or [MySQL](https://dev.mysql.com/downloads/mysql/) (5.7+), you can choose one of the two, such as MySQL requires JDBC Driver 8.0.33
- Registry Center: [ZooKeeper](https://zookeeper.apache.org/releases.html) (3.8.0+), [MYSQL](https://www.mysql.com/)(8.0.33), [ETCD](https://etcd.io/)
- Process tree analysis
- `pstree` for macOS
- `psmisc` for Fedora/Red/Hat/CentOS/Ubuntu/Debian
@ -42,32 +42,9 @@ chmod -R 755 apache-dolphinscheduler-*-bin
> **_NOTICE:_**
>
> - Due to DolphinScheduler's multi-tenant task switch user using command `sudo -u {linux-user}`, the deployment user needs to have `sudo` privileges and be password-free. If novice learners don’t understand, you can ignore this point for now.
> - Due to DolphinScheduler's multi-tenant task switch user using command `sudo -u {linux-user} -i`, the deployment user needs to have `sudo` privileges and be password-free. If novice learners don’t understand, you can ignore this point for now.
> - If you find the line "Defaults requirett" in the `/etc/sudoers` file, please comment the content.
### Configure Machine SSH Password-Free Login
Since resources need to be sent to different machines during installation, SSH password-free login is required between each machine. The following shows the steps to configure password-free login:
```shell
su dolphinscheduler
ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
chmod 600 ~/.ssh/authorized_keys
```
> **_Notice:_** After the configuration is complete, you can run the command `ssh localhost` to test works or not. If you can login with ssh without password stands for successful.
### Start ZooKeeper
Go to the ZooKeeper installation directory, copy configure file `zoo_sample.cfg` to `conf/zoo.cfg`, and change value of dataDir in `conf/zoo.cfg` to `dataDir=./tmp/zookeeper`.
```shell
# Start ZooKeeper
./bin/zkServer.sh start
```
## Modify Configuration
After completing the preparation of the basic environment, you need to modify the configuration file according to the
@ -75,11 +52,12 @@ environment you used. Change the environment configurations via `export <ENV_NAM
### Modify `dolphinscheduler_env.sh`
File `./bin/env/dolphinscheduler_env.sh` describes the following configurations:
File `./bin/env/dolphinscheduler_env.sh`,`api-server/conf/application.yaml`,
`master-server/conf/application.yaml`,`worker-server/conf/application.yaml`,`alert-server/conf/application.yaml` describes the following configurations:
- Database configuration of DolphinScheduler, see [Initialize the Database](#initialize-the-database) for detailed instructions.
- Some tasks which need external dependencies or libraries such as `JAVA_HOME` and `SPARK_HOME`.
- Registry center `zookeeper`.
- Default registry center is mysql.
- Server related configuration, such as cache type, timezone, etc.
You could ignore the task external dependencies if you do not use those tasks, but you have to change `JAVA_HOME`, registry center and database
@ -116,16 +94,35 @@ export DATAX_LAUNCHER=${DATAX_LAUNCHER:-/opt/soft/datax/bin/datax.py}
export PATH=$HADOOP_HOME/bin:$SPARK_HOME/bin:$PYTHON_LAUNCHER:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_LAUNCHER:$PATH
```
> **_Note:_** If you use MySQL database, set `DATABASE` to `mysql`, and modify `SPRING_DATASOURCE_URL`, `SPRING_DATASOURCE_USERNAME` and `SPRING_DATASOURCE_PASSWORD` to your database configuration.
>
> **_Note:_** The configuration in `dolphinscheduler_env.sh` will overwrite the configuration in each service's configuration file (application.yaml), so if you configure a parameter in the application.yaml file and also configure it in `dolphinscheduler_env.sh`,
> the configuration in `dolphinscheduler_env.sh` will take precedence. The configuration format in `dolphinscheduler_env.sh` is as follows:
> `SPRING_DATASOURCE_URL` in `application.yaml` is `spring.datasource.url`, and so on.
## Initialize the Database
Follow the instructions in [datasource-setting](../howto/datasource-setting.md) `Pseudo-Cluster/Cluster Initialize the Database` section to create and initialize database
## Start DolphinScheduler
Use **deployment user** you created above, running the command to complete the deployment, and the server log will be stored in the logs folder.
The server log will be stored in the `xxx-server/logs` folder.
```
# Start api-server
bash ./bin/dolphinscheduler-daemon.sh start api-server
# Start master-server
bash ./bin/dolphinscheduler-daemon.sh start master-server
# Start worker-server
bash ./bin/dolphinscheduler-daemon.sh start worker-server
# Start alert-server
bash ./bin/dolphinscheduler-daemon.sh start alert-server
```
> **_Note:_** For the first time deployment, there maybe occur five times of `sh: bin/dolphinscheduler-daemon.sh: No such file or directory` in the terminal,
> this is non-important information that you can ignore.
> **_Note:_** For the first time deployment, you can check the status of server through bash ./bin/dolphinscheduler-daemon.sh status xxx-server
## Login DolphinScheduler
@ -134,11 +131,8 @@ Access address `http://localhost:12345/dolphinscheduler/ui` and login DolphinSch
## Start or Stop Server
```shell
# Stop all DolphinScheduler server
bash ./bin/stop-all.sh
# Start all DolphinScheduler server
bash ./bin/start-all.sh
# Check the status of DolphinScheduler server
bash ./bin/dolphinscheduler-daemon.sh status xxx-server
# Start or stop DolphinScheduler Master
bash ./bin/dolphinscheduler-daemon.sh stop master-server

2
docs/docs/en/guide/installation/standalone.md

@ -11,7 +11,7 @@ If you want to deploy DolphinScheduler in production, we recommend you follow [c
## Preparation
- JDK:download [JDK][jdk] (1.8+), install and configure environment variable `JAVA_HOME` and append `bin` dir (included in `JAVA_HOME`) to `PATH` variable. You can skip this step if it already exists in your environment.
- JDK:download [JDK][jdk] (1.8 or 11), install and configure environment variable `JAVA_HOME` and append `bin` dir (included in `JAVA_HOME`) to `PATH` variable. You can skip this step if it already exists in your environment.
- Binary package: download the DolphinScheduler binary package at [download page](https://dolphinscheduler.apache.org/en-us/download/<version>). <!-- markdown-link-check-disable-line -->
## Start DolphinScheduler Standalone Server

7
docs/docs/zh/architecture/configuration.md

@ -19,13 +19,6 @@ DolphinScheduler的目录结构如下:
   ├── dolphinscheduler-daemon.sh 启动/关闭DolphinScheduler服务脚本
   ├── env 环境变量配置存放目录
     ├── dolphinscheduler_env.sh 当使用`dolphinscheduler-daemon.sh`脚本起停服务时,运行此脚本加载环境变量配置文件 [如:JAVA_HOME,HADOOP_HOME, HIVE_HOME ...]
     └── install_env.sh 当使用`install.sh` `start-all.sh` `stop-all.sh` `status-all.sh`脚本时,运行此脚本为DolphinScheduler安装加载环境变量配置
   ├── install.sh 当使用`集群`模式或`伪集群`模式部署DolphinScheduler时,运行此脚本自动安装服务
   ├── remove-zk-node.sh 清理zookeeper缓存文件脚本
   ├── scp-hosts.sh 安装文件传输脚本
   ├── start-all.sh 当使用`集群`模式或`伪集群`模式部署DolphinScheduler时,运行此脚本启动所有服务
   ├── status-all.sh 当使用`集群`模式或`伪集群`模式部署DolphinScheduler时,运行此脚本获取所有服务状态
   └── stop-all.sh 当使用`集群`模式或`伪集群`模式部署DolphinScheduler时,运行此脚本终止所有服务
├── alert-server DolphinScheduler alert-server命令、配置和依赖存放目录
   ├── bin

2
docs/docs/zh/faq.md

@ -477,7 +477,7 @@ A:1,被执行的服务器在哪里配置,以及实际执行的服务器是
## Q:生产环境部署方式有推荐的最佳实践吗
A:1,如果没有很多任务要运行,出于稳定性考虑我们建议使用 3 个节点,并且最好把 Master/Worder 服务部署在不同的节点。如果你只有一个节点,当然只能把所有的服务部署在同一个节点!通常来说,需要多少节点取决于你的业务,海豚调度系统本身不需要很多的资源。充分测试之后,你们将找到使用较少节点的合适的部署方式。
A:1,如果没有很多任务要运行,出于稳定性考虑我们建议使用 3 个节点,并且最好把 Master/Worker/Api 服务部署在不同的节点。如果你只有一个节点,当然只能把所有的服务部署在同一个节点!通常来说,需要多少节点取决于你的业务,海豚调度系统本身不需要很多的资源。充分测试之后,你们将找到使用较少节点的合适的部署方式。
---

5
docs/docs/zh/guide/installation/cluster.md

@ -10,7 +10,10 @@
### 前置准备工作 && 准备 DolphinScheduler 启动环境
其中除了[伪集群部署](pseudo-cluster.md)中的“前置准备工作”,“准备启动环境”除了“启动zookeeper”以及“初始化数据库”外,别的都需要在每台机器中进行配置
需要将安装包分发至每台集群的每台服务器上,并且需要在每台机器中进行配置执行[伪集群部署](pseudo-cluster.md)中的所有执行项
> **_注意:_** 请确保每台机器的配置文件都是一致的,否则会导致集群无法正常工作
> **_注意:_** 每个服务都是无状态且互相独立的,所以可以在每台机器上部署多个服务,但是需要注意端口冲突问题
### 修改相关配置

59
docs/docs/zh/guide/installation/pseudo-cluster.md

@ -8,10 +8,10 @@
伪分布式部署 DolphinScheduler 需要有外部软件的支持
- JDK:下载[JDK][jdk] (1.8+),安装并配置 `JAVA_HOME` 环境变量,并将其下的 `bin` 目录追加到 `PATH` 环境变量中。如果你的环境中已存在,可以跳过这步。
- JDK:下载[JDK][jdk] (1.8 或者 11),安装并配置 `JAVA_HOME` 环境变量,并将其下的 `bin` 目录追加到 `PATH` 环境变量中。如果你的环境中已存在,可以跳过这步。
- 二进制包:在[下载页面](https://dolphinscheduler.apache.org/zh-cn/download)下载 DolphinScheduler 二进制包
- 数据库:[PostgreSQL](https://www.postgresql.org/download/) (8.2.15+) 或者 [MySQL](https://dev.mysql.com/downloads/mysql/) (5.7+),两者任选其一即可,如 MySQL 则需要 JDBC Driver 8.0.16
- 注册中心:[ZooKeeper](https://zookeeper.apache.org/releases.html) (3.8.0+),[下载地址][zookeeper]
- 数据库:[PostgreSQL](https://www.postgresql.org/download/) (8.2.15+) 或者 [MySQL](https://dev.mysql.com/downloads/mysql/) (5.7+),两者任选其一即可,如 MySQL 则需要 JDBC Driver 8.0.33
- 注册中心:当前支持 [ZooKeeper](https://zookeeper.apache.org/releases.html) (3.8.0),[MYSQL](https://www.mysql.com/)(8.0.33),[ETCD](https://etcd.io/)
- 进程树分析
- macOS 安装`pstree`
- Fedora/Red/Hat/CentOS/Ubuntu/Debian 安装`psmisc`
@ -42,23 +42,9 @@ chmod -R 755 apache-dolphinscheduler-*-bin
> **_注意:_**
>
> - 因为任务执行服务是以 `sudo -u {linux-user}` 切换不同 linux 用户的方式来实现多租户运行作业,所以部署用户需要有 sudo 权限,而且是免密的。初学习者不理解的话,完全可以暂时忽略这一点
> - 因为任务执行服务是以 `sudo -u {linux-user} -i` 切换不同 linux 用户的方式来实现多租户运行作业,所以部署用户需要有 sudo 权限,而且是免密的。初学习者不理解的话,完全可以暂时忽略这一点
> - 如果发现 `/etc/sudoers` 文件中有 "Defaults requirett" 这行,也请注释掉
### 配置机器 SSH 免密登陆
由于安装的时候需要向不同机器发送资源,所以要求各台机器间能实现 SSH 免密登陆。配置免密登陆的步骤如下
```shell
su dolphinscheduler
ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
chmod 600 ~/.ssh/authorized_keys
```
> **_注意:_** 配置完成后,可以通过运行命令 `ssh localhost` 判断是否成功,如果不需要输入密码就能 ssh 登陆则证明成功
### 启动 zookeeper
进入 zookeeper 的安装目录,将 `zoo_sample.cfg` 配置文件复制到 `conf/zoo.cfg`,并将 `conf/zoo.cfg` 中 dataDir 中的值改成 `dataDir=./tmp/zookeeper`
@ -70,7 +56,8 @@ chmod 600 ~/.ssh/authorized_keys
## 修改相关配置
完成基础环境的准备后,需要根据你的机器环境修改配置文件。配置文件可以在目录 `bin/env/dolphinscheduler_env.sh` 中找到。
完成基础环境的准备后,需要根据你的机器环境修改配置文件。配置文件可以在目录 `bin/env/dolphinscheduler_env.sh`,`api-server/conf/application.yaml`,
`master-server/conf/application.yaml`,`worker-server/conf/application.yaml`,`alert-server/conf/application.yaml` 中找到。
### 修改 `dolphinscheduler_env.sh` 文件
@ -78,6 +65,8 @@ chmod 600 ~/.ssh/authorized_keys
- DolphinScheduler 的数据库配置,详细配置方法见[初始化数据库]
- 一些任务类型外部依赖路径或库文件,如 `JAVA_HOME``SPARK_HOME`都是在这里定义的
- 默认的注册中心是 mysql
- 服务器相关配置,如缓存类型、时区等
如果您不使用某些任务类型,您可以忽略任务外部依赖项,但您必须根据您的环境更改 `JAVA_HOME`、注册中心和数据库相关配置。
@ -112,15 +101,36 @@ export DATAX_LAUNCHER=${DATAX_LAUNCHER:-/opt/soft/datax/bin/python3}
export PATH=$HADOOP_HOME/bin:$SPARK_HOME/bin:$PYTHON_LAUNCHER:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_LAUNCHER:$PATH
```
> **_注意:_** 如果您使用的是 MySQL 数据库,需要将 `DATABASE` 设置为 `mysql`,并且修改 `SPRING_DATASOURCE_URL`、`SPRING_DATASOURCE_USERNAME` 和 `SPRING_DATASOURCE_PASSWORD` 为您的数据库配置
>
> **_注意:_** dolphinscheduler_env.sh 文件中的配置会覆盖各个服务的配置文件(application.yaml)中的配置,所以如果您在配置文件中配置了某个参数,
> 而且在 dolphinscheduler_env.sh 文件中也配置了,那么以 dolphinscheduler_env.sh 文件中的配置为准。dolphinscheduler_env.sh 里的配置项格式样例:
> SPRING_DATASOURCE_URL 在 application.yaml 为 spring.datasource.url,以此类推
## 初始化数据库
请参考 [数据源配置] `伪分布式/分布式安装初始化数据库` 创建并初始化数据库
## 启动 DolphinScheduler
使用上面创建的**部署用户**运行命令完成部署,部署后的运行日志将存放在 logs 文件夹内
部署后的运行日志将存放在 `xxx-server/logs` 文件夹内
> **_注意:_** 第一次部署的话,可能出现 5 次`sh: bin/dolphinscheduler-daemon.sh: No such file or directory`相关信息,此为非重要信息直接忽略即可
```
# 启动 api-server
bash ./bin/dolphinscheduler-daemon.sh start api-server
# 启动 master-server
bash ./bin/dolphinscheduler-daemon.sh start master-server
# 启动 worker-server
bash ./bin/dolphinscheduler-daemon.sh start worker-server
# 启动 alert-server
bash ./bin/dolphinscheduler-daemon.sh start alert-server
```
> **_注意:_** 第一次部署的话,可以通过 bash ./bin/dolphinscheduler-daemon.sh status xxx-server 来进行服务状态查询
## 登录 DolphinScheduler
@ -129,11 +139,8 @@ export PATH=$HADOOP_HOME/bin:$SPARK_HOME/bin:$PYTHON_LAUNCHER:$JAVA_HOME/bin:$HI
## 启停服务
```shell
# 一键停止集群所有服务
bash ./bin/stop-all.sh
# 一键开启集群所有服务
bash ./bin/start-all.sh
# 查询服务状态
bash ./bin/dolphinscheduler-daemon.sh status xxx-server
# 启停 Master
bash ./bin/dolphinscheduler-daemon.sh stop master-server

2
docs/docs/zh/guide/installation/standalone.md

@ -9,7 +9,7 @@ Standalone 仅适用于 DolphinScheduler 的快速体验.
## 前置准备工作
- JDK:下载[JDK][jdk] (1.8+),安装并配置 `JAVA_HOME` 环境变量,并将其下的 `bin` 目录追加到 `PATH` 环境变量中。如果你的环境中已存在,可以跳过这步。
- JDK:下载[JDK][jdk] (1.8 or 11),安装并配置 `JAVA_HOME` 环境变量,并将其下的 `bin` 目录追加到 `PATH` 环境变量中。如果你的环境中已存在,可以跳过这步。
- 二进制包:在[下载页面](https://dolphinscheduler.apache.org/en-us/download/<version>)下载 DolphinScheduler 二进制包 <!-- markdown-link-check-disable-line -->
## 启动 DolphinScheduler Standalone Server

7
dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/bin/start.sh

@ -15,13 +15,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
set -eo pipefail
BIN_DIR=$(dirname $0)
DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/..; pwd)}
BIN_DIR=$(dirname $(readlink -f "$0"))
DOLPHINSCHEDULER_HOME=$(cd ${BIN_DIR}/..;pwd)
source "$DOLPHINSCHEDULER_HOME/conf/dolphinscheduler_env.sh"
JVM_ARGS_ENV_FILE=${BIN_DIR}/jvm_args_env.sh
JVM_ARGS_ENV_FILE=${BIN_DIR}/bin/jvm_args_env.sh
JVM_ARGS="-server"
if [ -f $JVM_ARGS_ENV_FILE ]; then

7
dolphinscheduler-api/src/main/bin/start.sh

@ -15,13 +15,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
set -eo pipefail
BIN_DIR=$(dirname $0)
DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/..; pwd)}
BIN_DIR=$(dirname $(readlink -f "$0"))
DOLPHINSCHEDULER_HOME=$(cd ${BIN_DIR}/..;pwd)
source "$DOLPHINSCHEDULER_HOME/conf/dolphinscheduler_env.sh"
JVM_ARGS_ENV_FILE=${BIN_DIR}/jvm_args_env.sh
JVM_ARGS_ENV_FILE=${DOLPHINSCHEDULER_HOME}/bin/jvm_args_env.sh
JVM_ARGS="-server"
if [ -f $JVM_ARGS_ENV_FILE ]; then

3
dolphinscheduler-dist/src/main/assembly/dolphinscheduler-bin.xml vendored

@ -52,7 +52,6 @@
<excludes>
<exclude>libs/</exclude>
<exclude>bin/</exclude>
<exclude>dist-bin/</exclude>
</excludes>
</fileSet>
<fileSet>
@ -60,7 +59,7 @@
<outputDirectory>standalone-server/libs/standalone-server</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/../dolphinscheduler-standalone-server/target/standalone-server/dist-bin</directory>
<directory>${basedir}/../dolphinscheduler-standalone-server/target/standalone-server/bin</directory>
<outputDirectory>standalone-server/bin</outputDirectory>
</fileSet>

7
dolphinscheduler-master/src/main/bin/start.sh

@ -15,13 +15,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
set -eo pipefail
BIN_DIR=$(dirname $0)
DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/..; pwd)}
BIN_DIR=$(dirname $(readlink -f "$0"))
DOLPHINSCHEDULER_HOME=$(cd ${BIN_DIR}/..;pwd)
source "$DOLPHINSCHEDULER_HOME/conf/dolphinscheduler_env.sh"
JVM_ARGS_ENV_FILE=${BIN_DIR}/jvm_args_env.sh
JVM_ARGS_ENV_FILE=${DOLPHINSCHEDULER_HOME}/bin/jvm_args_env.sh
JVM_ARGS="-server"
if [ -f $JVM_ARGS_ENV_FILE ]; then

6
dolphinscheduler-standalone-server/src/main/assembly/dolphinscheduler-standalone-server.xml

@ -57,12 +57,6 @@
<fileMode>0755</fileMode>
<directoryMode>0755</directoryMode>
</fileSet>
<fileSet>
<directory>${basedir}/src/main/dist-bin</directory>
<outputDirectory>dist-bin</outputDirectory>
<fileMode>0755</fileMode>
<directoryMode>0755</directoryMode>
</fileSet>
<fileSet>
<directory>${basedir}/../script/env</directory>
<outputDirectory>conf</outputDirectory>

7
dolphinscheduler-standalone-server/src/main/bin/start.sh

@ -15,14 +15,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
set -eo pipefail
BIN_DIR=$(dirname $0)
DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/..; pwd)}
BIN_DIR=$(dirname $(readlink -f "$0"))
DOLPHINSCHEDULER_HOME=$(cd ${BIN_DIR}/..;pwd)
export DATABASE=${DATABASE:-h2}
source "$DOLPHINSCHEDULER_HOME/conf/dolphinscheduler_env.sh"
JVM_ARGS_ENV_FILE=${BIN_DIR}/jvm_args_env.sh
JVM_ARGS_ENV_FILE=${DOLPHINSCHEDULER_HOME}/bin/jvm_args_env.sh
JVM_ARGS="-server"
if [ -f $JVM_ARGS_ENV_FILE ]; then

43
dolphinscheduler-standalone-server/src/main/dist-bin/start.sh

@ -1,43 +0,0 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
BIN_DIR=$(dirname $0)
DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/..; pwd)}
export DATABASE=${DATABASE:-h2}
source "$DOLPHINSCHEDULER_HOME/conf/dolphinscheduler_env.sh"
JAVA_OPTS=${JAVA_OPTS:-"-server -Duser.timezone=${SPRING_JACKSON_TIME_ZONE} -Xms1g -Xmx1g -Xmn512m -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"}
if [[ "$DOCKER" == "true" ]]; then
JAVA_OPTS="${JAVA_OPTS} -XX:-UseContainerSupport"
fi
CP=$DOLPHINSCHEDULER_HOME/libs/standalone-server/*
for d in alert-server api-server master-server worker-server; do
for f in $DOLPHINSCHEDULER_HOME/../$d/libs/*.jar; do
JAR_FILE_NAME=${f##*/}
if [[ ! $CP =~ $JAR_FILE_NAME ]];then
CP=$CP:$f
fi
done
done
$JAVA_HOME/bin/java $JAVA_OPTS \
-cp "$DOLPHINSCHEDULER_HOME/conf":"$CP" \
org.apache.dolphinscheduler.StandaloneServer

9
dolphinscheduler-worker/src/main/bin/start.sh

@ -15,15 +15,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
set -eo pipefail
BIN_DIR=$(dirname $0)
DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/..; pwd)}
BIN_DIR=$(dirname $(readlink -f "$0"))
DOLPHINSCHEDULER_HOME=$(cd ${BIN_DIR}/..;pwd)
source "$DOLPHINSCHEDULER_HOME/conf/dolphinscheduler_env.sh"
export DOLPHINSCHEDULER_WORK_HOME=${DOLPHINSCHEDULER_HOME}
JVM_ARGS_ENV_FILE=${BIN_DIR}/jvm_args_env.sh
JVM_ARGS_ENV_FILE=${DOLPHINSCHEDULER_HOME}/bin/jvm_args_env.sh
JVM_ARGS="-server"
if [ -f $JVM_ARGS_ENV_FILE ]; then

1
script/dolphinscheduler-daemon.sh

@ -15,6 +15,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
set -eo pipefail
usage="Usage: dolphinscheduler-daemon.sh (start|stop|status) <api-server|master-server|worker-server|alert-server|standalone-server> "

50
script/remove-zk-node.sh

@ -1,50 +0,0 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
print_usage(){
printf $"USAGE:$0 rootNode\n"
exit 1
}
if [ $# -ne 1 ];then
print_usage
fi
rootNode=$1
BIN_DIR=`dirname $0`
BIN_DIR=`cd "$BIN_DIR"; pwd`
DOLPHINSCHEDULER_HOME=$BIN_DIR/..
source ${BIN_DIR}/env/install_env.sh
source ${BIN_DIR}/env/dolphinscheduler_env.sh
export JAVA_HOME=$JAVA_HOME
export DOLPHINSCHEDULER_CONF_DIR=$DOLPHINSCHEDULER_HOME/conf
export DOLPHINSCHEDULER_LIB_JARS=$DOLPHINSCHEDULER_HOME/api-server/libs/*
export DOLPHINSCHEDULER_OPTS="-Xmx1g -Xms1g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 "
export STOP_TIMEOUT=5
CLASS=org.apache.zookeeper.ZooKeeperMain
exec_command="$DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS -server $REGISTRY_ZOOKEEPER_CONNECT_STRING deleteall $rootNode"
cd $DOLPHINSCHEDULER_HOME
$JAVA_HOME/bin/java $exec_command

62
script/scp-hosts.sh

@ -1,62 +0,0 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
workDir=`dirname $0`
workDir=`cd ${workDir};pwd`
source ${workDir}/env/install_env.sh
workersGroup=(${workers//,/ })
for workerGroup in ${workersGroup[@]}
do
echo $workerGroup;
worker=`echo $workerGroup|awk -F':' '{print $1}'`
group=`echo $workerGroup|awk -F':' '{print $2}'`
workerNames+=($worker)
groupNames+=(${group:-default})
done
hostsArr=(${ips//,/ })
for host in ${hostsArr[@]}
do
if ! ssh -o StrictHostKeyChecking=no -p $sshPort $host test -e $installPath; then
ssh -o StrictHostKeyChecking=no -p $sshPort $host "sudo mkdir -p $installPath; sudo chown -R $deployUser:$deployUser $installPath"
fi
echo "scp dirs to $host/$installPath starting"
for i in ${!workerNames[@]}; do
if [[ ${workerNames[$i]} == $host ]]; then
workerIndex=$i
break
fi
done
# set worker groups in application.yaml
[[ -n ${workerIndex} ]] && sed -i "s/- default/- ${groupNames[$workerIndex]}/" $workDir/../worker-server/conf/application.yaml
for dsDir in bin master-server worker-server alert-server api-server ui tools
do
echo "start to scp $dsDir to $host/$installPath"
# Use quiet mode to reduce command line output
scp -q -P $sshPort -r $workDir/../$dsDir $host:$installPath
done
# restore worker groups to default
[[ -n ${workerIndex} ]] && sed -i "s/- ${groupNames[$workerIndex]}/- default/" $workDir/../worker-server/conf/application.yaml
echo "scp dirs to $host/$installPath complete"
done

58
script/start-all.sh

@ -1,58 +0,0 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
workDir=`dirname $0`
workDir=`cd ${workDir};pwd`
source ${workDir}/env/install_env.sh
workersGroup=(${workers//,/ })
for workerGroup in ${workersGroup[@]}
do
echo $workerGroup;
worker=`echo $workerGroup|awk -F':' '{print $1}'`
workerNames+=($worker)
done
mastersHost=(${masters//,/ })
for master in ${mastersHost[@]}
do
echo "$master master server is starting"
ssh -o StrictHostKeyChecking=no -p $sshPort $master "cd $installPath/; bash bin/dolphinscheduler-daemon.sh start master-server;"
done
for worker in ${workerNames[@]}
do
echo "$worker worker server is starting"
ssh -o StrictHostKeyChecking=no -p $sshPort $worker "cd $installPath/; bash bin/dolphinscheduler-daemon.sh start worker-server;"
done
ssh -o StrictHostKeyChecking=no -p $sshPort $alertServer "cd $installPath/; bash bin/dolphinscheduler-daemon.sh start alert-server;"
apiServersHost=(${apiServers//,/ })
for apiServer in ${apiServersHost[@]}
do
echo "$apiServer api server is starting"
ssh -o StrictHostKeyChecking=no -p $sshPort $apiServer "cd $installPath/; bash bin/dolphinscheduler-daemon.sh start api-server;"
done
# query server status
echo "query server status"
cd $installPath/; bash bin/status-all.sh

73
script/status-all.sh

@ -1,73 +0,0 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
workDir=`dirname $0`
workDir=`cd ${workDir};pwd`
source ${workDir}/env/install_env.sh
# install_env.sh info
echo -e '\n'
echo "====================== dolphinscheduler server config ============================="
echo -e "1.dolphinscheduler server node config hosts:[ \033[1;32m ${ips} \033[0m ]"
echo -e "2.master server node config hosts:[ \033[1;32m ${masters} \033[0m ]"
echo -e "3.worker server node config hosts:[ \033[1;32m ${workers} \033[0m ]"
echo -e "4.alert server node config hosts:[ \033[1;32m ${alertServer} \033[0m ]"
echo -e "5.api server node config hosts:[ \033[1;32m ${apiServers} \033[0m ]"
# all server check state
echo -e '\n'
echo "====================== dolphinscheduler server status ============================="
firstColumn="node server state"
echo $firstColumn
echo -e '\n'
workersGroup=(${workers//,/ })
for workerGroup in ${workersGroup[@]}
do
worker=`echo $workerGroup|awk -F':' '{print $1}'`
workerNames+=($worker)
done
StateRunning="Running"
# 1.master server check state
mastersHost=(${masters//,/ })
for master in ${mastersHost[@]}
do
masterState=`ssh -o StrictHostKeyChecking=no -p $sshPort $master "cd $installPath/; bash bin/dolphinscheduler-daemon.sh status master-server;"`
echo "$master $masterState"
done
# 2.worker server check state
for worker in ${workerNames[@]}
do
workerState=`ssh -o StrictHostKeyChecking=no -p $sshPort $worker "cd $installPath/; bash bin/dolphinscheduler-daemon.sh status worker-server;"`
echo "$worker $workerState"
done
# 3.alter server check state
alertState=`ssh -o StrictHostKeyChecking=no -p $sshPort $alertServer "cd $installPath/; bash bin/dolphinscheduler-daemon.sh status alert-server;"`
echo "$alertServer $alertState"
# 4.api server check state
apiServersHost=(${apiServers//,/ })
for apiServer in ${apiServersHost[@]}
do
apiState=`ssh -o StrictHostKeyChecking=no -p $sshPort $apiServer "cd $installPath/; bash bin/dolphinscheduler-daemon.sh status api-server;"`
echo "$apiServer $apiState"
done

53
script/stop-all.sh

@ -1,53 +0,0 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
workDir=`dirname $0`
workDir=`cd ${workDir};pwd`
source ${workDir}/env/install_env.sh
workersGroup=(${workers//,/ })
for workerGroup in ${workersGroup[@]}
do
echo $workerGroup;
worker=`echo $workerGroup|awk -F':' '{print $1}'`
workerNames+=($worker)
done
mastersHost=(${masters//,/ })
for master in ${mastersHost[@]}
do
echo "$master master server is stopping"
ssh -o StrictHostKeyChecking=no -p $sshPort $master "cd $installPath/; bash bin/dolphinscheduler-daemon.sh stop master-server;"
done
for worker in ${workerNames[@]}
do
echo "$worker worker server is stopping"
ssh -o StrictHostKeyChecking=no -p $sshPort $worker "cd $installPath/; bash bin/dolphinscheduler-daemon.sh stop worker-server;"
done
ssh -o StrictHostKeyChecking=no -p $sshPort $alertServer "cd $installPath/; bash bin/dolphinscheduler-daemon.sh stop alert-server;"
apiServersHost=(${apiServers//,/ })
for apiServer in ${apiServersHost[@]}
do
echo "$apiServer api server is stopping"
ssh -o StrictHostKeyChecking=no -p $sshPort $apiServer "cd $installPath/; bash bin/dolphinscheduler-daemon.sh stop api-server;"
done
Loading…
Cancel
Save