From 515c3631480c33c3b12fd0188fe064d59ade7b94 Mon Sep 17 00:00:00 2001 From: Eric Gao Date: Wed, 27 Apr 2022 13:44:37 +0800 Subject: [PATCH] [Feature][Deploy] Add missing default values in dolphinscheduler_env.sh (#9733) * [Feature][Deploy] Add missing default values in dolphinscheduler_env.sh (#9675) * [Feature][Deploy] Restore dolphinscheduler_env.sh to avoid e2e failure (#9675) * [Feature][Deploy] Change mysql driver class name to com.mysql.cj.jdbc.Driver (#9675) * [Feature][Deploy] Remove redundant config file (#9675) --- .../en/guide/installation/pseudo-cluster.md | 71 +++++++++++-------- docs/docs/en/guide/upgrade.md | 9 ++- .../zh/guide/installation/pseudo-cluster.md | 65 ++++++++++------- docs/docs/zh/guide/upgrade.md | 8 ++- .../main/assembly/dolphinscheduler-tools.xml | 9 --- .../src/main/bin/create-schema.sh | 3 +- .../src/main/bin/upgrade-schema.sh | 2 +- 7 files changed, 94 insertions(+), 73 deletions(-) diff --git a/docs/docs/en/guide/installation/pseudo-cluster.md b/docs/docs/en/guide/installation/pseudo-cluster.md index ee29a04407..b96679a63b 100644 --- a/docs/docs/en/guide/installation/pseudo-cluster.md +++ b/docs/docs/en/guide/installation/pseudo-cluster.md @@ -67,24 +67,6 @@ Go to the ZooKeeper installation directory, copy configure file `zoo_sample.cfg` ./bin/zkServer.sh start ``` - - ## Modify Configuration After completing the preparation of the basic environment, you need to modify the configuration file according to the @@ -115,26 +97,48 @@ deployUser="dolphinscheduler" ### Modify `dolphinscheduler_env.sh` -File `dolphinscheduler_env.sh` describes the database configuration of DolphinScheduler, which in the path `bin/env/dolphinscheduler_env.sh` -and some tasks which need external dependencies or libraries such as `JAVA_HOME` and `SPARK_HOME`. You could ignore the -task external dependencies if you do not use those tasks, but you have to change `JAVA_HOME`, registry center and database +File `./bin/env/dolphinscheduler_env.sh` describes the following configurations: + +* Database configuration of DolphinScheduler, see [Initialize the Database](#initialize-the-database) for detailed instructions. +* Some tasks which need external dependencies or libraries such as `JAVA_HOME` and `SPARK_HOME`. +* Registry center `zookeeper`. +* Server related configuration, such as cache type, timezone, etc. + +You could ignore the task external dependencies if you do not use those tasks, but you have to change `JAVA_HOME`, registry center and database related configurations based on your environment. -```sh +```shell # JAVA_HOME, will use it to start DolphinScheduler server -export JAVA_HOME=${JAVA_HOME:-/custom/path} +export JAVA_HOME=${JAVA_HOME:-/opt/soft/java} # Database related configuration, set database type, username and password export DATABASE=${DATABASE:-postgresql} export SPRING_PROFILES_ACTIVE=${DATABASE} export SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.postgresql.Driver -export SPRING_DATASOURCE_URL="jdbc:postgresql://127.0.0.1:5432/dolphinscheduler" -export SPRING_DATASOURCE_USERNAME="username" -export SPRING_DATASOURCE_PASSWORD="password" +export SPRING_DATASOURCE_URL=jdbc:postgresql://127.0.0.1:5432/dolphinscheduler +export SPRING_DATASOURCE_USERNAME={user} +export SPRING_DATASOURCE_PASSWORD={password} + +# DolphinScheduler server related configuration +export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none} +export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC} +export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10} # Registry center configuration, determines the type and link of the registry center export REGISTRY_TYPE=${REGISTRY_TYPE:-zookeeper} export REGISTRY_ZOOKEEPER_CONNECT_STRING=${REGISTRY_ZOOKEEPER_CONNECT_STRING:-localhost:2181} + +# Tasks related configurations, need to change the configuration if you use the related tasks. +export HADOOP_HOME=${HADOOP_HOME:-/opt/soft/hadoop} +export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/opt/soft/hadoop/etc/hadoop} +export SPARK_HOME1=${SPARK_HOME1:-/opt/soft/spark1} +export SPARK_HOME2=${SPARK_HOME2:-/opt/soft/spark2} +export PYTHON_HOME=${PYTHON_HOME:-/opt/soft/python} +export HIVE_HOME=${HIVE_HOME:-/opt/soft/hive} +export FLINK_HOME=${FLINK_HOME:-/opt/soft/flink} +export DATAX_HOME=${DATAX_HOME:-/opt/soft/datax} + +export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH ``` ## Initialize the Database @@ -168,13 +172,20 @@ mysql> GRANT ALL PRIVILEGES ON dolphinscheduler.* TO '{user}'@'%'; mysql> CREATE USER '{user}'@'localhost' IDENTIFIED BY '{password}'; mysql> GRANT ALL PRIVILEGES ON dolphinscheduler.* TO '{user}'@'localhost'; mysql> FLUSH PRIVILEGES; -``` +``` -Change the username and password in `tools/conf/application.yaml` to {user} and {password} you set in the previous step. +Then, modify `./bin/env/dolphinscheduler_env.sh` to use mysql, change {user} and {password} to what you set in the previous step. -Then, modify `tools/bin/dolphinscheduler_env.sh`, set mysql as default database `export DATABASE=${DATABASE:-mysql}`. +```shell +export DATABASE=${DATABASE:-mysql} +export SPRING_PROFILES_ACTIVE=${DATABASE} +export SPRING_DATASOURCE_DRIVER_CLASS_NAME=com.mysql.cj.jdbc.Driver +export SPRING_DATASOURCE_URL=jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 +export SPRING_DATASOURCE_USERNAME={user} +export SPRING_DATASOURCE_PASSWORD={password} +``` -After the above steps done you would create a new database for DolphinScheduler, then run Shell scripts to init database: +After the above steps done you would create a new database for DolphinScheduler, then run the Shell script to init database: ```shell sh tools/bin/create-schema.sh diff --git a/docs/docs/en/guide/upgrade.md b/docs/docs/en/guide/upgrade.md index a7c8d109b4..f3d3f4c779 100644 --- a/docs/docs/en/guide/upgrade.md +++ b/docs/docs/en/guide/upgrade.md @@ -13,12 +13,15 @@ ## Database Upgrade -- Change `username` and `password` in `./tools/conf/application.yaml` to yours. - -- If using MySQL as the database to run DolphinScheduler, please config it in `./tools/bin/dolphinscheduler_env.sh`, and add MYSQL connector jar into lib dir `./tools/lib`, here we download `mysql-connector-java-8.0.16.jar`, and then correctly configure database connection information. You can download MYSQL connector jar from [here](https://downloads.MySQL.com/archives/c-j/). Otherwise, PostgreSQL is the default database. +- If using MySQL as the database to run DolphinScheduler, please config it in `./bin/env/dolphinscheduler_env.sh`, change username and password to yours, and add MYSQL connector jar into lib dir `./tools/libs`, here we download `mysql-connector-java-8.0.16.jar`, and then correctly configure database connection information. You can download MYSQL connector jar from [here](https://downloads.MySQL.com/archives/c-j/). Otherwise, PostgreSQL is the default database. ```shell export DATABASE=${DATABASE:-mysql} + export SPRING_PROFILES_ACTIVE=${DATABASE} + export SPRING_DATASOURCE_DRIVER_CLASS_NAME=com.mysql.cj.jdbc.Driver + export SPRING_DATASOURCE_URL=jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 + export SPRING_DATASOURCE_USERNAME={user} + export SPRING_DATASOURCE_PASSWORD={password} ``` - Execute database upgrade script: diff --git a/docs/docs/zh/guide/installation/pseudo-cluster.md b/docs/docs/zh/guide/installation/pseudo-cluster.md index bb3fe816a1..2e0570cce2 100644 --- a/docs/docs/zh/guide/installation/pseudo-cluster.md +++ b/docs/docs/zh/guide/installation/pseudo-cluster.md @@ -67,24 +67,6 @@ chmod 600 ~/.ssh/authorized_keys ./bin/zkServer.sh start ``` - - ## 修改相关配置 完成基础环境的准备后，需要根据你的机器环境修改配置文件。配置文件可以在目录 `bin/env` 中找到，他们分别是 并命名为 `install_env.sh` 和 `dolphinscheduler_env.sh`。 @@ -113,25 +95,47 @@ deployUser="dolphinscheduler" ### 修改 `dolphinscheduler_env.sh` 文件 -文件 `dolphinscheduler_env.sh` 描述了 DolphinScheduler 的数据库配置，一些任务类型外部依赖路径或库文件，注册中心，其中 `JAVA_HOME` -和 `SPARK_HOME`都是在这里定义的，其路径是 `bin/env/dolphinscheduler_env.sh`。如果您不使用某些任务类型，您可以忽略任务外部依赖项， -但您必须根据您的环境更改 `JAVA_HOME`、注册中心和数据库相关配置。 +文件 `./bin/env/dolphinscheduler_env.sh` 描述了下列配置： + +* DolphinScheduler 的数据库配置，详细配置方法见[初始化数据库](#初始化数据库) +* 一些任务类型外部依赖路径或库文件，如 `JAVA_HOME` 和 `SPARK_HOME`都是在这里定义的 +* 注册中心`zookeeper` +* 服务端相关配置，比如缓存，时区设置等 + +如果您不使用某些任务类型，您可以忽略任务外部依赖项，但您必须根据您的环境更改 `JAVA_HOME`、注册中心和数据库相关配置。 ```sh # JAVA_HOME, will use it to start DolphinScheduler server -export JAVA_HOME=${JAVA_HOME:-/custom/path} +export JAVA_HOME=${JAVA_HOME:-/opt/soft/java} # Database related configuration, set database type, username and password export DATABASE=${DATABASE:-postgresql} export SPRING_PROFILES_ACTIVE=${DATABASE} export SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.postgresql.Driver -export SPRING_DATASOURCE_URL="jdbc:postgresql://127.0.0.1:5432/dolphinscheduler" -export SPRING_DATASOURCE_USERNAME="username" -export SPRING_DATASOURCE_PASSWORD="password" +export SPRING_DATASOURCE_URL=jdbc:postgresql://127.0.0.1:5432/dolphinscheduler +export SPRING_DATASOURCE_USERNAME={user} +export SPRING_DATASOURCE_PASSWORD={password} + +# DolphinScheduler server related configuration +export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none} +export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC} +export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10} # Registry center configuration, determines the type and link of the registry center export REGISTRY_TYPE=${REGISTRY_TYPE:-zookeeper} export REGISTRY_ZOOKEEPER_CONNECT_STRING=${REGISTRY_ZOOKEEPER_CONNECT_STRING:-localhost:2181} + +# Tasks related configurations, need to change the configuration if you use the related tasks. +export HADOOP_HOME=${HADOOP_HOME:-/opt/soft/hadoop} +export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/opt/soft/hadoop/etc/hadoop} +export SPARK_HOME1=${SPARK_HOME1:-/opt/soft/spark1} +export SPARK_HOME2=${SPARK_HOME2:-/opt/soft/spark2} +export PYTHON_HOME=${PYTHON_HOME:-/opt/soft/python} +export HIVE_HOME=${HIVE_HOME:-/opt/soft/hive} +export FLINK_HOME=${FLINK_HOME:-/opt/soft/flink} +export DATAX_HOME=${DATAX_HOME:-/opt/soft/datax} + +export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH ``` ## 初始化数据库 @@ -167,9 +171,16 @@ mysql> GRANT ALL PRIVILEGES ON dolphinscheduler.* TO '{user}'@'localhost'; mysql> FLUSH PRIVILEGES; ``` -将`tools/conf/application.yaml`中的username和password改成你在上一步中设置的用户名{user}和密码{password} +然后修改`./bin/env/dolphinscheduler_env.sh`，将username和password改成你在上一步中设置的用户名{user}和密码{password} -然后修改`tools/bin/dolphinscheduler_env.sh`，将mysql设置为默认数据类型`export DATABASE=${DATABASE:-mysql}`. +```shell +export DATABASE=${DATABASE:-mysql} +export SPRING_PROFILES_ACTIVE=${DATABASE} +export SPRING_DATASOURCE_DRIVER_CLASS_NAME=com.mysql.cj.jdbc.Driver +export SPRING_DATASOURCE_URL=jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 +export SPRING_DATASOURCE_USERNAME={user} +export SPRING_DATASOURCE_PASSWORD={password} +``` 完成上述步骤后，您已经为 DolphinScheduler 创建一个新数据库，现在你可以通过快速的 Shell 脚本来初始化数据库 diff --git a/docs/docs/zh/guide/upgrade.md b/docs/docs/zh/guide/upgrade.md index 69a8b76597..dc1d6cb732 100644 --- a/docs/docs/zh/guide/upgrade.md +++ b/docs/docs/zh/guide/upgrade.md @@ -13,12 +13,16 @@ - 以下升级操作都需要在新版本的目录进行 ## 4. 数据库升级 -- 将`./tools/conf/application.yaml`中的username和password改成你设定数据库用户名和密码 -- 如果选择 MySQL，请修改`./tools/bin/dolphinscheduler_env.sh`中的如下配置, 还需要手动添加 [ mysql-connector-java 驱动 jar ](https://downloads.MySQL.com/archives/c-j/) 包到 lib 目录（`./tools/lib`）下，这里下载的是mysql-connector-java-8.0.16.jar +- 如果选择 MySQL，请修改`./bin/env/dolphinscheduler_env.sh`中的如下配置（{user}和{password}改成你数据库的用户名和密码）, 还需要手动添加 [ mysql-connector-java 驱动 jar ](https://downloads.MySQL.com/archives/c-j/) 包到 lib 目录（`./tools/lib`）下，这里下载的是mysql-connector-java-8.0.16.jar ```shell export DATABASE=${DATABASE:-mysql} + export SPRING_PROFILES_ACTIVE=${DATABASE} + export SPRING_DATASOURCE_DRIVER_CLASS_NAME=com.mysql.cj.jdbc.Driver + export SPRING_DATASOURCE_URL=jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 + export SPRING_DATASOURCE_USERNAME={user} + export SPRING_DATASOURCE_PASSWORD={password} ``` - 执行数据库升级脚本 diff --git a/dolphinscheduler-tools/src/main/assembly/dolphinscheduler-tools.xml b/dolphinscheduler-tools/src/main/assembly/dolphinscheduler-tools.xml index 02a656420d..49a95d25d0 100644 --- a/dolphinscheduler-tools/src/main/assembly/dolphinscheduler-tools.xml +++ b/dolphinscheduler-tools/src/main/assembly/dolphinscheduler-tools.xml @@ -38,15 +38,6 @@ 0755 0755 - - ${basedir}/../script/env - conf - - dolphinscheduler_env.sh - - 0755 - 0755 - ${basedir}/../dolphinscheduler-dao/src/main/resources diff --git a/dolphinscheduler-tools/src/main/bin/create-schema.sh b/dolphinscheduler-tools/src/main/bin/create-schema.sh index e8eb6e3d5d..29cc5d7711 100755 --- a/dolphinscheduler-tools/src/main/bin/create-schema.sh +++ b/dolphinscheduler-tools/src/main/bin/create-schema.sh @@ -18,7 +18,8 @@ BIN_DIR=$(dirname $0) DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/../..; pwd)} -source "$DOLPHINSCHEDULER_HOME/tools/bin/dolphinscheduler_env.sh" + +source "$DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh" JAVA_OPTS=${JAVA_OPTS:-"-server -Duser.timezone=${SPRING_JACKSON_TIME_ZONE} -Xms1g -Xmx1g -Xmn512m -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"} diff --git a/dolphinscheduler-tools/src/main/bin/upgrade-schema.sh b/dolphinscheduler-tools/src/main/bin/upgrade-schema.sh index f545e294e2..d11dc6a2c1 100755 --- a/dolphinscheduler-tools/src/main/bin/upgrade-schema.sh +++ b/dolphinscheduler-tools/src/main/bin/upgrade-schema.sh @@ -19,7 +19,7 @@ BIN_DIR=$(dirname $0) DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/../..; pwd)} -source "$DOLPHINSCHEDULER_HOME/conf/dolphinscheduler_env.sh" +source "$DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh" JAVA_OPTS=${JAVA_OPTS:-"-server -Duser.timezone=${SPRING_JACKSON_TIME_ZONE} -Xms1g -Xmx1g -Xmn512m -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"}