From ea1ea2bfb26cba8f6a791f45f972d9f61e7eb843 Mon Sep 17 00:00:00 2001 From: qiaozhanwei Date: Wed, 9 Oct 2019 20:06:04 +0800 Subject: [PATCH] data_source.properties rename (#965) * add ConnectionFactoryTest and ConnectionFactory read datasource from appliction.yml * .escheduler_env.sh to dolphinscheduler_env.sh * dao yml assembly to conf directory * table name modify * entity title table name modify * logback log name modify * running through the big process * running through the big process error modify * logback log name modify * data_source.properties rename * logback log name modify * install.sh optimization * install.sh optimization --- install.sh | 111 ++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 75 insertions(+), 36 deletions(-) diff --git a/install.sh b/install.sh index bf87f686dc..fb72f6a497 100644 --- a/install.sh +++ b/install.sh @@ -13,14 +13,14 @@ elif [[ "$OSTYPE" == "linux-gnu" ]]; then txt="" elif [[ "$OSTYPE" == "cygwin" ]]; then # POSIX compatibility layer and Linux environment emulation for Windows - echo "Easy Scheduler not support Windows operating system" + echo "DolphinScheduler not support Windows operating system" exit 1 elif [[ "$OSTYPE" == "msys" ]]; then # Lightweight shell and GNU utilities compiled for Windows (part of MinGW) - echo "Easy Scheduler not support Windows operating system" + echo "DolphinScheduler not support Windows operating system" exit 1 elif [[ "$OSTYPE" == "win32" ]]; then - echo "Easy Scheduler not support Windows operating system" + echo "DolphinScheduler not support Windows operating system" exit 1 elif [[ "$OSTYPE" == "freebsd"* ]]; then # ... @@ -34,27 +34,30 @@ fi source ${workDir}/conf/config/run_config.conf source ${workDir}/conf/config/install_config.conf -# mysql config -# mysql address and port -mysqlHost="192.168.xx.xx:3306" +# for example mysql or postgresql ... +dbtype="mysql" + +# db config +# db address and port +dbhost="192.168.xx.xx:3306" # mysql database -mysqlDb="escheduler" +dbname="dolphinscheduler" # mysql username -mysqlUserName="xx" +username="xx" # mysql passwprd # Note: if there are special characters, please use the \ transfer character to transfer -mysqlPassword="xx" +passowrd="xx" # conf/config/install_config.conf config # Note: the installation path is not the same as the current path (pwd) -installPath="/data1_1T/escheduler" +installPath="/data1_1T/dolphinscheduler" # deployment user # Note: the deployment user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled, the root directory needs to be created by itself -deployUser="escheduler" +deployUser="dolphinscheduler" # zk cluster zkQuorum="192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181" @@ -99,6 +102,8 @@ mailPassword="xxxxxxxxxx" # TLS mail protocol support starttlsEnable="false" +sslTrust="xxxxxxxxxx" + # SSL mail protocol support # note: The SSL protocol is enabled by default. # only one of TLS and SSL can be in the true state. @@ -127,8 +132,8 @@ monitorServerState="false" resUploadStartupType="NONE" # if resUploadStartupType is HDFS，defaultFS write namenode address，HA you need to put core-site.xml and hdfs-site.xml in the conf directory. -# if S3，write S3 address，HA，for example ：s3a://escheduler， -# Note，s3 be sure to create the root directory /escheduler +# if S3，write S3 address，HA，for example ：s3a://dolphinscheduler， +# Note，s3 be sure to create the root directory /dolphinscheduler defaultFS="hdfs://mycluster:8020" # if S3 is configured, the following configuration is required. @@ -144,7 +149,7 @@ singleYarnIp="ark1" # hdfs root path, the owner of the root path must be the deployment user. # versions prior to 1.1.0 do not automatically create the hdfs root directory, you need to create it yourself. -hdfsPath="/escheduler" +hdfsPath="/dolphinscheduler" # have users who create directory permissions under hdfs root path / # Note: if kerberos is enabled, hdfsRootUser="" can be used directly. @@ -152,13 +157,13 @@ hdfsRootUser="hdfs" # common config # Program root path -programPath="/tmp/escheduler" +programPath="/tmp/dolphinscheduler" # download path -downloadPath="/tmp/escheduler/download" +downloadPath="/tmp/dolphinscheduler/download" # task execute path -execPath="/tmp/escheduler/exec" +execPath="/tmp/dolphinscheduler/exec" # SHELL environmental variable path shellEnvPath="$installPath/conf/env/.dolphinscheduler_env.sh" @@ -185,10 +190,10 @@ keytabPath="$installPath/conf/hdfs.headless.keytab" # zk config # zk root directory -zkRoot="/escheduler" +zkRoot="/dolphinscheduler" # used to record the zk directory of the hanging machine -zkDeadServers="/escheduler/dead-servers" +zkDeadServers="$zkRoot/dead-servers" # masters directory zkMasters="$zkRoot/masters" @@ -272,7 +277,7 @@ apiServerPort="12345" apiServerSessionTimeout="7200" # api server context path -apiServerContextPath="/escheduler/" +apiServerContextPath="/dolphinscheduler/" # spring max file size springMaxFileSize="1024MB" @@ -283,15 +288,48 @@ springMaxRequestSize="1024MB" # api max http post size apiMaxHttpPostSize="5000000" + +# db config +# db address and port +dbhost="192.168.xx.xx:3306" + +# mysql database +dbname="dolphinscheduler" + +# mysql username +username="xx" + +# mysql passwprd +# Note: if there are special characters, please use the \ transfer character to transfer +passowrd="xx" + # 1,replace file echo "1,replace file" -sed -i ${txt} "s#spring.datasource.url.*#spring.datasource.url=jdbc:mysql://${mysqlHost}/${mysqlDb}?characterEncoding=UTF-8#g" conf/dao/data_source.properties -sed -i ${txt} "s#spring.datasource.username.*#spring.datasource.username=${mysqlUserName}#g" conf/dao/data_source.properties -sed -i ${txt} "s#spring.datasource.password.*#spring.datasource.password=${mysqlPassword}#g" conf/dao/data_source.properties +if [ $dbtype == "mysql" ];then + sed -i ${txt} "s#spring.datasource.url.*#spring.datasource.url=jdbc:mysql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" application.yml + sed -i ${txt} "s#spring.datasource.username.*#spring.datasource.username=${username}#g" application.yml + sed -i ${txt} "s#spring.datasource.password.*#spring.datasource.password=${passowrd}#g" application.yml + sed -i ${txt} "s#spring.datasource.driver-class-name.*#spring.datasource.driver-class-name=com.mysql.jdbc.Driver#g" application.yml + + + sed -i ${txt} "s#org.quartz.dataSource.myDs.URL.*#org.quartz.dataSource.myDs.URL=jdbc:mysql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/quartz.properties + sed -i ${txt} "s#org.quartz.dataSource.myDs.user.*#org.quartz.dataSource.myDs.user=${username}#g" conf/quartz.properties + sed -i ${txt} "s#org.quartz.dataSource.myDs.password.*#org.quartz.dataSource.myDs.password=${passowrd}#g" conf/quartz.properties + sed -i ${txt} "s#org.quartz.dataSource.myDs.driver.*#org.quartz.dataSource.myDs.driver=com.mysql.jdbc.Driver#g" conf/quartz.properties +fi + +if [ $dbtype == "postgresql" ];then + sed -i ${txt} "s#spring.datasource.url.*#spring.datasource.url=jdbc:postgresql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" application.yml + sed -i ${txt} "s#spring.datasource.username.*#spring.datasource.username=${username}#g" application.yml + sed -i ${txt} "s#spring.datasource.password.*#spring.datasource.password=${passowrd}#g" application.yml + sed -i ${txt} "s#spring.datasource.driver-class-name.*#spring.datasource.driver-class-name=org.postgresql.Driver#g" application.yml + + sed -i ${txt} "s#org.quartz.dataSource.myDs.URL.*#org.quartz.dataSource.myDs.URL=jdbc:mysql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/quartz.properties + sed -i ${txt} "s#org.quartz.dataSource.myDs.user.*#org.quartz.dataSource.myDs.user=${username}#g" conf/quartz.properties + sed -i ${txt} "s#org.quartz.dataSource.myDs.password.*#org.quartz.dataSource.myDs.password=${passowrd}#g" conf/quartz.properties + sed -i ${txt} "s#org.quartz.dataSource.myDs.driver.*#org.quartz.dataSource.myDs.driver=org.postgresql.Driver#g" conf/quartz.properties +fi -sed -i ${txt} "s#org.quartz.dataSource.myDs.URL.*#org.quartz.dataSource.myDs.URL=jdbc:mysql://${mysqlHost}/${mysqlDb}?characterEncoding=UTF-8#g" conf/quartz.properties -sed -i ${txt} "s#org.quartz.dataSource.myDs.user.*#org.quartz.dataSource.myDs.user=${mysqlUserName}#g" conf/quartz.properties -sed -i ${txt} "s#org.quartz.dataSource.myDs.password.*#org.quartz.dataSource.myDs.password=${mysqlPassword}#g" conf/quartz.properties sed -i ${txt} "s#fs.defaultFS.*#fs.defaultFS=${defaultFS}#g" conf/common/hadoop/hadoop.properties @@ -308,7 +346,7 @@ sed -i ${txt} "s#process.exec.basepath.*#process.exec.basepath=${execPath}#g" co sed -i ${txt} "s#hdfs.root.user.*#hdfs.root.user=${hdfsRootUser}#g" conf/common/common.properties sed -i ${txt} "s#data.store2hdfs.basepath.*#data.store2hdfs.basepath=${hdfsPath}#g" conf/common/common.properties sed -i ${txt} "s#res.upload.startup.type.*#res.upload.startup.type=${resUploadStartupType}#g" conf/common/common.properties -sed -i ${txt} "s#escheduler.env.path.*#escheduler.env.path=${shellEnvPath}#g" conf/common/common.properties +sed -i ${txt} "s#dolphinscheduler.env.path.*#dolphinscheduler.env.path=${shellEnvPath}#g" conf/common/common.properties sed -i ${txt} "s#resource.view.suffixs.*#resource.view.suffixs=${resSuffixs}#g" conf/common/common.properties sed -i ${txt} "s#development.state.*#development.state=${devState}#g" conf/common/common.properties sed -i ${txt} "s#hadoop.security.authentication.startup.state.*#hadoop.security.authentication.startup.state=${kerberosStartUp}#g" conf/common/common.properties @@ -317,15 +355,15 @@ sed -i ${txt} "s#login.user.keytab.username.*#login.user.keytab.username=${keyta sed -i ${txt} "s#login.user.keytab.path.*#login.user.keytab.path=${keytabPath}#g" conf/common/common.properties sed -i ${txt} "s#zookeeper.quorum.*#zookeeper.quorum=${zkQuorum}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.escheduler.root.*#zookeeper.escheduler.root=${zkRoot}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.escheduler.dead.servers.*#zookeeper.escheduler.dead.servers=${zkDeadServers}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.escheduler.masters.*#zookeeper.escheduler.masters=${zkMasters}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.escheduler.workers.*#zookeeper.escheduler.workers=${zkWorkers}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.escheduler.lock.masters.*#zookeeper.escheduler.lock.masters=${mastersLock}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.escheduler.lock.workers.*#zookeeper.escheduler.lock.workers=${workersLock}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.escheduler.lock.failover.masters.*#zookeeper.escheduler.lock.failover.masters=${mastersFailover}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.escheduler.lock.failover.workers.*#zookeeper.escheduler.lock.failover.workers=${workersFailover}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.escheduler.lock.failover.startup.masters.*#zookeeper.escheduler.lock.failover.startup.masters=${mastersStartupFailover}#g" conf/zookeeper.properties +sed -i ${txt} "s#zookeeper.dolphinscheduler.root.*#zookeeper.dolphinscheduler.root=${zkRoot}#g" conf/zookeeper.properties +sed -i ${txt} "s#zookeeper.dolphinscheduler.dead.servers.*#zookeeper.dolphinscheduler.dead.servers=${zkDeadServers}#g" conf/zookeeper.properties +sed -i ${txt} "s#zookeeper.dolphinscheduler.masters.*#zookeeper.dolphinscheduler.masters=${zkMasters}#g" conf/zookeeper.properties +sed -i ${txt} "s#zookeeper.dolphinscheduler.workers.*#zookeeper.dolphinscheduler.workers=${zkWorkers}#g" conf/zookeeper.properties +sed -i ${txt} "s#zookeeper.dolphinscheduler.lock.masters.*#zookeeper.dolphinscheduler.lock.masters=${mastersLock}#g" conf/zookeeper.properties +sed -i ${txt} "s#zookeeper.dolphinscheduler.lock.workers.*#zookeeper.dolphinscheduler.lock.workers=${workersLock}#g" conf/zookeeper.properties +sed -i ${txt} "s#zookeeper.dolphinscheduler.lock.failover.masters.*#zookeeper.dolphinscheduler.lock.failover.masters=${mastersFailover}#g" conf/zookeeper.properties +sed -i ${txt} "s#zookeeper.dolphinscheduler.lock.failover.workers.*#zookeeper.dolphinscheduler.lock.failover.workers=${workersFailover}#g" conf/zookeeper.properties +sed -i ${txt} "s#zookeeper.dolphinscheduler.lock.failover.startup.masters.*#zookeeper.dolphinscheduler.lock.failover.startup.masters=${mastersStartupFailover}#g" conf/zookeeper.properties sed -i ${txt} "s#zookeeper.session.timeout.*#zookeeper.session.timeout=${zkSessionTimeout}#g" conf/zookeeper.properties sed -i ${txt} "s#zookeeper.connection.timeout.*#zookeeper.connection.timeout=${zkConnectionTimeout}#g" conf/zookeeper.properties sed -i ${txt} "s#zookeeper.retry.sleep.*#zookeeper.retry.sleep=${zkRetrySleep}#g" conf/zookeeper.properties @@ -361,6 +399,7 @@ sed -i ${txt} "s#mail.server.port.*#mail.server.port=${mailServerPort}#g" conf/a sed -i ${txt} "s#mail.sender.*#mail.sender=${mailSender}#g" conf/alert.properties sed -i ${txt} "s#mail.passwd.*#mail.passwd=${mailPassword}#g" conf/alert.properties sed -i ${txt} "s#mail.smtp.starttls.enable.*#mail.smtp.starttls.enable=${starttlsEnable}#g" conf/alert.properties +sed -i ${txt} "s#mail.smtp.ssl.trust.*#mail.smtp.ssl.trust=${sslTrust}#g" conf/alert.properties sed -i ${txt} "s#mail.smtp.ssl.enable.*#mail.smtp.ssl.enable=${sslEnable}#g" conf/alert.properties sed -i ${txt} "s#xls.file.path.*#xls.file.path=${xlsFilePath}#g" conf/alert.properties sed -i ${txt} "s#enterprise.wechat.corp.id.*#enterprise.wechat.corp.id=${enterpriseWechatCorpId}#g" conf/alert.properties