[Feature] merge some configurations #1635 (#1636)

* merge hadoop.properties into common.properties

* merge hadoop,zookeeper.properties into common.properties
remove combined.properties/master.properties/worker.properties

* change db user/pwd to test/test

dolphinscheduler-api/src/main/resources/application-combined.properties

@@ -1,40 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-logging.config=classpath:combined_logback.xml
-
-# server port
-server.port=12345
-
-# session config
-server.servlet.session.timeout=7200
-
-server.servlet.context-path=/dolphinscheduler/
-
-# file size limit for upload
-spring.servlet.multipart.max-file-size=1024MB
-spring.servlet.multipart.max-request-size=1024MB
-
-#post content
-server.jetty.max-http-post-size=5000000
-
-spring.messages.encoding=UTF-8
-
-#i18n classpath folder , file prefix messages， if have many files, use "," seperator
-spring.messages.basename=i18n/messages
-
-server.is-combined-server=true

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java

@@ -26,25 +26,10 @@ import java.util.regex.Pattern;
  */
 public final class Constants {
 
-    /**
-     * zookeeper properties path
-     */
-    public static final String ZOOKEEPER_PROPERTIES_PATH = "zookeeper.properties";
-
-    /**
-     * hadoop properties path
-     */
-    public static final String HADOOP_PROPERTIES_PATH = "/common/hadoop/hadoop.properties";
-
     /**
      * common properties path
      */
-    public static final String COMMON_PROPERTIES_PATH = "/common/common.properties";
-
-    /**
-     * dao properties path
-     */
-    public static final String DAO_PROPERTIES_PATH = "application.properties";
+    public static final String COMMON_PROPERTIES_PATH = "/common.properties";
 
     /**
      * fs.defaultFS
@@ -227,6 +212,8 @@ public final class Constants {
     public static final String ZOOKEEPER_CONNECTION_TIMEOUT = "zookeeper.connection.timeout";
 
     public static final String ZOOKEEPER_RETRY_SLEEP = "zookeeper.retry.sleep";
+    public static final String ZOOKEEPER_RETRY_BASE_SLEEP = "zookeeper.retry.base.sleep";
+    public static final String ZOOKEEPER_RETRY_MAX_SLEEP = "zookeeper.retry.max.sleep";
 
     public static final String ZOOKEEPER_RETRY_MAXTIME = "zookeeper.retry.maxtime";
 
@@ -469,7 +456,7 @@ public final class Constants {
     /**
      * task record configuration path
      */
-    public static final String APPLICATION_PROPERTIES = "application-dao.properties";
+    public static final String APPLICATION_PROPERTIES = "application.properties";
 
     public static final String TASK_RECORD_URL = "task.record.datasource.url";
 

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java

@@ -29,7 +29,6 @@ import java.util.Map;
 import java.util.Properties;
 
 import static org.apache.dolphinscheduler.common.Constants.COMMON_PROPERTIES_PATH;
-import static org.apache.dolphinscheduler.common.Constants.HADOOP_PROPERTIES_PATH;
 
 /**
  * property utils
@@ -51,7 +50,7 @@ public class PropertyUtils {
     }
 
     private void init(){
-        String[] propertyFiles = new String[]{HADOOP_PROPERTIES_PATH,COMMON_PROPERTIES_PATH};
+        String[] propertyFiles = new String[]{COMMON_PROPERTIES_PATH};
         for (String fileName : propertyFiles) {
             InputStream fis = null;
             try {

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperConfig.java

@@ -24,7 +24,7 @@ import org.springframework.stereotype.Component;
  * zookeeper conf
  */
 @Component
-@PropertySource("classpath:zookeeper.properties")
+@PropertySource("classpath:common.properties")
 public class ZookeeperConfig {
 
     //zk connect config

dolphinscheduler-common/src/main/resources/common.properties

@@ -18,14 +18,21 @@
 #task queue implementation, default "zookeeper"
 dolphinscheduler.queue.impl=zookeeper
 
-# user data directory path, self configuration, please make sure the directory exists and have read write permissions
-data.basedir.path=/tmp/dolphinscheduler
+#zookeeper cluster.  multiple are separated by commas.  eg. 192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181
+zookeeper.quorum=localhost:2181
 
-# directory path for user data download. self configuration, please make sure the directory exists and have read write permissions
-data.download.basedir.path=/tmp/dolphinscheduler/download
+#dolphinscheduler root directory
+zookeeper.dolphinscheduler.root=/dolphinscheduler
 
-# process execute directory. self configuration, please make sure the directory exists and have read write permissions
-process.exec.basepath=/tmp/dolphinscheduler/exec
+#dolphinscheduler failover directory
+zookeeper.session.timeout=300
+zookeeper.connection.timeout=300
+zookeeper.retry.base.sleep=100
+zookeeper.retry.max.sleep=30000
+zookeeper.retry.maxtime=5
+
+# resource upload startup type : HDFS,S3,NONE
+res.upload.startup.type=NONE
 
 # Users who have permission to create directories under the HDFS root path
 hdfs.root.user=hdfs
@@ -33,8 +40,15 @@ hdfs.root.user=hdfs
 # data base dir, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended
 data.store2hdfs.basepath=/dolphinscheduler
 
-# resource upload startup type : HDFS,S3,NONE
-res.upload.startup.type=NONE
+# user data directory path, self configuration, please make sure the directory exists and have read write permissions
+data.basedir.path=/tmp/dolphinscheduler
+
+# directory path for user data download. self configuration, please make sure the directory exists and have read write permissions
+data.download.basedir.path=/tmp/dolphinscheduler/download
+
+# process execute directory. self configuration, please make sure the directory exists and have read write permissions
+process.exec.basepath=/tmp/dolphinscheduler/exec
+
 
 # whether kerberos starts
 hadoop.security.authentication.startup.state=false
@@ -57,3 +71,24 @@ resource.view.suffixs=txt,log,sh,conf,cfg,py,java,sql,hql,xml,properties
 # is development state? default "false"
 development.state=true
 
+
+# ha or single namenode,If namenode ha needs to copy core-site.xml and hdfs-site.xml
+# to the conf directory，support s3，for example : s3a://dolphinscheduler
+fs.defaultFS=hdfs://mycluster:8020
+
+# s3 need，s3 endpoint
+fs.s3a.endpoint=http://192.168.199.91:9010
+
+# s3 need，s3 access key
+fs.s3a.access.key=A3DXS30FO22544RE
+
+# s3 need，s3 secret key
+fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK
+
+#resourcemanager ha note this need ips , this empty if single
+yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx
+
+# If it is a single resourcemanager, you only need to configure one host name. If it is resourcemanager HA, the default configuration is fine
+yarn.application.status.address=http://ark1:8088/ws/v1/cluster/apps/%s
+
+

dolphinscheduler-common/src/main/resources/common/hadoop/hadoop.properties

@@ -1,35 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# ha or single namenode,If namenode ha needs to copy core-site.xml and hdfs-site.xml
-# to the conf directory，support s3，for example : s3a://dolphinscheduler
-fs.defaultFS=hdfs://mycluster:8020
-
-# s3 need，s3 endpoint
-fs.s3a.endpoint=http://192.168.199.91:9010
-
-# s3 need，s3 access key
-fs.s3a.access.key=A3DXS30FO22544RE
-
-# s3 need，s3 secret key
-fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK
-
-#resourcemanager ha note this need ips , this empty if single
-yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx
-
-# If it is a single resourcemanager, you only need to configure one host name. If it is resourcemanager HA, the default configuration is fine
-yarn.application.status.address=http://ark1:8088/ws/v1/cluster/apps/%s

dolphinscheduler-common/src/main/resources/quartz.properties

@@ -18,6 +18,15 @@
 #============================================================================
 # Configure Main Scheduler Properties
 #============================================================================
+#org.quartz.jobStore.driverDelegateClass = org.quartz.impl.jdbcjobstore.StdJDBCDelegate
+org.quartz.jobStore.driverDelegateClass = org.quartz.impl.jdbcjobstore.PostgreSQLDelegate
+#org.quartz.dataSource.myDs.driver = com.mysql.jdbc.Driver
+org.quartz.dataSource.myDs.driver = org.postgresql.Driver
+#org.quartz.dataSource.myDs.URL = jdbc:mysql://192.168.xx.xx:3306/dolphinscheduler?characterEncoding=utf8
+org.quartz.dataSource.myDs.URL = jdbc:postgresql://localhost:5432/dolphinscheduler?characterEncoding=utf8
+org.quartz.dataSource.myDs.user = test
+org.quartz.dataSource.myDs.password = test
+
 org.quartz.scheduler.instanceName = DolphinScheduler
 org.quartz.scheduler.instanceId = AUTO
 org.quartz.scheduler.makeSchedulerThreadDaemon = true
@@ -37,8 +46,7 @@ org.quartz.threadPool.threadPriority = 5
 #============================================================================
  
 org.quartz.jobStore.class = org.quartz.impl.jdbcjobstore.JobStoreTX
-#org.quartz.jobStore.driverDelegateClass = org.quartz.impl.jdbcjobstore.StdJDBCDelegate
-org.quartz.jobStore.driverDelegateClass = org.quartz.impl.jdbcjobstore.PostgreSQLDelegate
+
 org.quartz.jobStore.tablePrefix = QRTZ_
 org.quartz.jobStore.isClustered = true
 org.quartz.jobStore.misfireThreshold = 60000
@@ -49,11 +57,5 @@ org.quartz.jobStore.dataSource = myDs
 # Configure Datasources  
 #============================================================================
 org.quartz.dataSource.myDs.connectionProvider.class = org.apache.dolphinscheduler.server.quartz.DruidConnectionProvider
-#org.quartz.dataSource.myDs.driver = com.mysql.jdbc.Driver
-org.quartz.dataSource.myDs.driver = org.postgresql.Driver
-#org.quartz.dataSource.myDs.URL = jdbc:mysql://192.168.xx.xx:3306/dolphinscheduler?characterEncoding=utf8
-org.quartz.dataSource.myDs.URL = jdbc:postgresql://192.168.xx.xx:5432/dolphinscheduler?characterEncoding=utf8
-org.quartz.dataSource.myDs.user = xx
-org.quartz.dataSource.myDs.password = xx
 org.quartz.dataSource.myDs.maxConnections = 10
 org.quartz.dataSource.myDs.validationQuery = select 1

dolphinscheduler-common/src/main/resources/zookeeper.properties

@@ -1,43 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-#zookeeper cluster.  multiple are separated by commas.  eg. 192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181
-zookeeper.quorum=localhost:2181
-
-#dolphinscheduler root directory
-zookeeper.dolphinscheduler.root=/dolphinscheduler
-
-#zookeeper server dirctory
-#zookeeper.dolphinscheduler.dead.servers=/dolphinscheduler/dead-servers
-#zookeeper.dolphinscheduler.masters=/dolphinscheduler/masters
-#zookeeper.dolphinscheduler.workers=/dolphinscheduler/workers
-
-#zookeeper lock dirctory
-#zookeeper.dolphinscheduler.lock.masters=/dolphinscheduler/lock/masters
-#zookeeper.dolphinscheduler.lock.workers=/dolphinscheduler/lock/workers
-
-#dolphinscheduler failover directory
-#zookeeper.dolphinscheduler.lock.failover.masters=/dolphinscheduler/lock/failover/masters
-#zookeeper.dolphinscheduler.lock.failover.workers=/dolphinscheduler/lock/failover/workers
-#zookeeper.dolphinscheduler.lock.failover.startup.masters=/dolphinscheduler/lock/failover/startup-masters
-
-#dolphinscheduler failover directory
-zookeeper.session.timeout=300
-zookeeper.connection.timeout=300
-zookeeper.retry.base.sleep=100
-zookeeper.retry.max.sleep=30000
-zookeeper.retry.maxtime=5

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PropertyUtils.java

@@ -49,7 +49,7 @@ public class PropertyUtils {
      * init
      */
     private void init(){
-        String[] propertyFiles = new String[]{Constants.DAO_PROPERTIES_PATH};
+        String[] propertyFiles = new String[]{Constants.APPLICATION_PROPERTIES};
         for (String fileName : propertyFiles) {
             InputStream fis = null;
             try {

dolphinscheduler-server/src/main/resources/application-master.properties

@@ -1,18 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-logging.config=classpath:master_logback.xml

dolphinscheduler-server/src/main/resources/application-worker.properties

@@ -1,18 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-logging.config=classpath:worker_logback.xml

install.sh

@@ -218,30 +218,6 @@ keytabPath="$installPath/conf/hdfs.headless.keytab"
 # zk root directory
 zkRoot="/dolphinscheduler"
 
-# used to record the zk directory of the hanging machine
-zkDeadServers="$zkRoot/dead-servers"
-
-# masters directory
-zkMasters="$zkRoot/masters"
-
-# workers directory
-zkWorkers="$zkRoot/workers"
-
-# zk master distributed lock
-mastersLock="$zkRoot/lock/masters"
-
-# zk worker distributed lock
-workersLock="$zkRoot/lock/workers"
-
-# zk master fault-tolerant distributed lock
-mastersFailover="$zkRoot/lock/failover/masters"
-
-# zk worker fault-tolerant distributed lock
-workersFailover="$zkRoot/lock/failover/workers"
-
-# zk master start fault tolerant distributed lock
-mastersStartupFailover="$zkRoot/lock/failover/startup-masters"
-
 # zk session timeout
 zkSessionTimeout="300"
 
@@ -321,11 +297,10 @@ apiMaxHttpPostSize="5000000"
 # 1,replace file
 echo "1,replace file"
 if [ $dbtype == "mysql" ];then
-    sed -i ${txt} "s#spring.datasource.url.*#spring.datasource.url=jdbc:mysql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/application-dao.properties
-    sed -i ${txt} "s#spring.datasource.username.*#spring.datasource.username=${username}#g" conf/application-dao.properties
-    sed -i ${txt} "s#spring.datasource.password.*#spring.datasource.password=${passowrd}#g" conf/application-dao.properties
-    sed -i ${txt} "s#spring.datasource.driver-class-name.*#spring.datasource.driver-class-name=com.mysql.jdbc.Driver#g" conf/application-dao.properties
-
+    sed -i ${txt} "s#spring.datasource.url.*#spring.datasource.url=jdbc:mysql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/application.properties
+    sed -i ${txt} "s#spring.datasource.username.*#spring.datasource.username=${username}#g" conf/application.properties
+    sed -i ${txt} "s#spring.datasource.password.*#spring.datasource.password=${passowrd}#g" conf/application.properties
+    sed -i ${txt} "s#spring.datasource.driver-class-name.*#spring.datasource.driver-class-name=com.mysql.jdbc.Driver#g" conf/application.properties
 
     sed -i ${txt} "s#org.quartz.dataSource.myDs.URL.*#org.quartz.dataSource.myDs.URL=jdbc:mysql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/quartz.properties
     sed -i ${txt} "s#org.quartz.dataSource.myDs.user.*#org.quartz.dataSource.myDs.user=${username}#g" conf/quartz.properties
@@ -335,10 +310,10 @@ if [ $dbtype == "mysql" ];then
 fi
 
 if [ $dbtype == "postgresql" ];then
-    sed -i ${txt} "s#spring.datasource.url.*#spring.datasource.url=jdbc:postgresql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/application-dao.properties
-    sed -i ${txt} "s#spring.datasource.username.*#spring.datasource.username=${username}#g" conf/application-dao.properties
-    sed -i ${txt} "s#spring.datasource.password.*#spring.datasource.password=${passowrd}#g" conf/application-dao.properties
-    sed -i ${txt} "s#spring.datasource.driver-class-name.*#spring.datasource.driver-class-name=org.postgresql.Driver#g" conf/application-dao.properties
+    sed -i ${txt} "s#spring.datasource.url.*#spring.datasource.url=jdbc:postgresql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/application.properties
+    sed -i ${txt} "s#spring.datasource.username.*#spring.datasource.username=${username}#g" conf/application.properties
+    sed -i ${txt} "s#spring.datasource.password.*#spring.datasource.password=${passowrd}#g" conf/application.properties
+    sed -i ${txt} "s#spring.datasource.driver-class-name.*#spring.datasource.driver-class-name=org.postgresql.Driver#g" conf/application.properties
 
     sed -i ${txt} "s#org.quartz.dataSource.myDs.URL.*#org.quartz.dataSource.myDs.URL=jdbc:postgresql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/quartz.properties
     sed -i ${txt} "s#org.quartz.dataSource.myDs.user.*#org.quartz.dataSource.myDs.user=${username}#g" conf/quartz.properties
@@ -349,46 +324,34 @@ fi
 
 
 
-sed -i ${txt} "s#fs.defaultFS.*#fs.defaultFS=${defaultFS}#g" conf/common/hadoop/hadoop.properties
-sed -i ${txt} "s#fs.s3a.endpoint.*#fs.s3a.endpoint=${s3Endpoint}#g" conf/common/hadoop/hadoop.properties
-sed -i ${txt} "s#fs.s3a.access.key.*#fs.s3a.access.key=${s3AccessKey}#g" conf/common/hadoop/hadoop.properties
-sed -i ${txt} "s#fs.s3a.secret.key.*#fs.s3a.secret.key=${s3SecretKey}#g" conf/common/hadoop/hadoop.properties
-sed -i ${txt} "s#yarn.resourcemanager.ha.rm.ids.*#yarn.resourcemanager.ha.rm.ids=${yarnHaIps}#g" conf/common/hadoop/hadoop.properties
-sed -i ${txt} "s#yarn.application.status.address.*#yarn.application.status.address=http://${singleYarnIp}:8088/ws/v1/cluster/apps/%s#g" conf/common/hadoop/hadoop.properties
-
-
-sed -i ${txt} "s#data.basedir.path.*#data.basedir.path=${programPath}#g" conf/common/common.properties
-sed -i ${txt} "s#data.download.basedir.path.*#data.download.basedir.path=${downloadPath}#g" conf/common/common.properties
-sed -i ${txt} "s#process.exec.basepath.*#process.exec.basepath=${execPath}#g" conf/common/common.properties
-sed -i ${txt} "s#hdfs.root.user.*#hdfs.root.user=${hdfsRootUser}#g" conf/common/common.properties
-sed -i ${txt} "s#data.store2hdfs.basepath.*#data.store2hdfs.basepath=${hdfsPath}#g" conf/common/common.properties
-sed -i ${txt} "s#res.upload.startup.type.*#res.upload.startup.type=${resUploadStartupType}#g" conf/common/common.properties
-sed -i ${txt} "s#dolphinscheduler.env.path.*#dolphinscheduler.env.path=${shellEnvPath}#g" conf/common/common.properties
-sed -i ${txt} "s#resource.view.suffixs.*#resource.view.suffixs=${resSuffixs}#g" conf/common/common.properties
-sed -i ${txt} "s#development.state.*#development.state=${devState}#g" conf/common/common.properties
-sed -i ${txt} "s#hadoop.security.authentication.startup.state.*#hadoop.security.authentication.startup.state=${kerberosStartUp}#g" conf/common/common.properties
-sed -i ${txt} "s#java.security.krb5.conf.path.*#java.security.krb5.conf.path=${krb5ConfPath}#g" conf/common/common.properties
-sed -i ${txt} "s#login.user.keytab.username.*#login.user.keytab.username=${keytabUserName}#g" conf/common/common.properties
-sed -i ${txt} "s#login.user.keytab.path.*#login.user.keytab.path=${keytabPath}#g" conf/common/common.properties
-
-sed -i ${txt} "s#zookeeper.quorum.*#zookeeper.quorum=${zkQuorum}#g" conf/zookeeper.properties
-sed -i ${txt} "s#zookeeper.dolphinscheduler.root.*#zookeeper.dolphinscheduler.root=${zkRoot}#g" conf/zookeeper.properties
-sed -i ${txt} "s#zookeeper.dolphinscheduler.dead.servers.*#zookeeper.dolphinscheduler.dead.servers=${zkDeadServers}#g" conf/zookeeper.properties
-sed -i ${txt} "s#zookeeper.dolphinscheduler.masters.*#zookeeper.dolphinscheduler.masters=${zkMasters}#g" conf/zookeeper.properties
-sed -i ${txt} "s#zookeeper.dolphinscheduler.workers.*#zookeeper.dolphinscheduler.workers=${zkWorkers}#g" conf/zookeeper.properties
-sed -i ${txt} "s#zookeeper.dolphinscheduler.lock.masters.*#zookeeper.dolphinscheduler.lock.masters=${mastersLock}#g" conf/zookeeper.properties
-sed -i ${txt} "s#zookeeper.dolphinscheduler.lock.workers.*#zookeeper.dolphinscheduler.lock.workers=${workersLock}#g" conf/zookeeper.properties
-sed -i ${txt} "s#zookeeper.dolphinscheduler.lock.failover.masters.*#zookeeper.dolphinscheduler.lock.failover.masters=${mastersFailover}#g" conf/zookeeper.properties
-sed -i ${txt} "s#zookeeper.dolphinscheduler.lock.failover.workers.*#zookeeper.dolphinscheduler.lock.failover.workers=${workersFailover}#g" conf/zookeeper.properties
-sed -i ${txt} "s#zookeeper.dolphinscheduler.lock.failover.startup.masters.*#zookeeper.dolphinscheduler.lock.failover.startup.masters=${mastersStartupFailover}#g" conf/zookeeper.properties
-sed -i ${txt} "s#zookeeper.session.timeout.*#zookeeper.session.timeout=${zkSessionTimeout}#g" conf/zookeeper.properties
-sed -i ${txt} "s#zookeeper.connection.timeout.*#zookeeper.connection.timeout=${zkConnectionTimeout}#g" conf/zookeeper.properties
-sed -i ${txt} "s#zookeeper.retry.sleep.*#zookeeper.retry.sleep=${zkRetrySleep}#g" conf/zookeeper.properties
-sed -i ${txt} "s#zookeeper.retry.maxtime.*#zookeeper.retry.maxtime=${zkRetryMaxtime}#g" conf/zookeeper.properties
-
-sed -i ${txt} "s#server.port.*#server.port=${masterPort}#g" conf/application-master.properties
-sed -i ${txt} "s#server.port.*#server.port=${workerPort}#g" conf/application-worker.properties
-
+sed -i ${txt} "s#fs.defaultFS.*#fs.defaultFS=${defaultFS}#g" conf/common.properties
+sed -i ${txt} "s#fs.s3a.endpoint.*#fs.s3a.endpoint=${s3Endpoint}#g" conf/common.properties
+sed -i ${txt} "s#fs.s3a.access.key.*#fs.s3a.access.key=${s3AccessKey}#g" conf/common.properties
+sed -i ${txt} "s#fs.s3a.secret.key.*#fs.s3a.secret.key=${s3SecretKey}#g" conf/common.properties
+sed -i ${txt} "s#yarn.resourcemanager.ha.rm.ids.*#yarn.resourcemanager.ha.rm.ids=${yarnHaIps}#g" conf/common.properties
+sed -i ${txt} "s#yarn.application.status.address.*#yarn.application.status.address=http://${singleYarnIp}:8088/ws/v1/cluster/apps/%s#g" conf/common.properties
+
+
+sed -i ${txt} "s#data.basedir.path.*#data.basedir.path=${programPath}#g" conf/common.properties
+sed -i ${txt} "s#data.download.basedir.path.*#data.download.basedir.path=${downloadPath}#g" conf/common.properties
+sed -i ${txt} "s#process.exec.basepath.*#process.exec.basepath=${execPath}#g" conf/common.properties
+sed -i ${txt} "s#hdfs.root.user.*#hdfs.root.user=${hdfsRootUser}#g" conf/common.properties
+sed -i ${txt} "s#data.store2hdfs.basepath.*#data.store2hdfs.basepath=${hdfsPath}#g" conf/common.properties
+sed -i ${txt} "s#res.upload.startup.type.*#res.upload.startup.type=${resUploadStartupType}#g" conf/common.properties
+sed -i ${txt} "s#dolphinscheduler.env.path.*#dolphinscheduler.env.path=${shellEnvPath}#g" conf/common.properties
+sed -i ${txt} "s#resource.view.suffixs.*#resource.view.suffixs=${resSuffixs}#g" conf/common.properties
+sed -i ${txt} "s#development.state.*#development.state=${devState}#g" conf/common.properties
+sed -i ${txt} "s#hadoop.security.authentication.startup.state.*#hadoop.security.authentication.startup.state=${kerberosStartUp}#g" conf/common.properties
+sed -i ${txt} "s#java.security.krb5.conf.path.*#java.security.krb5.conf.path=${krb5ConfPath}#g" conf/common.properties
+sed -i ${txt} "s#login.user.keytab.username.*#login.user.keytab.username=${keytabUserName}#g" conf/common.properties
+sed -i ${txt} "s#login.user.keytab.path.*#login.user.keytab.path=${keytabPath}#g" conf/common.properties
+
+sed -i ${txt} "s#zookeeper.quorum.*#zookeeper.quorum=${zkQuorum}#g" conf/common.properties
+sed -i ${txt} "s#zookeeper.dolphinscheduler.root.*#zookeeper.dolphinscheduler.root=${zkRoot}#g" conf/common.properties
+sed -i ${txt} "s#zookeeper.session.timeout.*#zookeeper.session.timeout=${zkSessionTimeout}#g" conf/common.properties
+sed -i ${txt} "s#zookeeper.connection.timeout.*#zookeeper.connection.timeout=${zkConnectionTimeout}#g" conf/common.properties
+sed -i ${txt} "s#zookeeper.retry.sleep.*#zookeeper.retry.sleep=${zkRetrySleep}#g" conf/common.properties
+sed -i ${txt} "s#zookeeper.retry.maxtime.*#zookeeper.retry.maxtime=${zkRetryMaxtime}#g" conf/common.properties
 
 sed -i ${txt} "s#server.port.*#server.port=${apiServerPort}#g" conf/application-api.properties
 sed -i ${txt} "s#server.servlet.session.timeout.*#server.servlet.session.timeout=${apiServerSessionTimeout}#g" conf/application-api.properties

script/dolphinscheduler-daemon.sh

@@ -57,13 +57,13 @@ pid=$DOLPHINSCHEDULER_LOG_DIR/dolphinscheduler-$command.pid
 cd $DOLPHINSCHEDULER_HOME
 
 if [ "$command" = "api-server" ]; then
-  LOG_FILE="-Dspring.profiles.active=api"
+  LOG_FILE="-Dlogging.config=classpath:apiserver_logback.xml -Dspring.profiles.active=api"
   CLASS=org.apache.dolphinscheduler.api.ApiApplicationServer
 elif [ "$command" = "master-server" ]; then
-  LOG_FILE="-Dspring.profiles.active=master -Ddruid.mysql.usePingMethod=false"
+  LOG_FILE="-Dlogging.config=classpath:master_logback.xml -Ddruid.mysql.usePingMethod=false"
   CLASS=org.apache.dolphinscheduler.server.master.MasterServer
 elif [ "$command" = "worker-server" ]; then
-  LOG_FILE="-Dspring.profiles.active=worker -Ddruid.mysql.usePingMethod=false"
+  LOG_FILE="-Dlogging.config=classpath:worker_logback.xml -Ddruid.mysql.usePingMethod=false"
   CLASS=org.apache.dolphinscheduler.server.worker.WorkerServer
 elif [ "$command" = "alert-server" ]; then
   LOG_FILE="-Dlogback.configurationFile=conf/alert_logback.xml"
@@ -71,7 +71,7 @@ elif [ "$command" = "alert-server" ]; then
 elif [ "$command" = "logger-server" ]; then
   CLASS=org.apache.dolphinscheduler.server.rpc.LoggerServer
 elif [ "$command" = "combined-server" ]; then
-  LOG_FILE="-Dspring.profiles.active=combined"
+  LOG_FILE="-Dlogging.config=classpath:combined_logback.xml -Dspring.profiles.active=api -Dserver.is-combined-server=true"
   CLASS=org.apache.dolphinscheduler.api.CombinedApplicationServer
 else
   echo "Error: No command named \`$command' was found."