diff --git a/dolphinscheduler-api/src/main/resources/application-combined.properties b/dolphinscheduler-api/src/main/resources/application-combined.properties deleted file mode 100644 index 0264c4f4b6..0000000000 --- a/dolphinscheduler-api/src/main/resources/application-combined.properties +++ /dev/null @@ -1,40 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -logging.config=classpath:combined_logback.xml - -# server port -server.port=12345 - -# session config -server.servlet.session.timeout=7200 - -server.servlet.context-path=/dolphinscheduler/ - -# file size limit for upload -spring.servlet.multipart.max-file-size=1024MB -spring.servlet.multipart.max-request-size=1024MB - -#post content -server.jetty.max-http-post-size=5000000 - -spring.messages.encoding=UTF-8 - -#i18n classpath folder , file prefix messages, if have many files, use "," seperator -spring.messages.basename=i18n/messages - -server.is-combined-server=true diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java index 6073f218a4..1222faa1a7 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java @@ -26,25 +26,10 @@ import java.util.regex.Pattern; */ public final class Constants { - /** - * zookeeper properties path - */ - public static final String ZOOKEEPER_PROPERTIES_PATH = "zookeeper.properties"; - - /** - * hadoop properties path - */ - public static final String HADOOP_PROPERTIES_PATH = "/common/hadoop/hadoop.properties"; - /** * common properties path */ - public static final String COMMON_PROPERTIES_PATH = "/common/common.properties"; - - /** - * dao properties path - */ - public static final String DAO_PROPERTIES_PATH = "application.properties"; + public static final String COMMON_PROPERTIES_PATH = "/common.properties"; /** * fs.defaultFS @@ -227,6 +212,8 @@ public final class Constants { public static final String ZOOKEEPER_CONNECTION_TIMEOUT = "zookeeper.connection.timeout"; public static final String ZOOKEEPER_RETRY_SLEEP = "zookeeper.retry.sleep"; + public static final String ZOOKEEPER_RETRY_BASE_SLEEP = "zookeeper.retry.base.sleep"; + public static final String ZOOKEEPER_RETRY_MAX_SLEEP = "zookeeper.retry.max.sleep"; public static final String ZOOKEEPER_RETRY_MAXTIME = "zookeeper.retry.maxtime"; @@ -469,7 +456,7 @@ public final class Constants { /** * task record configuration path */ - public static final String APPLICATION_PROPERTIES = "application-dao.properties"; + public static final String APPLICATION_PROPERTIES = "application.properties"; public static final String TASK_RECORD_URL = "task.record.datasource.url"; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java index ea01d025a7..c3e8197079 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java @@ -29,7 +29,6 @@ import java.util.Map; import java.util.Properties; import static org.apache.dolphinscheduler.common.Constants.COMMON_PROPERTIES_PATH; -import static org.apache.dolphinscheduler.common.Constants.HADOOP_PROPERTIES_PATH; /** * property utils @@ -51,7 +50,7 @@ public class PropertyUtils { } private void init(){ - String[] propertyFiles = new String[]{HADOOP_PROPERTIES_PATH,COMMON_PROPERTIES_PATH}; + String[] propertyFiles = new String[]{COMMON_PROPERTIES_PATH}; for (String fileName : propertyFiles) { InputStream fis = null; try { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperConfig.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperConfig.java index a90a147425..75a9f6c5f4 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperConfig.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperConfig.java @@ -24,7 +24,7 @@ import org.springframework.stereotype.Component; * zookeeper conf */ @Component -@PropertySource("classpath:zookeeper.properties") +@PropertySource("classpath:common.properties") public class ZookeeperConfig { //zk connect config diff --git a/dolphinscheduler-common/src/main/resources/common/common.properties b/dolphinscheduler-common/src/main/resources/common.properties similarity index 67% rename from dolphinscheduler-common/src/main/resources/common/common.properties rename to dolphinscheduler-common/src/main/resources/common.properties index 01e321167d..9826c56593 100644 --- a/dolphinscheduler-common/src/main/resources/common/common.properties +++ b/dolphinscheduler-common/src/main/resources/common.properties @@ -18,14 +18,21 @@ #task queue implementation, default "zookeeper" dolphinscheduler.queue.impl=zookeeper -# user data directory path, self configuration, please make sure the directory exists and have read write permissions -data.basedir.path=/tmp/dolphinscheduler +#zookeeper cluster. multiple are separated by commas. eg. 192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181 +zookeeper.quorum=localhost:2181 -# directory path for user data download. self configuration, please make sure the directory exists and have read write permissions -data.download.basedir.path=/tmp/dolphinscheduler/download +#dolphinscheduler root directory +zookeeper.dolphinscheduler.root=/dolphinscheduler -# process execute directory. self configuration, please make sure the directory exists and have read write permissions -process.exec.basepath=/tmp/dolphinscheduler/exec +#dolphinscheduler failover directory +zookeeper.session.timeout=300 +zookeeper.connection.timeout=300 +zookeeper.retry.base.sleep=100 +zookeeper.retry.max.sleep=30000 +zookeeper.retry.maxtime=5 + +# resource upload startup type : HDFS,S3,NONE +res.upload.startup.type=NONE # Users who have permission to create directories under the HDFS root path hdfs.root.user=hdfs @@ -33,8 +40,15 @@ hdfs.root.user=hdfs # data base dir, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended data.store2hdfs.basepath=/dolphinscheduler -# resource upload startup type : HDFS,S3,NONE -res.upload.startup.type=NONE +# user data directory path, self configuration, please make sure the directory exists and have read write permissions +data.basedir.path=/tmp/dolphinscheduler + +# directory path for user data download. self configuration, please make sure the directory exists and have read write permissions +data.download.basedir.path=/tmp/dolphinscheduler/download + +# process execute directory. self configuration, please make sure the directory exists and have read write permissions +process.exec.basepath=/tmp/dolphinscheduler/exec + # whether kerberos starts hadoop.security.authentication.startup.state=false @@ -57,3 +71,24 @@ resource.view.suffixs=txt,log,sh,conf,cfg,py,java,sql,hql,xml,properties # is development state? default "false" development.state=true + +# ha or single namenode,If namenode ha needs to copy core-site.xml and hdfs-site.xml +# to the conf directory,support s3,for example : s3a://dolphinscheduler +fs.defaultFS=hdfs://mycluster:8020 + +# s3 need,s3 endpoint +fs.s3a.endpoint=http://192.168.199.91:9010 + +# s3 need,s3 access key +fs.s3a.access.key=A3DXS30FO22544RE + +# s3 need,s3 secret key +fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK + +#resourcemanager ha note this need ips , this empty if single +yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx + +# If it is a single resourcemanager, you only need to configure one host name. If it is resourcemanager HA, the default configuration is fine +yarn.application.status.address=http://ark1:8088/ws/v1/cluster/apps/%s + + diff --git a/dolphinscheduler-common/src/main/resources/common/hadoop/hadoop.properties b/dolphinscheduler-common/src/main/resources/common/hadoop/hadoop.properties deleted file mode 100644 index 2c19b4a52e..0000000000 --- a/dolphinscheduler-common/src/main/resources/common/hadoop/hadoop.properties +++ /dev/null @@ -1,35 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# ha or single namenode,If namenode ha needs to copy core-site.xml and hdfs-site.xml -# to the conf directory,support s3,for example : s3a://dolphinscheduler -fs.defaultFS=hdfs://mycluster:8020 - -# s3 need,s3 endpoint -fs.s3a.endpoint=http://192.168.199.91:9010 - -# s3 need,s3 access key -fs.s3a.access.key=A3DXS30FO22544RE - -# s3 need,s3 secret key -fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK - -#resourcemanager ha note this need ips , this empty if single -yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx - -# If it is a single resourcemanager, you only need to configure one host name. If it is resourcemanager HA, the default configuration is fine -yarn.application.status.address=http://ark1:8088/ws/v1/cluster/apps/%s \ No newline at end of file diff --git a/dolphinscheduler-common/src/main/resources/quartz.properties b/dolphinscheduler-common/src/main/resources/quartz.properties index d388691c3f..684aabcd66 100644 --- a/dolphinscheduler-common/src/main/resources/quartz.properties +++ b/dolphinscheduler-common/src/main/resources/quartz.properties @@ -18,6 +18,15 @@ #============================================================================ # Configure Main Scheduler Properties #============================================================================ +#org.quartz.jobStore.driverDelegateClass = org.quartz.impl.jdbcjobstore.StdJDBCDelegate +org.quartz.jobStore.driverDelegateClass = org.quartz.impl.jdbcjobstore.PostgreSQLDelegate +#org.quartz.dataSource.myDs.driver = com.mysql.jdbc.Driver +org.quartz.dataSource.myDs.driver = org.postgresql.Driver +#org.quartz.dataSource.myDs.URL = jdbc:mysql://192.168.xx.xx:3306/dolphinscheduler?characterEncoding=utf8 +org.quartz.dataSource.myDs.URL = jdbc:postgresql://localhost:5432/dolphinscheduler?characterEncoding=utf8 +org.quartz.dataSource.myDs.user = test +org.quartz.dataSource.myDs.password = test + org.quartz.scheduler.instanceName = DolphinScheduler org.quartz.scheduler.instanceId = AUTO org.quartz.scheduler.makeSchedulerThreadDaemon = true @@ -37,8 +46,7 @@ org.quartz.threadPool.threadPriority = 5 #============================================================================ org.quartz.jobStore.class = org.quartz.impl.jdbcjobstore.JobStoreTX -#org.quartz.jobStore.driverDelegateClass = org.quartz.impl.jdbcjobstore.StdJDBCDelegate -org.quartz.jobStore.driverDelegateClass = org.quartz.impl.jdbcjobstore.PostgreSQLDelegate + org.quartz.jobStore.tablePrefix = QRTZ_ org.quartz.jobStore.isClustered = true org.quartz.jobStore.misfireThreshold = 60000 @@ -49,11 +57,5 @@ org.quartz.jobStore.dataSource = myDs # Configure Datasources #============================================================================ org.quartz.dataSource.myDs.connectionProvider.class = org.apache.dolphinscheduler.server.quartz.DruidConnectionProvider -#org.quartz.dataSource.myDs.driver = com.mysql.jdbc.Driver -org.quartz.dataSource.myDs.driver = org.postgresql.Driver -#org.quartz.dataSource.myDs.URL = jdbc:mysql://192.168.xx.xx:3306/dolphinscheduler?characterEncoding=utf8 -org.quartz.dataSource.myDs.URL = jdbc:postgresql://192.168.xx.xx:5432/dolphinscheduler?characterEncoding=utf8 -org.quartz.dataSource.myDs.user = xx -org.quartz.dataSource.myDs.password = xx org.quartz.dataSource.myDs.maxConnections = 10 org.quartz.dataSource.myDs.validationQuery = select 1 \ No newline at end of file diff --git a/dolphinscheduler-common/src/main/resources/zookeeper.properties b/dolphinscheduler-common/src/main/resources/zookeeper.properties deleted file mode 100644 index a560de4f23..0000000000 --- a/dolphinscheduler-common/src/main/resources/zookeeper.properties +++ /dev/null @@ -1,43 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -#zookeeper cluster. multiple are separated by commas. eg. 192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181 -zookeeper.quorum=localhost:2181 - -#dolphinscheduler root directory -zookeeper.dolphinscheduler.root=/dolphinscheduler - -#zookeeper server dirctory -#zookeeper.dolphinscheduler.dead.servers=/dolphinscheduler/dead-servers -#zookeeper.dolphinscheduler.masters=/dolphinscheduler/masters -#zookeeper.dolphinscheduler.workers=/dolphinscheduler/workers - -#zookeeper lock dirctory -#zookeeper.dolphinscheduler.lock.masters=/dolphinscheduler/lock/masters -#zookeeper.dolphinscheduler.lock.workers=/dolphinscheduler/lock/workers - -#dolphinscheduler failover directory -#zookeeper.dolphinscheduler.lock.failover.masters=/dolphinscheduler/lock/failover/masters -#zookeeper.dolphinscheduler.lock.failover.workers=/dolphinscheduler/lock/failover/workers -#zookeeper.dolphinscheduler.lock.failover.startup.masters=/dolphinscheduler/lock/failover/startup-masters - -#dolphinscheduler failover directory -zookeeper.session.timeout=300 -zookeeper.connection.timeout=300 -zookeeper.retry.base.sleep=100 -zookeeper.retry.max.sleep=30000 -zookeeper.retry.maxtime=5 \ No newline at end of file diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PropertyUtils.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PropertyUtils.java index e92491e096..cdd481a5d7 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PropertyUtils.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PropertyUtils.java @@ -49,7 +49,7 @@ public class PropertyUtils { * init */ private void init(){ - String[] propertyFiles = new String[]{Constants.DAO_PROPERTIES_PATH}; + String[] propertyFiles = new String[]{Constants.APPLICATION_PROPERTIES}; for (String fileName : propertyFiles) { InputStream fis = null; try { diff --git a/dolphinscheduler-dao/src/main/resources/application-dao.properties b/dolphinscheduler-dao/src/main/resources/application.properties similarity index 100% rename from dolphinscheduler-dao/src/main/resources/application-dao.properties rename to dolphinscheduler-dao/src/main/resources/application.properties diff --git a/dolphinscheduler-server/src/main/resources/application-master.properties b/dolphinscheduler-server/src/main/resources/application-master.properties deleted file mode 100644 index 49d28e810f..0000000000 --- a/dolphinscheduler-server/src/main/resources/application-master.properties +++ /dev/null @@ -1,18 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -logging.config=classpath:master_logback.xml \ No newline at end of file diff --git a/dolphinscheduler-server/src/main/resources/application-worker.properties b/dolphinscheduler-server/src/main/resources/application-worker.properties deleted file mode 100644 index be4319d204..0000000000 --- a/dolphinscheduler-server/src/main/resources/application-worker.properties +++ /dev/null @@ -1,18 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -logging.config=classpath:worker_logback.xml diff --git a/install.sh b/install.sh index b73ca09079..85e470051f 100644 --- a/install.sh +++ b/install.sh @@ -218,30 +218,6 @@ keytabPath="$installPath/conf/hdfs.headless.keytab" # zk root directory zkRoot="/dolphinscheduler" -# used to record the zk directory of the hanging machine -zkDeadServers="$zkRoot/dead-servers" - -# masters directory -zkMasters="$zkRoot/masters" - -# workers directory -zkWorkers="$zkRoot/workers" - -# zk master distributed lock -mastersLock="$zkRoot/lock/masters" - -# zk worker distributed lock -workersLock="$zkRoot/lock/workers" - -# zk master fault-tolerant distributed lock -mastersFailover="$zkRoot/lock/failover/masters" - -# zk worker fault-tolerant distributed lock -workersFailover="$zkRoot/lock/failover/workers" - -# zk master start fault tolerant distributed lock -mastersStartupFailover="$zkRoot/lock/failover/startup-masters" - # zk session timeout zkSessionTimeout="300" @@ -321,11 +297,10 @@ apiMaxHttpPostSize="5000000" # 1,replace file echo "1,replace file" if [ $dbtype == "mysql" ];then - sed -i ${txt} "s#spring.datasource.url.*#spring.datasource.url=jdbc:mysql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/application-dao.properties - sed -i ${txt} "s#spring.datasource.username.*#spring.datasource.username=${username}#g" conf/application-dao.properties - sed -i ${txt} "s#spring.datasource.password.*#spring.datasource.password=${passowrd}#g" conf/application-dao.properties - sed -i ${txt} "s#spring.datasource.driver-class-name.*#spring.datasource.driver-class-name=com.mysql.jdbc.Driver#g" conf/application-dao.properties - + sed -i ${txt} "s#spring.datasource.url.*#spring.datasource.url=jdbc:mysql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/application.properties + sed -i ${txt} "s#spring.datasource.username.*#spring.datasource.username=${username}#g" conf/application.properties + sed -i ${txt} "s#spring.datasource.password.*#spring.datasource.password=${passowrd}#g" conf/application.properties + sed -i ${txt} "s#spring.datasource.driver-class-name.*#spring.datasource.driver-class-name=com.mysql.jdbc.Driver#g" conf/application.properties sed -i ${txt} "s#org.quartz.dataSource.myDs.URL.*#org.quartz.dataSource.myDs.URL=jdbc:mysql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/quartz.properties sed -i ${txt} "s#org.quartz.dataSource.myDs.user.*#org.quartz.dataSource.myDs.user=${username}#g" conf/quartz.properties @@ -335,10 +310,10 @@ if [ $dbtype == "mysql" ];then fi if [ $dbtype == "postgresql" ];then - sed -i ${txt} "s#spring.datasource.url.*#spring.datasource.url=jdbc:postgresql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/application-dao.properties - sed -i ${txt} "s#spring.datasource.username.*#spring.datasource.username=${username}#g" conf/application-dao.properties - sed -i ${txt} "s#spring.datasource.password.*#spring.datasource.password=${passowrd}#g" conf/application-dao.properties - sed -i ${txt} "s#spring.datasource.driver-class-name.*#spring.datasource.driver-class-name=org.postgresql.Driver#g" conf/application-dao.properties + sed -i ${txt} "s#spring.datasource.url.*#spring.datasource.url=jdbc:postgresql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/application.properties + sed -i ${txt} "s#spring.datasource.username.*#spring.datasource.username=${username}#g" conf/application.properties + sed -i ${txt} "s#spring.datasource.password.*#spring.datasource.password=${passowrd}#g" conf/application.properties + sed -i ${txt} "s#spring.datasource.driver-class-name.*#spring.datasource.driver-class-name=org.postgresql.Driver#g" conf/application.properties sed -i ${txt} "s#org.quartz.dataSource.myDs.URL.*#org.quartz.dataSource.myDs.URL=jdbc:postgresql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/quartz.properties sed -i ${txt} "s#org.quartz.dataSource.myDs.user.*#org.quartz.dataSource.myDs.user=${username}#g" conf/quartz.properties @@ -349,46 +324,34 @@ fi -sed -i ${txt} "s#fs.defaultFS.*#fs.defaultFS=${defaultFS}#g" conf/common/hadoop/hadoop.properties -sed -i ${txt} "s#fs.s3a.endpoint.*#fs.s3a.endpoint=${s3Endpoint}#g" conf/common/hadoop/hadoop.properties -sed -i ${txt} "s#fs.s3a.access.key.*#fs.s3a.access.key=${s3AccessKey}#g" conf/common/hadoop/hadoop.properties -sed -i ${txt} "s#fs.s3a.secret.key.*#fs.s3a.secret.key=${s3SecretKey}#g" conf/common/hadoop/hadoop.properties -sed -i ${txt} "s#yarn.resourcemanager.ha.rm.ids.*#yarn.resourcemanager.ha.rm.ids=${yarnHaIps}#g" conf/common/hadoop/hadoop.properties -sed -i ${txt} "s#yarn.application.status.address.*#yarn.application.status.address=http://${singleYarnIp}:8088/ws/v1/cluster/apps/%s#g" conf/common/hadoop/hadoop.properties - - -sed -i ${txt} "s#data.basedir.path.*#data.basedir.path=${programPath}#g" conf/common/common.properties -sed -i ${txt} "s#data.download.basedir.path.*#data.download.basedir.path=${downloadPath}#g" conf/common/common.properties -sed -i ${txt} "s#process.exec.basepath.*#process.exec.basepath=${execPath}#g" conf/common/common.properties -sed -i ${txt} "s#hdfs.root.user.*#hdfs.root.user=${hdfsRootUser}#g" conf/common/common.properties -sed -i ${txt} "s#data.store2hdfs.basepath.*#data.store2hdfs.basepath=${hdfsPath}#g" conf/common/common.properties -sed -i ${txt} "s#res.upload.startup.type.*#res.upload.startup.type=${resUploadStartupType}#g" conf/common/common.properties -sed -i ${txt} "s#dolphinscheduler.env.path.*#dolphinscheduler.env.path=${shellEnvPath}#g" conf/common/common.properties -sed -i ${txt} "s#resource.view.suffixs.*#resource.view.suffixs=${resSuffixs}#g" conf/common/common.properties -sed -i ${txt} "s#development.state.*#development.state=${devState}#g" conf/common/common.properties -sed -i ${txt} "s#hadoop.security.authentication.startup.state.*#hadoop.security.authentication.startup.state=${kerberosStartUp}#g" conf/common/common.properties -sed -i ${txt} "s#java.security.krb5.conf.path.*#java.security.krb5.conf.path=${krb5ConfPath}#g" conf/common/common.properties -sed -i ${txt} "s#login.user.keytab.username.*#login.user.keytab.username=${keytabUserName}#g" conf/common/common.properties -sed -i ${txt} "s#login.user.keytab.path.*#login.user.keytab.path=${keytabPath}#g" conf/common/common.properties - -sed -i ${txt} "s#zookeeper.quorum.*#zookeeper.quorum=${zkQuorum}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.dolphinscheduler.root.*#zookeeper.dolphinscheduler.root=${zkRoot}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.dolphinscheduler.dead.servers.*#zookeeper.dolphinscheduler.dead.servers=${zkDeadServers}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.dolphinscheduler.masters.*#zookeeper.dolphinscheduler.masters=${zkMasters}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.dolphinscheduler.workers.*#zookeeper.dolphinscheduler.workers=${zkWorkers}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.dolphinscheduler.lock.masters.*#zookeeper.dolphinscheduler.lock.masters=${mastersLock}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.dolphinscheduler.lock.workers.*#zookeeper.dolphinscheduler.lock.workers=${workersLock}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.dolphinscheduler.lock.failover.masters.*#zookeeper.dolphinscheduler.lock.failover.masters=${mastersFailover}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.dolphinscheduler.lock.failover.workers.*#zookeeper.dolphinscheduler.lock.failover.workers=${workersFailover}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.dolphinscheduler.lock.failover.startup.masters.*#zookeeper.dolphinscheduler.lock.failover.startup.masters=${mastersStartupFailover}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.session.timeout.*#zookeeper.session.timeout=${zkSessionTimeout}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.connection.timeout.*#zookeeper.connection.timeout=${zkConnectionTimeout}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.retry.sleep.*#zookeeper.retry.sleep=${zkRetrySleep}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.retry.maxtime.*#zookeeper.retry.maxtime=${zkRetryMaxtime}#g" conf/zookeeper.properties - -sed -i ${txt} "s#server.port.*#server.port=${masterPort}#g" conf/application-master.properties -sed -i ${txt} "s#server.port.*#server.port=${workerPort}#g" conf/application-worker.properties - +sed -i ${txt} "s#fs.defaultFS.*#fs.defaultFS=${defaultFS}#g" conf/common.properties +sed -i ${txt} "s#fs.s3a.endpoint.*#fs.s3a.endpoint=${s3Endpoint}#g" conf/common.properties +sed -i ${txt} "s#fs.s3a.access.key.*#fs.s3a.access.key=${s3AccessKey}#g" conf/common.properties +sed -i ${txt} "s#fs.s3a.secret.key.*#fs.s3a.secret.key=${s3SecretKey}#g" conf/common.properties +sed -i ${txt} "s#yarn.resourcemanager.ha.rm.ids.*#yarn.resourcemanager.ha.rm.ids=${yarnHaIps}#g" conf/common.properties +sed -i ${txt} "s#yarn.application.status.address.*#yarn.application.status.address=http://${singleYarnIp}:8088/ws/v1/cluster/apps/%s#g" conf/common.properties + + +sed -i ${txt} "s#data.basedir.path.*#data.basedir.path=${programPath}#g" conf/common.properties +sed -i ${txt} "s#data.download.basedir.path.*#data.download.basedir.path=${downloadPath}#g" conf/common.properties +sed -i ${txt} "s#process.exec.basepath.*#process.exec.basepath=${execPath}#g" conf/common.properties +sed -i ${txt} "s#hdfs.root.user.*#hdfs.root.user=${hdfsRootUser}#g" conf/common.properties +sed -i ${txt} "s#data.store2hdfs.basepath.*#data.store2hdfs.basepath=${hdfsPath}#g" conf/common.properties +sed -i ${txt} "s#res.upload.startup.type.*#res.upload.startup.type=${resUploadStartupType}#g" conf/common.properties +sed -i ${txt} "s#dolphinscheduler.env.path.*#dolphinscheduler.env.path=${shellEnvPath}#g" conf/common.properties +sed -i ${txt} "s#resource.view.suffixs.*#resource.view.suffixs=${resSuffixs}#g" conf/common.properties +sed -i ${txt} "s#development.state.*#development.state=${devState}#g" conf/common.properties +sed -i ${txt} "s#hadoop.security.authentication.startup.state.*#hadoop.security.authentication.startup.state=${kerberosStartUp}#g" conf/common.properties +sed -i ${txt} "s#java.security.krb5.conf.path.*#java.security.krb5.conf.path=${krb5ConfPath}#g" conf/common.properties +sed -i ${txt} "s#login.user.keytab.username.*#login.user.keytab.username=${keytabUserName}#g" conf/common.properties +sed -i ${txt} "s#login.user.keytab.path.*#login.user.keytab.path=${keytabPath}#g" conf/common.properties + +sed -i ${txt} "s#zookeeper.quorum.*#zookeeper.quorum=${zkQuorum}#g" conf/common.properties +sed -i ${txt} "s#zookeeper.dolphinscheduler.root.*#zookeeper.dolphinscheduler.root=${zkRoot}#g" conf/common.properties +sed -i ${txt} "s#zookeeper.session.timeout.*#zookeeper.session.timeout=${zkSessionTimeout}#g" conf/common.properties +sed -i ${txt} "s#zookeeper.connection.timeout.*#zookeeper.connection.timeout=${zkConnectionTimeout}#g" conf/common.properties +sed -i ${txt} "s#zookeeper.retry.sleep.*#zookeeper.retry.sleep=${zkRetrySleep}#g" conf/common.properties +sed -i ${txt} "s#zookeeper.retry.maxtime.*#zookeeper.retry.maxtime=${zkRetryMaxtime}#g" conf/common.properties sed -i ${txt} "s#server.port.*#server.port=${apiServerPort}#g" conf/application-api.properties sed -i ${txt} "s#server.servlet.session.timeout.*#server.servlet.session.timeout=${apiServerSessionTimeout}#g" conf/application-api.properties diff --git a/script/dolphinscheduler-daemon.sh b/script/dolphinscheduler-daemon.sh index 292f2eca39..b3310c384f 100644 --- a/script/dolphinscheduler-daemon.sh +++ b/script/dolphinscheduler-daemon.sh @@ -57,13 +57,13 @@ pid=$DOLPHINSCHEDULER_LOG_DIR/dolphinscheduler-$command.pid cd $DOLPHINSCHEDULER_HOME if [ "$command" = "api-server" ]; then - LOG_FILE="-Dspring.profiles.active=api" + LOG_FILE="-Dlogging.config=classpath:apiserver_logback.xml -Dspring.profiles.active=api" CLASS=org.apache.dolphinscheduler.api.ApiApplicationServer elif [ "$command" = "master-server" ]; then - LOG_FILE="-Dspring.profiles.active=master -Ddruid.mysql.usePingMethod=false" + LOG_FILE="-Dlogging.config=classpath:master_logback.xml -Ddruid.mysql.usePingMethod=false" CLASS=org.apache.dolphinscheduler.server.master.MasterServer elif [ "$command" = "worker-server" ]; then - LOG_FILE="-Dspring.profiles.active=worker -Ddruid.mysql.usePingMethod=false" + LOG_FILE="-Dlogging.config=classpath:worker_logback.xml -Ddruid.mysql.usePingMethod=false" CLASS=org.apache.dolphinscheduler.server.worker.WorkerServer elif [ "$command" = "alert-server" ]; then LOG_FILE="-Dlogback.configurationFile=conf/alert_logback.xml" @@ -71,7 +71,7 @@ elif [ "$command" = "alert-server" ]; then elif [ "$command" = "logger-server" ]; then CLASS=org.apache.dolphinscheduler.server.rpc.LoggerServer elif [ "$command" = "combined-server" ]; then - LOG_FILE="-Dspring.profiles.active=combined" + LOG_FILE="-Dlogging.config=classpath:combined_logback.xml -Dspring.profiles.active=api -Dserver.is-combined-server=true" CLASS=org.apache.dolphinscheduler.api.CombinedApplicationServer else echo "Error: No command named \`$command' was found."