diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java index 864af98bc9..f56b409ad5 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java @@ -92,6 +92,15 @@ public final class Constants { public static final String FS_S3A_SECRET_KEY = "fs.s3a.secret.key"; + /** + * hadoop configuration + */ + public static final String HADOOP_RM_STATE_ACTIVE = "ACTIVE"; + + public static final String HADOOP_RM_STATE_STANDBY = "STANDBY"; + + public static final String HADOOP_RESOURCE_MANAGER_HTTPADDRESS_PORT = "resource.manager.httpaddress.port"; + /** * yarn.resourcemanager.ha.rm.ids */ @@ -258,15 +267,6 @@ public final class Constants { public static final String AT_SIGN = "@"; - public static final String WORKER_MAX_CPULOAD_AVG = "worker.max.cpuload.avg"; - - public static final String WORKER_RESERVED_MEMORY = "worker.reserved.memory"; - - public static final String MASTER_MAX_CPULOAD_AVG = "master.max.cpuload.avg"; - - public static final String MASTER_RESERVED_MEMORY = "master.reserved.memory"; - - /** * date format of yyyy-MM-dd HH:mm:ss */ @@ -466,15 +466,6 @@ public final class Constants { */ public static final String CMDPARAM_COMPLEMENT_DATA_END_DATE = "complementEndDate"; - /** - * hadoop configuration - */ - public static final String HADOOP_RM_STATE_ACTIVE = "ACTIVE"; - - public static final String HADOOP_RM_STATE_STANDBY = "STANDBY"; - - public static final String HADOOP_RESOURCE_MANAGER_HTTPADDRESS_PORT = "resource.manager.httpaddress.port"; - /** * data source config diff --git a/dolphinscheduler-server/src/main/resources/config/install_config.conf b/dolphinscheduler-server/src/main/resources/config/install_config.conf index 236617fa96..f722a8cf0f 100644 --- a/dolphinscheduler-server/src/main/resources/config/install_config.conf +++ b/dolphinscheduler-server/src/main/resources/config/install_config.conf @@ -73,10 +73,15 @@ sslEnable="false" #note: sslTrust is the same as mailServerHost sslTrust="smtp.exmail.qq.com" +# user data local directory path, please make sure the directory exists and have read write permissions +dataBasedirPath="/tmp/dolphinscheduler" # resource storage type: HDFS, S3, NONE resourceStorageType="NONE" +# resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended +resourceUploadPath="/dolphinscheduler" + # if resourceStorageType is HDFS,defaultFS write namenode address,HA you need to put core-site.xml and hdfs-site.xml in the conf directory. # if S3,write S3 address,HA,for example :s3a://dolphinscheduler, # Note,s3 be sure to create the root directory /dolphinscheduler @@ -87,15 +92,15 @@ s3Endpoint="http://192.168.xx.xx:9010" s3AccessKey="xxxxxxxxxx" s3SecretKey="xxxxxxxxxx" +# resourcemanager port, the default value is 8088 if not specified +resourceManagerHttpAddressPort="8088" + # if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single, keep this value empty yarnHaIps="192.168.xx.xx,192.168.xx.xx" # if resourcemanager HA is enabled or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname singleYarnIp="yarnIp1" -# resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended -resourceUploadPath="/dolphinscheduler" - # who have permissions to create directory under HDFS/S3 root path # Note: if kerberos is enabled, please config hdfsRootUser= hdfsRootUser="hdfs" @@ -109,7 +114,8 @@ krb5ConfPath="$installPath/conf/krb5.conf" keytabUserName="hdfs-mycluster@ESZ.COM" # username keytab path keytabPath="$installPath/conf/hdfs.headless.keytab" - +# kerberos expire time, the unit is hour +kerberosExpireTime="2" # api server port apiServerPort="12345" diff --git a/dolphinscheduler-service/src/main/resources/zookeeper.properties b/dolphinscheduler-service/src/main/resources/zookeeper.properties index 006b12036d..ad1fb8e93e 100644 --- a/dolphinscheduler-service/src/main/resources/zookeeper.properties +++ b/dolphinscheduler-service/src/main/resources/zookeeper.properties @@ -19,7 +19,7 @@ zookeeper.quorum=localhost:2181 # dolphinscheduler root directory -#zookeeper.dolphinscheduler.root=/dolphinscheduler +zookeeper.dolphinscheduler.root=/dolphinscheduler # dolphinscheduler failover directory #zookeeper.session.timeout=60000 diff --git a/install.sh b/install.sh index a74fe67531..df0dd9e078 100755 --- a/install.sh +++ b/install.sh @@ -34,24 +34,27 @@ datasourceDriverClassname="com.mysql.jdbc.Driver" if [ $dbtype == "postgresql" ];then datasourceDriverClassname="org.postgresql.Driver" fi -sed -i ${txt} "s#spring.datasource.driver-class-name.*#spring.datasource.driver-class-name=${datasourceDriverClassname}#g" conf/datasource.properties -sed -i ${txt} "s#spring.datasource.url.*#spring.datasource.url=jdbc:${dbtype}://${dbhost}/${dbname}?characterEncoding=UTF-8\&allowMultiQueries=true#g" conf/datasource.properties -sed -i ${txt} "s#spring.datasource.username.*#spring.datasource.username=${username}#g" conf/datasource.properties -sed -i ${txt} "s#spring.datasource.password.*#spring.datasource.password=${password}#g" conf/datasource.properties +sed -i ${txt} "s#^spring.datasource.driver-class-name.*#spring.datasource.driver-class-name=${datasourceDriverClassname}#g" conf/datasource.properties +sed -i ${txt} "s#^spring.datasource.url.*#spring.datasource.url=jdbc:${dbtype}://${dbhost}/${dbname}?characterEncoding=UTF-8\&allowMultiQueries=true#g" conf/datasource.properties +sed -i ${txt} "s#^spring.datasource.username.*#spring.datasource.username=${username}#g" conf/datasource.properties +sed -i ${txt} "s#^spring.datasource.password.*#spring.datasource.password=${password}#g" conf/datasource.properties sed -i ${txt} "s#fs.defaultFS.*#fs.defaultFS=${defaultFS}#g" conf/common.properties sed -i ${txt} "s#fs.s3a.endpoint.*#fs.s3a.endpoint=${s3Endpoint}#g" conf/common.properties sed -i ${txt} "s#fs.s3a.access.key.*#fs.s3a.access.key=${s3AccessKey}#g" conf/common.properties sed -i ${txt} "s#fs.s3a.secret.key.*#fs.s3a.secret.key=${s3SecretKey}#g" conf/common.properties +sed -i ${txt} "s#resource.manager.httpaddress.port.*#resource.manager.httpaddress.port=${resourceManagerHttpAddressPort}#g" conf/common.properties sed -i ${txt} "s#yarn.resourcemanager.ha.rm.ids.*#yarn.resourcemanager.ha.rm.ids=${yarnHaIps}#g" conf/common.properties -sed -i ${txt} "s#yarn.application.status.address.*#yarn.application.status.address=http://${singleYarnIp}:8088/ws/v1/cluster/apps/%s#g" conf/common.properties +sed -i ${txt} "s#yarn.application.status.address.*#yarn.application.status.address=http://${singleYarnIp}:%s/ws/v1/cluster/apps/%s#g" conf/common.properties sed -i ${txt} "s#hdfs.root.user.*#hdfs.root.user=${hdfsRootUser}#g" conf/common.properties sed -i ${txt} "s#resource.upload.path.*#resource.upload.path=${resourceUploadPath}#g" conf/common.properties +sed -i ${txt} "s#data.basedir.path=.*#data.basedir.path=${dataBasedirPath}#g" conf/common.properties sed -i ${txt} "s#resource.storage.type.*#resource.storage.type=${resourceStorageType}#g" conf/common.properties sed -i ${txt} "s#hadoop.security.authentication.startup.state.*#hadoop.security.authentication.startup.state=${kerberosStartUp}#g" conf/common.properties sed -i ${txt} "s#java.security.krb5.conf.path.*#java.security.krb5.conf.path=${krb5ConfPath}#g" conf/common.properties sed -i ${txt} "s#login.user.keytab.username.*#login.user.keytab.username=${keytabUserName}#g" conf/common.properties sed -i ${txt} "s#login.user.keytab.path.*#login.user.keytab.path=${keytabPath}#g" conf/common.properties +sed -i ${txt} "s#kerberos.expire.time=.*#kerberos.expire.time=${kerberosExpireTime}#g" conf/common.properties sed -i ${txt} "s#zookeeper.quorum.*#zookeeper.quorum=${zkQuorum}#g" conf/zookeeper.properties sed -i ${txt} "s#server.port.*#server.port=${apiServerPort}#g" conf/application-api.properties sed -i ${txt} "s#mail.server.host.*#mail.server.host=${mailServerHost}#g" conf/alert.properties diff --git a/script/scp-hosts.sh b/script/scp-hosts.sh index 258ebe5325..21d5701fd0 100755 --- a/script/scp-hosts.sh +++ b/script/scp-hosts.sh @@ -58,7 +58,7 @@ do do # if worker in workersGroupMap if [[ "${workersGroupMap[${host}]}" ]] && [[ "${dsDir}" == "conf" ]]; then - sed -i ${txt} "s#worker.groups.*#worker.groups=${workersGroupMap[${host}]}#g" ${dsDir}/worker.properties + sed -i ${txt} "s@^#\?worker.groups=.*@worker.groups=${workersGroupMap[${host}]}@g" ${dsDir}/worker.properties fi echo "start to scp $dsDir to $host/$installPath"