xiangzihao
5 months ago
committed by
GitHub
42 changed files with 140 additions and 533 deletions
@ -1,61 +0,0 @@
|
||||
# |
||||
# Licensed to the Apache Software Foundation (ASF) under one or more |
||||
# contributor license agreements. See the NOTICE file distributed with |
||||
# this work for additional information regarding copyright ownership. |
||||
# The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
# (the "License"); you may not use this file except in compliance with |
||||
# the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
# |
||||
|
||||
# --------------------------------------------------------- |
||||
# INSTALL MACHINE |
||||
# --------------------------------------------------------- |
||||
# A comma separated list of machine hostname or IP would be installed DolphinScheduler, |
||||
# including master, worker, api, alert. If you want to deploy in pseudo-distributed |
||||
# mode, just write a pseudo-distributed hostname |
||||
# Example for hostnames: ips="ds1,ds2,ds3,ds4,ds5", Example for IPs: ips="192.168.8.1,192.168.8.2,192.168.8.3,192.168.8.4,192.168.8.5" |
||||
ips=${ips:-"localhost"} |
||||
|
||||
# Port of SSH protocol, default value is 22. For now we only support same port in all `ips` machine |
||||
# modify it if you use different ssh port |
||||
sshPort=${sshPort:-"22"} |
||||
|
||||
# A comma separated list of machine hostname or IP would be installed Master server, it |
||||
# must be a subset of configuration `ips`. |
||||
# Example for hostnames: masters="ds1,ds2", Example for IPs: masters="192.168.8.1,192.168.8.2" |
||||
masters=${masters:-"localhost"} |
||||
|
||||
# A comma separated list of machine <hostname>:<workerGroup> or <IP>:<workerGroup>.All hostname or IP must be a |
||||
# subset of configuration `ips`, And workerGroup have default value as `default`, but we recommend you declare behind the hosts |
||||
# Example for hostnames: workers="ds1:default,ds2:default,ds3:default", Example for IPs: workers="192.168.8.1:default,192.168.8.2:default,192.168.8.3:default" |
||||
workers=${workers:-"localhost:default"} |
||||
|
||||
# A comma separated list of machine hostname or IP would be installed Alert server, it |
||||
# must be a subset of configuration `ips`. |
||||
# Example for hostname: alertServer="ds3", Example for IP: alertServer="192.168.8.3" |
||||
alertServer=${alertServer:-"localhost"} |
||||
|
||||
# A comma separated list of machine hostname or IP would be installed API server, it |
||||
# must be a subset of configuration `ips`. |
||||
# Example for hostname: apiServers="ds1", Example for IP: apiServers="192.168.8.1" |
||||
apiServers=${apiServers:-"localhost"} |
||||
|
||||
# The directory to install DolphinScheduler for all machine we config above. It will automatically be created by `install.sh` script if not exists. |
||||
# Do not set this configuration same as the current path (pwd) |
||||
installPath=${installPath:-"/root/apache-dolphinscheduler-*-SNAPSHOT-bin"} |
||||
|
||||
# The user to deploy DolphinScheduler for all machine we config above. For now user must create by yourself before running `install.sh` |
||||
# script. The user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled than the root directory needs |
||||
# to be created by this user |
||||
deployUser=${deployUser:-"dolphinscheduler"} |
||||
|
||||
# The root of zookeeper, for now DolphinScheduler default registry server is zookeeper. |
||||
zkRoot=${zkRoot:-"/dolphinscheduler"} |
||||
@ -1,43 +0,0 @@
|
||||
#!/bin/bash |
||||
# |
||||
# Licensed to the Apache Software Foundation (ASF) under one or more |
||||
# contributor license agreements. See the NOTICE file distributed with |
||||
# this work for additional information regarding copyright ownership. |
||||
# The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
# (the "License"); you may not use this file except in compliance with |
||||
# the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
# |
||||
|
||||
BIN_DIR=$(dirname $0) |
||||
DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/..; pwd)} |
||||
|
||||
export DATABASE=${DATABASE:-h2} |
||||
source "$DOLPHINSCHEDULER_HOME/conf/dolphinscheduler_env.sh" |
||||
|
||||
JAVA_OPTS=${JAVA_OPTS:-"-server -Duser.timezone=${SPRING_JACKSON_TIME_ZONE} -Xms1g -Xmx1g -Xmn512m -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"} |
||||
|
||||
if [[ "$DOCKER" == "true" ]]; then |
||||
JAVA_OPTS="${JAVA_OPTS} -XX:-UseContainerSupport" |
||||
fi |
||||
|
||||
CP=$DOLPHINSCHEDULER_HOME/libs/standalone-server/* |
||||
for d in alert-server api-server master-server worker-server; do |
||||
for f in $DOLPHINSCHEDULER_HOME/../$d/libs/*.jar; do |
||||
JAR_FILE_NAME=${f##*/} |
||||
if [[ ! $CP =~ $JAR_FILE_NAME ]];then |
||||
CP=$CP:$f |
||||
fi |
||||
done |
||||
done |
||||
|
||||
$JAVA_HOME/bin/java $JAVA_OPTS \ |
||||
-cp "$DOLPHINSCHEDULER_HOME/conf":"$CP" \ |
||||
org.apache.dolphinscheduler.StandaloneServer |
||||
@ -1,50 +0,0 @@
|
||||
#!/bin/bash |
||||
# |
||||
# Licensed to the Apache Software Foundation (ASF) under one or more |
||||
# contributor license agreements. See the NOTICE file distributed with |
||||
# this work for additional information regarding copyright ownership. |
||||
# The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
# (the "License"); you may not use this file except in compliance with |
||||
# the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
# |
||||
|
||||
print_usage(){ |
||||
printf $"USAGE:$0 rootNode\n" |
||||
exit 1 |
||||
} |
||||
|
||||
if [ $# -ne 1 ];then |
||||
print_usage |
||||
fi |
||||
|
||||
rootNode=$1 |
||||
|
||||
BIN_DIR=`dirname $0` |
||||
BIN_DIR=`cd "$BIN_DIR"; pwd` |
||||
DOLPHINSCHEDULER_HOME=$BIN_DIR/.. |
||||
|
||||
source ${BIN_DIR}/env/install_env.sh |
||||
source ${BIN_DIR}/env/dolphinscheduler_env.sh |
||||
|
||||
export JAVA_HOME=$JAVA_HOME |
||||
|
||||
export DOLPHINSCHEDULER_CONF_DIR=$DOLPHINSCHEDULER_HOME/conf |
||||
export DOLPHINSCHEDULER_LIB_JARS=$DOLPHINSCHEDULER_HOME/api-server/libs/* |
||||
|
||||
export DOLPHINSCHEDULER_OPTS="-Xmx1g -Xms1g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 " |
||||
export STOP_TIMEOUT=5 |
||||
|
||||
CLASS=org.apache.zookeeper.ZooKeeperMain |
||||
|
||||
exec_command="$DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS -server $REGISTRY_ZOOKEEPER_CONNECT_STRING deleteall $rootNode" |
||||
|
||||
cd $DOLPHINSCHEDULER_HOME |
||||
$JAVA_HOME/bin/java $exec_command |
||||
@ -1,62 +0,0 @@
|
||||
#!/bin/bash |
||||
# |
||||
# Licensed to the Apache Software Foundation (ASF) under one or more |
||||
# contributor license agreements. See the NOTICE file distributed with |
||||
# this work for additional information regarding copyright ownership. |
||||
# The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
# (the "License"); you may not use this file except in compliance with |
||||
# the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
# |
||||
|
||||
workDir=`dirname $0` |
||||
workDir=`cd ${workDir};pwd` |
||||
|
||||
source ${workDir}/env/install_env.sh |
||||
|
||||
workersGroup=(${workers//,/ }) |
||||
for workerGroup in ${workersGroup[@]} |
||||
do |
||||
echo $workerGroup; |
||||
worker=`echo $workerGroup|awk -F':' '{print $1}'` |
||||
group=`echo $workerGroup|awk -F':' '{print $2}'` |
||||
workerNames+=($worker) |
||||
groupNames+=(${group:-default}) |
||||
done |
||||
|
||||
hostsArr=(${ips//,/ }) |
||||
for host in ${hostsArr[@]} |
||||
do |
||||
|
||||
if ! ssh -o StrictHostKeyChecking=no -p $sshPort $host test -e $installPath; then |
||||
ssh -o StrictHostKeyChecking=no -p $sshPort $host "sudo mkdir -p $installPath; sudo chown -R $deployUser:$deployUser $installPath" |
||||
fi |
||||
|
||||
echo "scp dirs to $host/$installPath starting" |
||||
for i in ${!workerNames[@]}; do |
||||
if [[ ${workerNames[$i]} == $host ]]; then |
||||
workerIndex=$i |
||||
break |
||||
fi |
||||
done |
||||
# set worker groups in application.yaml |
||||
[[ -n ${workerIndex} ]] && sed -i "s/- default/- ${groupNames[$workerIndex]}/" $workDir/../worker-server/conf/application.yaml |
||||
|
||||
for dsDir in bin master-server worker-server alert-server api-server ui tools |
||||
do |
||||
echo "start to scp $dsDir to $host/$installPath" |
||||
# Use quiet mode to reduce command line output |
||||
scp -q -P $sshPort -r $workDir/../$dsDir $host:$installPath |
||||
done |
||||
# restore worker groups to default |
||||
[[ -n ${workerIndex} ]] && sed -i "s/- ${groupNames[$workerIndex]}/- default/" $workDir/../worker-server/conf/application.yaml |
||||
|
||||
echo "scp dirs to $host/$installPath complete" |
||||
done |
||||
@ -1,58 +0,0 @@
|
||||
#!/bin/bash |
||||
# |
||||
# Licensed to the Apache Software Foundation (ASF) under one or more |
||||
# contributor license agreements. See the NOTICE file distributed with |
||||
# this work for additional information regarding copyright ownership. |
||||
# The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
# (the "License"); you may not use this file except in compliance with |
||||
# the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
# |
||||
|
||||
workDir=`dirname $0` |
||||
workDir=`cd ${workDir};pwd` |
||||
|
||||
source ${workDir}/env/install_env.sh |
||||
|
||||
workersGroup=(${workers//,/ }) |
||||
for workerGroup in ${workersGroup[@]} |
||||
do |
||||
echo $workerGroup; |
||||
worker=`echo $workerGroup|awk -F':' '{print $1}'` |
||||
workerNames+=($worker) |
||||
done |
||||
|
||||
mastersHost=(${masters//,/ }) |
||||
for master in ${mastersHost[@]} |
||||
do |
||||
echo "$master master server is starting" |
||||
ssh -o StrictHostKeyChecking=no -p $sshPort $master "cd $installPath/; bash bin/dolphinscheduler-daemon.sh start master-server;" |
||||
|
||||
done |
||||
|
||||
for worker in ${workerNames[@]} |
||||
do |
||||
echo "$worker worker server is starting" |
||||
|
||||
ssh -o StrictHostKeyChecking=no -p $sshPort $worker "cd $installPath/; bash bin/dolphinscheduler-daemon.sh start worker-server;" |
||||
done |
||||
|
||||
ssh -o StrictHostKeyChecking=no -p $sshPort $alertServer "cd $installPath/; bash bin/dolphinscheduler-daemon.sh start alert-server;" |
||||
|
||||
apiServersHost=(${apiServers//,/ }) |
||||
for apiServer in ${apiServersHost[@]} |
||||
do |
||||
echo "$apiServer api server is starting" |
||||
ssh -o StrictHostKeyChecking=no -p $sshPort $apiServer "cd $installPath/; bash bin/dolphinscheduler-daemon.sh start api-server;" |
||||
done |
||||
|
||||
# query server status |
||||
echo "query server status" |
||||
cd $installPath/; bash bin/status-all.sh |
||||
@ -1,73 +0,0 @@
|
||||
#!/bin/bash |
||||
# |
||||
# Licensed to the Apache Software Foundation (ASF) under one or more |
||||
# contributor license agreements. See the NOTICE file distributed with |
||||
# this work for additional information regarding copyright ownership. |
||||
# The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
# (the "License"); you may not use this file except in compliance with |
||||
# the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
# |
||||
|
||||
workDir=`dirname $0` |
||||
workDir=`cd ${workDir};pwd` |
||||
|
||||
source ${workDir}/env/install_env.sh |
||||
|
||||
# install_env.sh info |
||||
echo -e '\n' |
||||
echo "====================== dolphinscheduler server config =============================" |
||||
echo -e "1.dolphinscheduler server node config hosts:[ \033[1;32m ${ips} \033[0m ]" |
||||
echo -e "2.master server node config hosts:[ \033[1;32m ${masters} \033[0m ]" |
||||
echo -e "3.worker server node config hosts:[ \033[1;32m ${workers} \033[0m ]" |
||||
echo -e "4.alert server node config hosts:[ \033[1;32m ${alertServer} \033[0m ]" |
||||
echo -e "5.api server node config hosts:[ \033[1;32m ${apiServers} \033[0m ]" |
||||
|
||||
# all server check state |
||||
echo -e '\n' |
||||
echo "====================== dolphinscheduler server status =============================" |
||||
firstColumn="node server state" |
||||
echo $firstColumn |
||||
echo -e '\n' |
||||
|
||||
workersGroup=(${workers//,/ }) |
||||
for workerGroup in ${workersGroup[@]} |
||||
do |
||||
worker=`echo $workerGroup|awk -F':' '{print $1}'` |
||||
workerNames+=($worker) |
||||
done |
||||
|
||||
StateRunning="Running" |
||||
# 1.master server check state |
||||
mastersHost=(${masters//,/ }) |
||||
for master in ${mastersHost[@]} |
||||
do |
||||
masterState=`ssh -o StrictHostKeyChecking=no -p $sshPort $master "cd $installPath/; bash bin/dolphinscheduler-daemon.sh status master-server;"` |
||||
echo "$master $masterState" |
||||
done |
||||
|
||||
# 2.worker server check state |
||||
for worker in ${workerNames[@]} |
||||
do |
||||
workerState=`ssh -o StrictHostKeyChecking=no -p $sshPort $worker "cd $installPath/; bash bin/dolphinscheduler-daemon.sh status worker-server;"` |
||||
echo "$worker $workerState" |
||||
done |
||||
|
||||
# 3.alter server check state |
||||
alertState=`ssh -o StrictHostKeyChecking=no -p $sshPort $alertServer "cd $installPath/; bash bin/dolphinscheduler-daemon.sh status alert-server;"` |
||||
echo "$alertServer $alertState" |
||||
|
||||
# 4.api server check state |
||||
apiServersHost=(${apiServers//,/ }) |
||||
for apiServer in ${apiServersHost[@]} |
||||
do |
||||
apiState=`ssh -o StrictHostKeyChecking=no -p $sshPort $apiServer "cd $installPath/; bash bin/dolphinscheduler-daemon.sh status api-server;"` |
||||
echo "$apiServer $apiState" |
||||
done |
||||
@ -1,53 +0,0 @@
|
||||
#!/bin/bash |
||||
# |
||||
# Licensed to the Apache Software Foundation (ASF) under one or more |
||||
# contributor license agreements. See the NOTICE file distributed with |
||||
# this work for additional information regarding copyright ownership. |
||||
# The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
# (the "License"); you may not use this file except in compliance with |
||||
# the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
# |
||||
|
||||
workDir=`dirname $0` |
||||
workDir=`cd ${workDir};pwd` |
||||
|
||||
source ${workDir}/env/install_env.sh |
||||
|
||||
workersGroup=(${workers//,/ }) |
||||
for workerGroup in ${workersGroup[@]} |
||||
do |
||||
echo $workerGroup; |
||||
worker=`echo $workerGroup|awk -F':' '{print $1}'` |
||||
workerNames+=($worker) |
||||
done |
||||
|
||||
mastersHost=(${masters//,/ }) |
||||
for master in ${mastersHost[@]} |
||||
do |
||||
echo "$master master server is stopping" |
||||
ssh -o StrictHostKeyChecking=no -p $sshPort $master "cd $installPath/; bash bin/dolphinscheduler-daemon.sh stop master-server;" |
||||
|
||||
done |
||||
|
||||
for worker in ${workerNames[@]} |
||||
do |
||||
echo "$worker worker server is stopping" |
||||
ssh -o StrictHostKeyChecking=no -p $sshPort $worker "cd $installPath/; bash bin/dolphinscheduler-daemon.sh stop worker-server;" |
||||
done |
||||
|
||||
ssh -o StrictHostKeyChecking=no -p $sshPort $alertServer "cd $installPath/; bash bin/dolphinscheduler-daemon.sh stop alert-server;" |
||||
|
||||
apiServersHost=(${apiServers//,/ }) |
||||
for apiServer in ${apiServersHost[@]} |
||||
do |
||||
echo "$apiServer api server is stopping" |
||||
ssh -o StrictHostKeyChecking=no -p $sshPort $apiServer "cd $installPath/; bash bin/dolphinscheduler-daemon.sh stop api-server;" |
||||
done |
||||
Loading…
Reference in new issue