Browse Source

dockerfile optimization (#1165)

* add ConnectionFactoryTest and ConnectionFactory read datasource from appliction.yml

* .escheduler_env.sh to dolphinscheduler_env.sh

* dao yml assembly to conf directory

* table name modify

* entity title table  name modify

* logback log name modify

* running through the big process

* running through the big process error modify

* logback log name modify

* data_source.properties rename

* logback log name modify

* install.sh optimization

* install.sh optimization

* command count modify

* command state update

* countCommandState sql update

* countCommandState sql update

* remove application.yml file

* master.properties modify

* install.sh modify

* install.sh modify

* api server startup modify

* the current user quits and the session is completely emptied. bug fix

* remove pom package resources

* checkQueueNameExist method update

* checkQueueExist

* install.sh error output update

* signOut error update

* ProcessDao is null bug fix

* install.sh add mail.user

* request url variables replace

* process define import bug fix

* process define import export bug fix

* processdefine import export bug fix

* down log suffix format modify

* import export process define contains crontab error bug fix

* add Flink local mode

* ProcessDao is null bug fix

* loadAverage display problem bug fix

* MasterServer rename Server

* rollback .env

* rollback .env

* MasterServer rename Server

* the task is abnormal and task is running bug fix

* owners and administrators can delete

* dockerfile optimization

* dockerfile optimization

* dockerfile optimization
pull/2/head
qiaozhanwei 5 years ago committed by bao liang
parent
commit
5956bfcc66
  1. 55
      dockerfile/Dockerfile
  2. 53
      dockerfile/conf/dolphinscheduler/conf/dao/data_source.properties
  3. 0
      dockerfile/conf/nginx/dolphinscheduler.conf
  4. 32
      dockerfile/startup.sh

55
dockerfile/Dockerfile

@ -1,13 +1,11 @@
FROM ubuntu:18.04
MAINTAINER journey "825193156@qq.com"
ENV LANG=C.UTF-8
ARG version
ARG tar_version
#1,安装jdk
#1,install jdk
RUN apt-get update \
&& apt-get -y install openjdk-8-jdk \
@ -17,17 +15,10 @@ ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
ENV PATH $JAVA_HOME/bin:$PATH
#安装wget
#install wget
RUN apt-get update && \
apt-get -y install wget
#2,安装ZK
#RUN cd /opt && \
# wget https://archive.apache.org/dist/zookeeper/zookeeper-3.4.6/zookeeper-3.4.6.tar.gz && \
# tar -zxvf zookeeper-3.4.6.tar.gz && \
# mv zookeeper-3.4.6 zookeeper && \
# rm -rf ./zookeeper-*tar.gz && \
# mkdir -p /tmp/zookeeper && \
# rm -rf /opt/zookeeper/conf/zoo_sample.cfg
#2,install ZK
RUN cd /opt && \
wget https://www-us.apache.org/dist/zookeeper/zookeeper-3.4.14/zookeeper-3.4.14.tar.gz && \
@ -41,7 +32,7 @@ ADD ./dockerfile/conf/zookeeper/zoo.cfg /opt/zookeeper/conf
ENV ZK_HOME=/opt/zookeeper
ENV PATH $PATH:$ZK_HOME/bin
#3,安装maven
#3,install maven
RUN cd /opt && \
wget http://apache-mirror.rbc.ru/pub/apache/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz && \
tar -zxvf apache-maven-3.3.9-bin.tar.gz && \
@ -52,7 +43,7 @@ ADD ./dockerfile/conf/maven/settings.xml /opt/maven/conf
ENV MAVEN_HOME=/opt/maven
ENV PATH $PATH:$MAVEN_HOME/bin
#4,安装node
#4,install node
RUN cd /opt && \
wget https://nodejs.org/download/release/v8.9.4/node-v8.9.4-linux-x64.tar.gz && \
tar -zxvf node-v8.9.4-linux-x64.tar.gz && \
@ -61,20 +52,22 @@ RUN cd /opt && \
ENV NODE_HOME=/opt/node
ENV PATH $PATH:$NODE_HOME/bin
ADD . /opt/easyscheduler_source
#5,add dolphinscheduler source code to /opt/dolphinscheduler_source
ADD . /opt/dolphinscheduler_source
#6,后端编译
RUN cd /opt/easyscheduler_source && \
#5,backend compilation
RUN cd /opt/dolphinscheduler_source && \
mvn -U clean package assembly:assembly -Dmaven.test.skip=true
#7,前端编译
RUN chmod -R 777 /opt/easyscheduler_source/dolphinscheduler-ui && \
cd /opt/easyscheduler_source/dolphinscheduler-ui && \
rm -rf /opt/easyscheduler_source/dolphinscheduler-ui/node_modules && \
#6,frontend compilation
RUN chmod -R 777 /opt/dolphinscheduler_source/dolphinscheduler-ui && \
cd /opt/dolphinscheduler_source/dolphinscheduler-ui && \
rm -rf /opt/dolphinscheduler_source/dolphinscheduler-ui/node_modules && \
npm install node-sass --unsafe-perm && \
npm install && \
npm run build
#8,安装mysql
#7,install mysql
RUN echo "deb http://cn.archive.ubuntu.com/ubuntu/ xenial main restricted universe multiverse" >> /etc/apt/sources.list
RUN echo "mysql-server mysql-server/root_password password root" | debconf-set-selections
@ -97,26 +90,26 @@ RUN sed -i -e "$ a [client]\n\n[mysql]\n\n[mysqld]" /etc/mysql/my.cnf && \
sed -i -e "s/\(\[mysqld\]\)/\1\ninit_connect='SET NAMES utf8'\ncharacter-set-server = utf8\ncollation-server=utf8_general_ci\nbind-address = 0.0.0.0/g" /etc/mysql/my.cnf
#9,安装nginx
#8,install nginx
RUN apt-get update && \
apt-get install -y nginx && \
rm -rf /var/lib/apt/lists/* && \
echo "\ndaemon off;" >> /etc/nginx/nginx.conf && \
chown -R www-data:www-data /var/lib/nginx
#10,修改dolphinscheduler配置文件
#后端配置
#9,modify dolphinscheduler configuration file
#backend configuration
RUN mkdir -p /opt/dolphinscheduler && \
tar -zxvf /opt/easyscheduler_source/target/dolphinscheduler-${tar_version}.tar.gz -C /opt/dolphinscheduler && \
tar -zxvf /opt/dolphinscheduler_source/target/dolphinscheduler-${tar_version}.tar.gz -C /opt/dolphinscheduler && \
rm -rf /opt/dolphinscheduler/conf
ADD ./dockerfile/conf/dolphinscheduler/conf /opt/dolphinscheduler/conf
#前端nginx配置
ADD ./dockerfile/conf/nginx/default.conf /etc/nginx/conf.d
#frontend nginx configuration
ADD ./dockerfile/conf/nginx/dolphinscheduler.conf /etc/nginx/conf.d
#11,开放端口
#10,open port
EXPOSE 2181 2888 3888 3306 80 12345 8888
#12,安装sudo,python,vim,ping和ssh
#11,install sudo,python,vim,ping and ssh command
RUN apt-get update && \
apt-get -y install sudo && \
apt-get -y install python && \
@ -128,7 +121,7 @@ RUN apt-get update && \
pip install kazoo
COPY ./dockerfile/startup.sh /root/startup.sh
#13,修改权限和设置软连
#12,modify permissions and set soft links
RUN chmod +x /root/startup.sh && \
chmod +x /opt/dolphinscheduler/script/create-dolphinscheduler.sh && \
chmod +x /opt/zookeeper/bin/zkServer.sh && \

53
dockerfile/conf/dolphinscheduler/conf/dao/data_source.properties

@ -1,53 +0,0 @@
# base spring data source configuration
spring.datasource.type=com.alibaba.druid.pool.DruidDataSource
spring.datasource.driver-class-name=com.mysql.jdbc.Driver
spring.datasource.url=jdbc:mysql://127.0.0.1:3306/dolphinscheduler?characterEncoding=UTF-8
spring.datasource.username=root
spring.datasource.password=root@123
# connection configuration
spring.datasource.initialSize=5
# min connection number
spring.datasource.minIdle=5
# max connection number
spring.datasource.maxActive=50
# max wait time for get a connection in milliseconds. if configuring maxWait, fair locks are enabled by default and concurrency efficiency decreases.
# If necessary, unfair locks can be used by configuring the useUnfairLock attribute to true.
spring.datasource.maxWait=60000
# milliseconds for check to close free connections
spring.datasource.timeBetweenEvictionRunsMillis=60000
# the Destroy thread detects the connection interval and closes the physical connection in milliseconds if the connection idle time is greater than or equal to minEvictableIdleTimeMillis.
spring.datasource.timeBetweenConnectErrorMillis=60000
# the longest time a connection remains idle without being evicted, in milliseconds
spring.datasource.minEvictableIdleTimeMillis=300000
#the SQL used to check whether the connection is valid requires a query statement. If validation Query is null, testOnBorrow, testOnReturn, and testWhileIdle will not work.
spring.datasource.validationQuery=SELECT 1
#check whether the connection is valid for timeout, in seconds
spring.datasource.validationQueryTimeout=3
# when applying for a connection, if it is detected that the connection is idle longer than time Between Eviction Runs Millis,
# validation Query is performed to check whether the connection is valid
spring.datasource.testWhileIdle=true
#execute validation to check if the connection is valid when applying for a connection
spring.datasource.testOnBorrow=true
#execute validation to check if the connection is valid when the connection is returned
spring.datasource.testOnReturn=false
spring.datasource.defaultAutoCommit=true
spring.datasource.keepAlive=true
# open PSCache, specify count PSCache for every connection
spring.datasource.poolPreparedStatements=true
spring.datasource.maxPoolPreparedStatementPerConnectionSize=20
# data quality analysis is not currently in use. please ignore the following configuration
# task record flag
task.record.flag=false
task.record.datasource.url=jdbc:mysql://192.168.xx.xx:3306/etl?characterEncoding=UTF-8
task.record.datasource.username=xx
task.record.datasource.password=xx

0
dockerfile/conf/nginx/default.conf → dockerfile/conf/nginx/dolphinscheduler.conf

32
dockerfile/startup.sh

@ -5,31 +5,31 @@ if [ `netstat -anop|grep mysql|wc -l` -gt 0 ];then
echo "MySQL is Running."
else
MYSQL_ROOT_PWD="root@123"
ESZ_DB="dolphinscheduler"
echo "启动mysql服务"
ESZ_DB="dolphinscheduler"
echo "start mysql service"
chown -R mysql:mysql /var/lib/mysql /var/run/mysqld
find /var/lib/mysql -type f -exec touch {} \; && service mysql restart $ sleep 10
if [ ! -f /nohup.out ];then
echo "设置mysql密码"
echo "setting mysql password"
mysql --user=root --password=root -e "UPDATE mysql.user set authentication_string=password('$MYSQL_ROOT_PWD') where user='root'; FLUSH PRIVILEGES;"
echo "设置mysql权限"
echo "setting mysql permission"
mysql --user=root --password=$MYSQL_ROOT_PWD -e "GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY '$MYSQL_ROOT_PWD' WITH GRANT OPTION; FLUSH PRIVILEGES;"
echo "创建dolphinscheduler数据库"
echo "create dolphinscheduler database"
mysql --user=root --password=$MYSQL_ROOT_PWD -e "CREATE DATABASE IF NOT EXISTS \`$ESZ_DB\` CHARACTER SET utf8 COLLATE utf8_general_ci; FLUSH PRIVILEGES;"
echo "导入mysql数据"
echo "import mysql data"
nohup /opt/dolphinscheduler/script/create-dolphinscheduler.sh &
sleep 90
fi
if [ `mysql --user=root --password=$MYSQL_ROOT_PWD -s -r -e "SELECT count(TABLE_NAME) FROM information_schema.TABLES WHERE TABLE_SCHEMA='dolphinscheduler';" | grep -v count` -eq 38 ];then
echo "\`$ESZ_DB\` 表个数正确"
echo "\`$ESZ_DB\` the number of tables is correct"
else
echo "\`$ESZ_DB\` 表个数不正确"
echo "\`$ESZ_DB\` the number of tables is incorrect"
mysql --user=root --password=$MYSQL_ROOT_PWD -e "DROP DATABASE \`$ESZ_DB\`;"
echo "创建dolphinscheduler数据库"
echo "create dolphinscheduler database"
mysql --user=root --password=$MYSQL_ROOT_PWD -e "CREATE DATABASE IF NOT EXISTS \`$ESZ_DB\` CHARACTER SET utf8 COLLATE utf8_general_ci; FLUSH PRIVILEGES;"
echo "导入mysql数据"
echo "import mysql data"
nohup /opt/dolphinscheduler/script/create-dolphinscheduler.sh &
sleep 90
fi
@ -39,29 +39,29 @@ fi
sleep 90
echo "启动api-server"
echo "start api-server"
/opt/dolphinscheduler/bin/dolphinscheduler-daemon.sh stop api-server
/opt/dolphinscheduler/bin/dolphinscheduler-daemon.sh start api-server
echo "启动master-server"
echo "start master-server"
/opt/dolphinscheduler/bin/dolphinscheduler-daemon.sh stop master-server
python /opt/dolphinscheduler/script/del-zk-node.py 127.0.0.1 /dolphinscheduler/masters
/opt/dolphinscheduler/bin/dolphinscheduler-daemon.sh start master-server
echo "启动worker-server"
echo "start worker-server"
/opt/dolphinscheduler/bin/dolphinscheduler-daemon.sh stop worker-server
python /opt/dolphinscheduler/script/del-zk-node.py 127.0.0.1 /dolphinscheduler/workers
/opt/dolphinscheduler/bin/dolphinscheduler-daemon.sh start worker-server
echo "启动logger-server"
echo "start logger-server"
/opt/dolphinscheduler/bin/dolphinscheduler-daemon.sh stop logger-server
/opt/dolphinscheduler/bin/dolphinscheduler-daemon.sh start logger-server
echo "启动alert-server"
echo "start alert-server"
/opt/dolphinscheduler/bin/dolphinscheduler-daemon.sh stop alert-server
/opt/dolphinscheduler/bin/dolphinscheduler-daemon.sh start alert-server
@ -69,7 +69,7 @@ echo "启动alert-server"
echo "启动nginx"
echo "start nginx"
/etc/init.d/nginx stop
nginx &

Loading…
Cancel
Save