Browse Source

dockerfile optimization (#1165)

* add ConnectionFactoryTest and ConnectionFactory read datasource from appliction.yml

* .escheduler_env.sh to dolphinscheduler_env.sh

* dao yml assembly to conf directory

* table name modify

* entity title table  name modify

* logback log name modify

* running through the big process

* running through the big process error modify

* logback log name modify

* data_source.properties rename

* logback log name modify

* install.sh optimization

* install.sh optimization

* command count modify

* command state update

* countCommandState sql update

* countCommandState sql update

* remove application.yml file

* master.properties modify

* install.sh modify

* install.sh modify

* api server startup modify

* the current user quits and the session is completely emptied. bug fix

* remove pom package resources

* checkQueueNameExist method update

* checkQueueExist

* install.sh error output update

* signOut error update

* ProcessDao is null bug fix

* install.sh add mail.user

* request url variables replace

* process define import bug fix

* process define import export bug fix

* processdefine import export bug fix

* down log suffix format modify

* import export process define contains crontab error bug fix

* add Flink local mode

* ProcessDao is null bug fix

* loadAverage display problem bug fix

* MasterServer rename Server

* rollback .env

* rollback .env

* MasterServer rename Server

* the task is abnormal and task is running bug fix

* owners and administrators can delete

* dockerfile optimization

* dockerfile optimization

* dockerfile optimization
pull/2/head
qiaozhanwei 5 years ago committed by bao liang
parent
commit
5956bfcc66
  1. 55
      dockerfile/Dockerfile
  2. 53
      dockerfile/conf/dolphinscheduler/conf/dao/data_source.properties
  3. 0
      dockerfile/conf/nginx/dolphinscheduler.conf
  4. 30
      dockerfile/startup.sh

55
dockerfile/Dockerfile

@ -1,13 +1,11 @@
FROM ubuntu:18.04 FROM ubuntu:18.04
MAINTAINER journey "825193156@qq.com"
ENV LANG=C.UTF-8 ENV LANG=C.UTF-8
ARG version ARG version
ARG tar_version ARG tar_version
#1,安装jdk #1,install jdk
RUN apt-get update \ RUN apt-get update \
&& apt-get -y install openjdk-8-jdk \ && apt-get -y install openjdk-8-jdk \
@ -17,17 +15,10 @@ ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
ENV PATH $JAVA_HOME/bin:$PATH ENV PATH $JAVA_HOME/bin:$PATH
#安装wget #install wget
RUN apt-get update && \ RUN apt-get update && \
apt-get -y install wget apt-get -y install wget
#2,安装ZK #2,install ZK
#RUN cd /opt && \
# wget https://archive.apache.org/dist/zookeeper/zookeeper-3.4.6/zookeeper-3.4.6.tar.gz && \
# tar -zxvf zookeeper-3.4.6.tar.gz && \
# mv zookeeper-3.4.6 zookeeper && \
# rm -rf ./zookeeper-*tar.gz && \
# mkdir -p /tmp/zookeeper && \
# rm -rf /opt/zookeeper/conf/zoo_sample.cfg
RUN cd /opt && \ RUN cd /opt && \
wget https://www-us.apache.org/dist/zookeeper/zookeeper-3.4.14/zookeeper-3.4.14.tar.gz && \ wget https://www-us.apache.org/dist/zookeeper/zookeeper-3.4.14/zookeeper-3.4.14.tar.gz && \
@ -41,7 +32,7 @@ ADD ./dockerfile/conf/zookeeper/zoo.cfg /opt/zookeeper/conf
ENV ZK_HOME=/opt/zookeeper ENV ZK_HOME=/opt/zookeeper
ENV PATH $PATH:$ZK_HOME/bin ENV PATH $PATH:$ZK_HOME/bin
#3,安装maven #3,install maven
RUN cd /opt && \ RUN cd /opt && \
wget http://apache-mirror.rbc.ru/pub/apache/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz && \ wget http://apache-mirror.rbc.ru/pub/apache/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz && \
tar -zxvf apache-maven-3.3.9-bin.tar.gz && \ tar -zxvf apache-maven-3.3.9-bin.tar.gz && \
@ -52,7 +43,7 @@ ADD ./dockerfile/conf/maven/settings.xml /opt/maven/conf
ENV MAVEN_HOME=/opt/maven ENV MAVEN_HOME=/opt/maven
ENV PATH $PATH:$MAVEN_HOME/bin ENV PATH $PATH:$MAVEN_HOME/bin
#4,安装node #4,install node
RUN cd /opt && \ RUN cd /opt && \
wget https://nodejs.org/download/release/v8.9.4/node-v8.9.4-linux-x64.tar.gz && \ wget https://nodejs.org/download/release/v8.9.4/node-v8.9.4-linux-x64.tar.gz && \
tar -zxvf node-v8.9.4-linux-x64.tar.gz && \ tar -zxvf node-v8.9.4-linux-x64.tar.gz && \
@ -61,20 +52,22 @@ RUN cd /opt && \
ENV NODE_HOME=/opt/node ENV NODE_HOME=/opt/node
ENV PATH $PATH:$NODE_HOME/bin ENV PATH $PATH:$NODE_HOME/bin
ADD . /opt/easyscheduler_source #5,add dolphinscheduler source code to /opt/dolphinscheduler_source
ADD . /opt/dolphinscheduler_source
#6,后端编译 #5,backend compilation
RUN cd /opt/easyscheduler_source && \ RUN cd /opt/dolphinscheduler_source && \
mvn -U clean package assembly:assembly -Dmaven.test.skip=true mvn -U clean package assembly:assembly -Dmaven.test.skip=true
#7,前端编译 #6,frontend compilation
RUN chmod -R 777 /opt/easyscheduler_source/dolphinscheduler-ui && \ RUN chmod -R 777 /opt/dolphinscheduler_source/dolphinscheduler-ui && \
cd /opt/easyscheduler_source/dolphinscheduler-ui && \ cd /opt/dolphinscheduler_source/dolphinscheduler-ui && \
rm -rf /opt/easyscheduler_source/dolphinscheduler-ui/node_modules && \ rm -rf /opt/dolphinscheduler_source/dolphinscheduler-ui/node_modules && \
npm install node-sass --unsafe-perm && \ npm install node-sass --unsafe-perm && \
npm install && \ npm install && \
npm run build npm run build
#8,安装mysql #7,install mysql
RUN echo "deb http://cn.archive.ubuntu.com/ubuntu/ xenial main restricted universe multiverse" >> /etc/apt/sources.list RUN echo "deb http://cn.archive.ubuntu.com/ubuntu/ xenial main restricted universe multiverse" >> /etc/apt/sources.list
RUN echo "mysql-server mysql-server/root_password password root" | debconf-set-selections RUN echo "mysql-server mysql-server/root_password password root" | debconf-set-selections
@ -97,26 +90,26 @@ RUN sed -i -e "$ a [client]\n\n[mysql]\n\n[mysqld]" /etc/mysql/my.cnf && \
sed -i -e "s/\(\[mysqld\]\)/\1\ninit_connect='SET NAMES utf8'\ncharacter-set-server = utf8\ncollation-server=utf8_general_ci\nbind-address = 0.0.0.0/g" /etc/mysql/my.cnf sed -i -e "s/\(\[mysqld\]\)/\1\ninit_connect='SET NAMES utf8'\ncharacter-set-server = utf8\ncollation-server=utf8_general_ci\nbind-address = 0.0.0.0/g" /etc/mysql/my.cnf
#9,安装nginx #8,install nginx
RUN apt-get update && \ RUN apt-get update && \
apt-get install -y nginx && \ apt-get install -y nginx && \
rm -rf /var/lib/apt/lists/* && \ rm -rf /var/lib/apt/lists/* && \
echo "\ndaemon off;" >> /etc/nginx/nginx.conf && \ echo "\ndaemon off;" >> /etc/nginx/nginx.conf && \
chown -R www-data:www-data /var/lib/nginx chown -R www-data:www-data /var/lib/nginx
#10,修改dolphinscheduler配置文件 #9,modify dolphinscheduler configuration file
#后端配置 #backend configuration
RUN mkdir -p /opt/dolphinscheduler && \ RUN mkdir -p /opt/dolphinscheduler && \
tar -zxvf /opt/easyscheduler_source/target/dolphinscheduler-${tar_version}.tar.gz -C /opt/dolphinscheduler && \ tar -zxvf /opt/dolphinscheduler_source/target/dolphinscheduler-${tar_version}.tar.gz -C /opt/dolphinscheduler && \
rm -rf /opt/dolphinscheduler/conf rm -rf /opt/dolphinscheduler/conf
ADD ./dockerfile/conf/dolphinscheduler/conf /opt/dolphinscheduler/conf ADD ./dockerfile/conf/dolphinscheduler/conf /opt/dolphinscheduler/conf
#前端nginx配置 #frontend nginx configuration
ADD ./dockerfile/conf/nginx/default.conf /etc/nginx/conf.d ADD ./dockerfile/conf/nginx/dolphinscheduler.conf /etc/nginx/conf.d
#11,开放端口 #10,open port
EXPOSE 2181 2888 3888 3306 80 12345 8888 EXPOSE 2181 2888 3888 3306 80 12345 8888
#12,安装sudo,python,vim,ping和ssh #11,install sudo,python,vim,ping and ssh command
RUN apt-get update && \ RUN apt-get update && \
apt-get -y install sudo && \ apt-get -y install sudo && \
apt-get -y install python && \ apt-get -y install python && \
@ -128,7 +121,7 @@ RUN apt-get update && \
pip install kazoo pip install kazoo
COPY ./dockerfile/startup.sh /root/startup.sh COPY ./dockerfile/startup.sh /root/startup.sh
#13,修改权限和设置软连 #12,modify permissions and set soft links
RUN chmod +x /root/startup.sh && \ RUN chmod +x /root/startup.sh && \
chmod +x /opt/dolphinscheduler/script/create-dolphinscheduler.sh && \ chmod +x /opt/dolphinscheduler/script/create-dolphinscheduler.sh && \
chmod +x /opt/zookeeper/bin/zkServer.sh && \ chmod +x /opt/zookeeper/bin/zkServer.sh && \

53
dockerfile/conf/dolphinscheduler/conf/dao/data_source.properties

@ -1,53 +0,0 @@
# base spring data source configuration
spring.datasource.type=com.alibaba.druid.pool.DruidDataSource
spring.datasource.driver-class-name=com.mysql.jdbc.Driver
spring.datasource.url=jdbc:mysql://127.0.0.1:3306/dolphinscheduler?characterEncoding=UTF-8
spring.datasource.username=root
spring.datasource.password=root@123
# connection configuration
spring.datasource.initialSize=5
# min connection number
spring.datasource.minIdle=5
# max connection number
spring.datasource.maxActive=50
# max wait time for get a connection in milliseconds. if configuring maxWait, fair locks are enabled by default and concurrency efficiency decreases.
# If necessary, unfair locks can be used by configuring the useUnfairLock attribute to true.
spring.datasource.maxWait=60000
# milliseconds for check to close free connections
spring.datasource.timeBetweenEvictionRunsMillis=60000
# the Destroy thread detects the connection interval and closes the physical connection in milliseconds if the connection idle time is greater than or equal to minEvictableIdleTimeMillis.
spring.datasource.timeBetweenConnectErrorMillis=60000
# the longest time a connection remains idle without being evicted, in milliseconds
spring.datasource.minEvictableIdleTimeMillis=300000
#the SQL used to check whether the connection is valid requires a query statement. If validation Query is null, testOnBorrow, testOnReturn, and testWhileIdle will not work.
spring.datasource.validationQuery=SELECT 1
#check whether the connection is valid for timeout, in seconds
spring.datasource.validationQueryTimeout=3
# when applying for a connection, if it is detected that the connection is idle longer than time Between Eviction Runs Millis,
# validation Query is performed to check whether the connection is valid
spring.datasource.testWhileIdle=true
#execute validation to check if the connection is valid when applying for a connection
spring.datasource.testOnBorrow=true
#execute validation to check if the connection is valid when the connection is returned
spring.datasource.testOnReturn=false
spring.datasource.defaultAutoCommit=true
spring.datasource.keepAlive=true
# open PSCache, specify count PSCache for every connection
spring.datasource.poolPreparedStatements=true
spring.datasource.maxPoolPreparedStatementPerConnectionSize=20
# data quality analysis is not currently in use. please ignore the following configuration
# task record flag
task.record.flag=false
task.record.datasource.url=jdbc:mysql://192.168.xx.xx:3306/etl?characterEncoding=UTF-8
task.record.datasource.username=xx
task.record.datasource.password=xx

0
dockerfile/conf/nginx/default.conf → dockerfile/conf/nginx/dolphinscheduler.conf

30
dockerfile/startup.sh

@ -6,30 +6,30 @@ if [ `netstat -anop|grep mysql|wc -l` -gt 0 ];then
else else
MYSQL_ROOT_PWD="root@123" MYSQL_ROOT_PWD="root@123"
ESZ_DB="dolphinscheduler" ESZ_DB="dolphinscheduler"
echo "启动mysql服务" echo "start mysql service"
chown -R mysql:mysql /var/lib/mysql /var/run/mysqld chown -R mysql:mysql /var/lib/mysql /var/run/mysqld
find /var/lib/mysql -type f -exec touch {} \; && service mysql restart $ sleep 10 find /var/lib/mysql -type f -exec touch {} \; && service mysql restart $ sleep 10
if [ ! -f /nohup.out ];then if [ ! -f /nohup.out ];then
echo "设置mysql密码" echo "setting mysql password"
mysql --user=root --password=root -e "UPDATE mysql.user set authentication_string=password('$MYSQL_ROOT_PWD') where user='root'; FLUSH PRIVILEGES;" mysql --user=root --password=root -e "UPDATE mysql.user set authentication_string=password('$MYSQL_ROOT_PWD') where user='root'; FLUSH PRIVILEGES;"
echo "设置mysql权限" echo "setting mysql permission"
mysql --user=root --password=$MYSQL_ROOT_PWD -e "GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY '$MYSQL_ROOT_PWD' WITH GRANT OPTION; FLUSH PRIVILEGES;" mysql --user=root --password=$MYSQL_ROOT_PWD -e "GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY '$MYSQL_ROOT_PWD' WITH GRANT OPTION; FLUSH PRIVILEGES;"
echo "创建dolphinscheduler数据库" echo "create dolphinscheduler database"
mysql --user=root --password=$MYSQL_ROOT_PWD -e "CREATE DATABASE IF NOT EXISTS \`$ESZ_DB\` CHARACTER SET utf8 COLLATE utf8_general_ci; FLUSH PRIVILEGES;" mysql --user=root --password=$MYSQL_ROOT_PWD -e "CREATE DATABASE IF NOT EXISTS \`$ESZ_DB\` CHARACTER SET utf8 COLLATE utf8_general_ci; FLUSH PRIVILEGES;"
echo "导入mysql数据" echo "import mysql data"
nohup /opt/dolphinscheduler/script/create-dolphinscheduler.sh & nohup /opt/dolphinscheduler/script/create-dolphinscheduler.sh &
sleep 90 sleep 90
fi fi
if [ `mysql --user=root --password=$MYSQL_ROOT_PWD -s -r -e "SELECT count(TABLE_NAME) FROM information_schema.TABLES WHERE TABLE_SCHEMA='dolphinscheduler';" | grep -v count` -eq 38 ];then if [ `mysql --user=root --password=$MYSQL_ROOT_PWD -s -r -e "SELECT count(TABLE_NAME) FROM information_schema.TABLES WHERE TABLE_SCHEMA='dolphinscheduler';" | grep -v count` -eq 38 ];then
echo "\`$ESZ_DB\` 表个数正确" echo "\`$ESZ_DB\` the number of tables is correct"
else else
echo "\`$ESZ_DB\` 表个数不正确" echo "\`$ESZ_DB\` the number of tables is incorrect"
mysql --user=root --password=$MYSQL_ROOT_PWD -e "DROP DATABASE \`$ESZ_DB\`;" mysql --user=root --password=$MYSQL_ROOT_PWD -e "DROP DATABASE \`$ESZ_DB\`;"
echo "创建dolphinscheduler数据库" echo "create dolphinscheduler database"
mysql --user=root --password=$MYSQL_ROOT_PWD -e "CREATE DATABASE IF NOT EXISTS \`$ESZ_DB\` CHARACTER SET utf8 COLLATE utf8_general_ci; FLUSH PRIVILEGES;" mysql --user=root --password=$MYSQL_ROOT_PWD -e "CREATE DATABASE IF NOT EXISTS \`$ESZ_DB\` CHARACTER SET utf8 COLLATE utf8_general_ci; FLUSH PRIVILEGES;"
echo "导入mysql数据" echo "import mysql data"
nohup /opt/dolphinscheduler/script/create-dolphinscheduler.sh & nohup /opt/dolphinscheduler/script/create-dolphinscheduler.sh &
sleep 90 sleep 90
fi fi
@ -39,29 +39,29 @@ fi
sleep 90 sleep 90
echo "启动api-server" echo "start api-server"
/opt/dolphinscheduler/bin/dolphinscheduler-daemon.sh stop api-server /opt/dolphinscheduler/bin/dolphinscheduler-daemon.sh stop api-server
/opt/dolphinscheduler/bin/dolphinscheduler-daemon.sh start api-server /opt/dolphinscheduler/bin/dolphinscheduler-daemon.sh start api-server
echo "启动master-server" echo "start master-server"
/opt/dolphinscheduler/bin/dolphinscheduler-daemon.sh stop master-server /opt/dolphinscheduler/bin/dolphinscheduler-daemon.sh stop master-server
python /opt/dolphinscheduler/script/del-zk-node.py 127.0.0.1 /dolphinscheduler/masters python /opt/dolphinscheduler/script/del-zk-node.py 127.0.0.1 /dolphinscheduler/masters
/opt/dolphinscheduler/bin/dolphinscheduler-daemon.sh start master-server /opt/dolphinscheduler/bin/dolphinscheduler-daemon.sh start master-server
echo "启动worker-server" echo "start worker-server"
/opt/dolphinscheduler/bin/dolphinscheduler-daemon.sh stop worker-server /opt/dolphinscheduler/bin/dolphinscheduler-daemon.sh stop worker-server
python /opt/dolphinscheduler/script/del-zk-node.py 127.0.0.1 /dolphinscheduler/workers python /opt/dolphinscheduler/script/del-zk-node.py 127.0.0.1 /dolphinscheduler/workers
/opt/dolphinscheduler/bin/dolphinscheduler-daemon.sh start worker-server /opt/dolphinscheduler/bin/dolphinscheduler-daemon.sh start worker-server
echo "启动logger-server" echo "start logger-server"
/opt/dolphinscheduler/bin/dolphinscheduler-daemon.sh stop logger-server /opt/dolphinscheduler/bin/dolphinscheduler-daemon.sh stop logger-server
/opt/dolphinscheduler/bin/dolphinscheduler-daemon.sh start logger-server /opt/dolphinscheduler/bin/dolphinscheduler-daemon.sh start logger-server
echo "启动alert-server" echo "start alert-server"
/opt/dolphinscheduler/bin/dolphinscheduler-daemon.sh stop alert-server /opt/dolphinscheduler/bin/dolphinscheduler-daemon.sh stop alert-server
/opt/dolphinscheduler/bin/dolphinscheduler-daemon.sh start alert-server /opt/dolphinscheduler/bin/dolphinscheduler-daemon.sh start alert-server
@ -69,7 +69,7 @@ echo "启动alert-server"
echo "启动nginx" echo "start nginx"
/etc/init.d/nginx stop /etc/init.d/nginx stop
nginx & nginx &

Loading…
Cancel
Save