From 5956bfcc6672040d10a168945971bbf716d00fb6 Mon Sep 17 00:00:00 2001 From: qiaozhanwei Date: Wed, 6 Nov 2019 15:05:13 +0800 Subject: [PATCH] dockerfile optimization (#1165) * add ConnectionFactoryTest and ConnectionFactory read datasource from appliction.yml * .escheduler_env.sh to dolphinscheduler_env.sh * dao yml assembly to conf directory * table name modify * entity title table name modify * logback log name modify * running through the big process * running through the big process error modify * logback log name modify * data_source.properties rename * logback log name modify * install.sh optimization * install.sh optimization * command count modify * command state update * countCommandState sql update * countCommandState sql update * remove application.yml file * master.properties modify * install.sh modify * install.sh modify * api server startup modify * the current user quits and the session is completely emptied. bug fix * remove pom package resources * checkQueueNameExist method update * checkQueueExist * install.sh error output update * signOut error update * ProcessDao is null bug fix * install.sh add mail.user * request url variables replace * process define import bug fix * process define import export bug fix * processdefine import export bug fix * down log suffix format modify * import export process define contains crontab error bug fix * add Flink local mode * ProcessDao is null bug fix * loadAverage display problem bug fix * MasterServer rename Server * rollback .env * rollback .env * MasterServer rename Server * the task is abnormal and task is running bug fix * owners and administrators can delete * dockerfile optimization * dockerfile optimization * dockerfile optimization --- dockerfile/Dockerfile | 55 ++++++++----------- .../conf/dao/data_source.properties | 53 ------------------ .../{default.conf => dolphinscheduler.conf} | 0 dockerfile/startup.sh | 32 +++++------ 4 files changed, 40 insertions(+), 100 deletions(-) delete mode 100644 dockerfile/conf/dolphinscheduler/conf/dao/data_source.properties rename dockerfile/conf/nginx/{default.conf => dolphinscheduler.conf} (100%) diff --git a/dockerfile/Dockerfile b/dockerfile/Dockerfile index 607a358e78..9cedb2e90b 100644 --- a/dockerfile/Dockerfile +++ b/dockerfile/Dockerfile @@ -1,13 +1,11 @@ FROM ubuntu:18.04 -MAINTAINER journey "825193156@qq.com" - ENV LANG=C.UTF-8 ARG version ARG tar_version -#1,安装jdk +#1,install jdk RUN apt-get update \ && apt-get -y install openjdk-8-jdk \ @@ -17,17 +15,10 @@ ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64 ENV PATH $JAVA_HOME/bin:$PATH -#安装wget +#install wget RUN apt-get update && \ apt-get -y install wget -#2,安装ZK -#RUN cd /opt && \ -# wget https://archive.apache.org/dist/zookeeper/zookeeper-3.4.6/zookeeper-3.4.6.tar.gz && \ -# tar -zxvf zookeeper-3.4.6.tar.gz && \ -# mv zookeeper-3.4.6 zookeeper && \ -# rm -rf ./zookeeper-*tar.gz && \ -# mkdir -p /tmp/zookeeper && \ -# rm -rf /opt/zookeeper/conf/zoo_sample.cfg +#2,install ZK RUN cd /opt && \ wget https://www-us.apache.org/dist/zookeeper/zookeeper-3.4.14/zookeeper-3.4.14.tar.gz && \ @@ -41,7 +32,7 @@ ADD ./dockerfile/conf/zookeeper/zoo.cfg /opt/zookeeper/conf ENV ZK_HOME=/opt/zookeeper ENV PATH $PATH:$ZK_HOME/bin -#3,安装maven +#3,install maven RUN cd /opt && \ wget http://apache-mirror.rbc.ru/pub/apache/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz && \ tar -zxvf apache-maven-3.3.9-bin.tar.gz && \ @@ -52,7 +43,7 @@ ADD ./dockerfile/conf/maven/settings.xml /opt/maven/conf ENV MAVEN_HOME=/opt/maven ENV PATH $PATH:$MAVEN_HOME/bin -#4,安装node +#4,install node RUN cd /opt && \ wget https://nodejs.org/download/release/v8.9.4/node-v8.9.4-linux-x64.tar.gz && \ tar -zxvf node-v8.9.4-linux-x64.tar.gz && \ @@ -61,20 +52,22 @@ RUN cd /opt && \ ENV NODE_HOME=/opt/node ENV PATH $PATH:$NODE_HOME/bin -ADD . /opt/easyscheduler_source +#5,add dolphinscheduler source code to /opt/dolphinscheduler_source +ADD . /opt/dolphinscheduler_source + -#6,后端编译 -RUN cd /opt/easyscheduler_source && \ +#5,backend compilation +RUN cd /opt/dolphinscheduler_source && \ mvn -U clean package assembly:assembly -Dmaven.test.skip=true -#7,前端编译 -RUN chmod -R 777 /opt/easyscheduler_source/dolphinscheduler-ui && \ - cd /opt/easyscheduler_source/dolphinscheduler-ui && \ - rm -rf /opt/easyscheduler_source/dolphinscheduler-ui/node_modules && \ +#6,frontend compilation +RUN chmod -R 777 /opt/dolphinscheduler_source/dolphinscheduler-ui && \ + cd /opt/dolphinscheduler_source/dolphinscheduler-ui && \ + rm -rf /opt/dolphinscheduler_source/dolphinscheduler-ui/node_modules && \ npm install node-sass --unsafe-perm && \ npm install && \ npm run build -#8,安装mysql +#7,install mysql RUN echo "deb http://cn.archive.ubuntu.com/ubuntu/ xenial main restricted universe multiverse" >> /etc/apt/sources.list RUN echo "mysql-server mysql-server/root_password password root" | debconf-set-selections @@ -97,26 +90,26 @@ RUN sed -i -e "$ a [client]\n\n[mysql]\n\n[mysqld]" /etc/mysql/my.cnf && \ sed -i -e "s/\(\[mysqld\]\)/\1\ninit_connect='SET NAMES utf8'\ncharacter-set-server = utf8\ncollation-server=utf8_general_ci\nbind-address = 0.0.0.0/g" /etc/mysql/my.cnf -#9,安装nginx +#8,install nginx RUN apt-get update && \ apt-get install -y nginx && \ rm -rf /var/lib/apt/lists/* && \ echo "\ndaemon off;" >> /etc/nginx/nginx.conf && \ chown -R www-data:www-data /var/lib/nginx -#10,修改dolphinscheduler配置文件 -#后端配置 +#9,modify dolphinscheduler configuration file +#backend configuration RUN mkdir -p /opt/dolphinscheduler && \ - tar -zxvf /opt/easyscheduler_source/target/dolphinscheduler-${tar_version}.tar.gz -C /opt/dolphinscheduler && \ + tar -zxvf /opt/dolphinscheduler_source/target/dolphinscheduler-${tar_version}.tar.gz -C /opt/dolphinscheduler && \ rm -rf /opt/dolphinscheduler/conf ADD ./dockerfile/conf/dolphinscheduler/conf /opt/dolphinscheduler/conf -#前端nginx配置 -ADD ./dockerfile/conf/nginx/default.conf /etc/nginx/conf.d +#frontend nginx configuration +ADD ./dockerfile/conf/nginx/dolphinscheduler.conf /etc/nginx/conf.d -#11,开放端口 +#10,open port EXPOSE 2181 2888 3888 3306 80 12345 8888 -#12,安装sudo,python,vim,ping和ssh +#11,install sudo,python,vim,ping and ssh command RUN apt-get update && \ apt-get -y install sudo && \ apt-get -y install python && \ @@ -128,7 +121,7 @@ RUN apt-get update && \ pip install kazoo COPY ./dockerfile/startup.sh /root/startup.sh -#13,修改权限和设置软连 +#12,modify permissions and set soft links RUN chmod +x /root/startup.sh && \ chmod +x /opt/dolphinscheduler/script/create-dolphinscheduler.sh && \ chmod +x /opt/zookeeper/bin/zkServer.sh && \ diff --git a/dockerfile/conf/dolphinscheduler/conf/dao/data_source.properties b/dockerfile/conf/dolphinscheduler/conf/dao/data_source.properties deleted file mode 100644 index 104f6bc5f2..0000000000 --- a/dockerfile/conf/dolphinscheduler/conf/dao/data_source.properties +++ /dev/null @@ -1,53 +0,0 @@ -# base spring data source configuration -spring.datasource.type=com.alibaba.druid.pool.DruidDataSource -spring.datasource.driver-class-name=com.mysql.jdbc.Driver -spring.datasource.url=jdbc:mysql://127.0.0.1:3306/dolphinscheduler?characterEncoding=UTF-8 -spring.datasource.username=root -spring.datasource.password=root@123 - -# connection configuration -spring.datasource.initialSize=5 -# min connection number -spring.datasource.minIdle=5 -# max connection number -spring.datasource.maxActive=50 - -# max wait time for get a connection in milliseconds. if configuring maxWait, fair locks are enabled by default and concurrency efficiency decreases. -# If necessary, unfair locks can be used by configuring the useUnfairLock attribute to true. -spring.datasource.maxWait=60000 - -# milliseconds for check to close free connections -spring.datasource.timeBetweenEvictionRunsMillis=60000 - -# the Destroy thread detects the connection interval and closes the physical connection in milliseconds if the connection idle time is greater than or equal to minEvictableIdleTimeMillis. -spring.datasource.timeBetweenConnectErrorMillis=60000 - -# the longest time a connection remains idle without being evicted, in milliseconds -spring.datasource.minEvictableIdleTimeMillis=300000 - -#the SQL used to check whether the connection is valid requires a query statement. If validation Query is null, testOnBorrow, testOnReturn, and testWhileIdle will not work. -spring.datasource.validationQuery=SELECT 1 -#check whether the connection is valid for timeout, in seconds -spring.datasource.validationQueryTimeout=3 - -# when applying for a connection, if it is detected that the connection is idle longer than time Between Eviction Runs Millis, -# validation Query is performed to check whether the connection is valid -spring.datasource.testWhileIdle=true - -#execute validation to check if the connection is valid when applying for a connection -spring.datasource.testOnBorrow=true -#execute validation to check if the connection is valid when the connection is returned -spring.datasource.testOnReturn=false -spring.datasource.defaultAutoCommit=true -spring.datasource.keepAlive=true - -# open PSCache, specify count PSCache for every connection -spring.datasource.poolPreparedStatements=true -spring.datasource.maxPoolPreparedStatementPerConnectionSize=20 - -# data quality analysis is not currently in use. please ignore the following configuration -# task record flag -task.record.flag=false -task.record.datasource.url=jdbc:mysql://192.168.xx.xx:3306/etl?characterEncoding=UTF-8 -task.record.datasource.username=xx -task.record.datasource.password=xx diff --git a/dockerfile/conf/nginx/default.conf b/dockerfile/conf/nginx/dolphinscheduler.conf similarity index 100% rename from dockerfile/conf/nginx/default.conf rename to dockerfile/conf/nginx/dolphinscheduler.conf diff --git a/dockerfile/startup.sh b/dockerfile/startup.sh index 1d7f76b173..9c29c0b555 100644 --- a/dockerfile/startup.sh +++ b/dockerfile/startup.sh @@ -5,31 +5,31 @@ if [ `netstat -anop|grep mysql|wc -l` -gt 0 ];then echo "MySQL is Running." else MYSQL_ROOT_PWD="root@123" - ESZ_DB="dolphinscheduler" - echo "启动mysql服务" + ESZ_DB="dolphinscheduler" + echo "start mysql service" chown -R mysql:mysql /var/lib/mysql /var/run/mysqld find /var/lib/mysql -type f -exec touch {} \; && service mysql restart $ sleep 10 if [ ! -f /nohup.out ];then - echo "设置mysql密码" + echo "setting mysql password" mysql --user=root --password=root -e "UPDATE mysql.user set authentication_string=password('$MYSQL_ROOT_PWD') where user='root'; FLUSH PRIVILEGES;" - echo "设置mysql权限" + echo "setting mysql permission" mysql --user=root --password=$MYSQL_ROOT_PWD -e "GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY '$MYSQL_ROOT_PWD' WITH GRANT OPTION; FLUSH PRIVILEGES;" - echo "创建dolphinscheduler数据库" + echo "create dolphinscheduler database" mysql --user=root --password=$MYSQL_ROOT_PWD -e "CREATE DATABASE IF NOT EXISTS \`$ESZ_DB\` CHARACTER SET utf8 COLLATE utf8_general_ci; FLUSH PRIVILEGES;" - echo "导入mysql数据" + echo "import mysql data" nohup /opt/dolphinscheduler/script/create-dolphinscheduler.sh & sleep 90 fi if [ `mysql --user=root --password=$MYSQL_ROOT_PWD -s -r -e "SELECT count(TABLE_NAME) FROM information_schema.TABLES WHERE TABLE_SCHEMA='dolphinscheduler';" | grep -v count` -eq 38 ];then - echo "\`$ESZ_DB\` 表个数正确" + echo "\`$ESZ_DB\` the number of tables is correct" else - echo "\`$ESZ_DB\` 表个数不正确" + echo "\`$ESZ_DB\` the number of tables is incorrect" mysql --user=root --password=$MYSQL_ROOT_PWD -e "DROP DATABASE \`$ESZ_DB\`;" - echo "创建dolphinscheduler数据库" + echo "create dolphinscheduler database" mysql --user=root --password=$MYSQL_ROOT_PWD -e "CREATE DATABASE IF NOT EXISTS \`$ESZ_DB\` CHARACTER SET utf8 COLLATE utf8_general_ci; FLUSH PRIVILEGES;" - echo "导入mysql数据" + echo "import mysql data" nohup /opt/dolphinscheduler/script/create-dolphinscheduler.sh & sleep 90 fi @@ -39,29 +39,29 @@ fi sleep 90 -echo "启动api-server" +echo "start api-server" /opt/dolphinscheduler/bin/dolphinscheduler-daemon.sh stop api-server /opt/dolphinscheduler/bin/dolphinscheduler-daemon.sh start api-server -echo "启动master-server" +echo "start master-server" /opt/dolphinscheduler/bin/dolphinscheduler-daemon.sh stop master-server python /opt/dolphinscheduler/script/del-zk-node.py 127.0.0.1 /dolphinscheduler/masters /opt/dolphinscheduler/bin/dolphinscheduler-daemon.sh start master-server -echo "启动worker-server" +echo "start worker-server" /opt/dolphinscheduler/bin/dolphinscheduler-daemon.sh stop worker-server python /opt/dolphinscheduler/script/del-zk-node.py 127.0.0.1 /dolphinscheduler/workers /opt/dolphinscheduler/bin/dolphinscheduler-daemon.sh start worker-server -echo "启动logger-server" +echo "start logger-server" /opt/dolphinscheduler/bin/dolphinscheduler-daemon.sh stop logger-server /opt/dolphinscheduler/bin/dolphinscheduler-daemon.sh start logger-server -echo "启动alert-server" +echo "start alert-server" /opt/dolphinscheduler/bin/dolphinscheduler-daemon.sh stop alert-server /opt/dolphinscheduler/bin/dolphinscheduler-daemon.sh start alert-server @@ -69,7 +69,7 @@ echo "启动alert-server" -echo "启动nginx" +echo "start nginx" /etc/init.d/nginx stop nginx &