Using docker/docker-swarm/docker-compose.yml to run pg and zk in ci_ut.yml (#2692)

* Using docker/docker-swarm/docker-compose.yml to run pg and zk in ci_ut.yml

1. add initdb volume for pg service in docker-compose.yml
2. modify ci_ut.yml

* add sudo in ci_ut.yml

* Query the metadata of indexes is not correct in postgresql/dolphinscheduler_ddl.sql

1. replace information_schema.STATISTICS to pg_stat_all_indexes
2. modify ProcessDefinitionMapperTest.java

* Remove inner zk and pg

* add inner pg

.github/workflows/ci_ut.yml

@@ -21,7 +21,6 @@ on:
     branches:
       - dev
 env:
-  DOCKER_DIR: ./docker
   LOG_DIR: /tmp/dolphinscheduler
 
 name: Unit Test
@@ -47,7 +46,11 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-maven-
       - name: Bootstrap database
-        run: cd ${DOCKER_DIR} && docker-compose up -d
+        run: |
+          sed -i "s/: root/: test/g" $(pwd)/docker/docker-swarm/docker-compose.yml
+          docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml create --force-recreate dolphinscheduler-zookeeper dolphinscheduler-postgresql
+          sudo cp $(pwd)/sql/dolphinscheduler-postgre.sql $(docker volume inspect docker-swarm_dolphinscheduler-postgresql-initdb | grep "Mountpoint" | awk -F "\"" '{print $4}')
+          docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml up -d dolphinscheduler-zookeeper dolphinscheduler-postgresql
       - name: Set up JDK 1.8
         uses: actions/setup-java@v1
         with:
@@ -82,6 +85,5 @@ jobs:
       - name: Collect logs
         run: |
           mkdir -p ${LOG_DIR}
-          cd ${DOCKER_DIR}
+          docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml logs dolphinscheduler-postgresql > ${LOG_DIR}/db.txt
-          docker-compose logs db > ${LOG_DIR}/db.txt
         continue-on-error: true

docker/docker-swarm/docker-compose.yml

@@ -30,6 +30,7 @@ services:
       POSTGRESQL_DATABASE: dolphinscheduler
     volumes:
     - dolphinscheduler-postgresql:/bitnami/postgresql
+    - dolphinscheduler-postgresql-initdb:/docker-entrypoint-initdb.d
     networks:
     - dolphinscheduler
 
@@ -175,10 +176,10 @@ services:
     image: apache/dolphinscheduler:latest
     container_name: dolphinscheduler-worker
     command: ["worker-server"]
-    ports: 
+    ports:
     - 1234:1234
     - 50051:50051
-    environment: 
+    environment:
       TZ: Asia/Shanghai
       WORKER_EXEC_THREADS: "100"
       WORKER_HEARTBEAT_INTERVAL: "10"
@@ -221,6 +222,7 @@ networks:
 
 volumes:
   dolphinscheduler-postgresql:
+  dolphinscheduler-postgresql-initdb:
   dolphinscheduler-zookeeper:
   dolphinscheduler-worker-data:
   dolphinscheduler-logs:

dockerfile/Dockerfile

@@ -37,32 +37,20 @@ RUN apk add openjdk8
 ENV JAVA_HOME /usr/lib/jvm/java-1.8-openjdk
 ENV PATH $JAVA_HOME/bin:$PATH
 
-#3. install zk
+#3. add dolphinscheduler
-RUN cd /opt && \
-    wget https://downloads.apache.org/zookeeper/zookeeper-3.5.7/apache-zookeeper-3.5.7-bin.tar.gz && \
-    tar -zxvf apache-zookeeper-3.5.7-bin.tar.gz && \
-    mv apache-zookeeper-3.5.7-bin zookeeper && \
-    mkdir -p /tmp/zookeeper && \
-    rm -rf ./zookeeper-*tar.gz && \
-    rm -rf /opt/zookeeper/conf/zoo_sample.cfg
-ADD ./conf/zookeeper/zoo.cfg /opt/zookeeper/conf
-ENV ZK_HOME /opt/zookeeper
-ENV PATH $ZK_HOME/bin:$PATH
-
-#4. install pg
-RUN apk add postgresql postgresql-contrib
-
-#5. add dolphinscheduler
 ADD ./apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz /opt/
 RUN mv /opt/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin/ /opt/dolphinscheduler/
 ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler
 
-#6. modify nginx
+#4. install pg
+RUN apk add postgresql postgresql-contrib
+
+#5. modify nginx
 RUN echo "daemon off;" >> /etc/nginx/nginx.conf && \
     rm -rf /etc/nginx/conf.d/*
 ADD ./conf/nginx/dolphinscheduler.conf /etc/nginx/conf.d
 
-#7. add configuration and modify permissions and set soft links
+#6. add configuration and modify permissions and set soft links
 ADD ./checkpoint.sh /root/checkpoint.sh
 ADD ./startup-init-conf.sh /root/startup-init-conf.sh
 ADD ./startup.sh /root/startup.sh
@@ -75,22 +63,21 @@ RUN chmod +x /root/checkpoint.sh && \
     chmod +x /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh && \
     chmod +x /opt/dolphinscheduler/script/*.sh && \
     chmod +x /opt/dolphinscheduler/bin/*.sh && \
-    chmod +x /opt/zookeeper/bin/*.sh && \
     dos2unix /root/checkpoint.sh && \
     dos2unix /root/startup-init-conf.sh && \
     dos2unix /root/startup.sh && \
     dos2unix /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh && \
     dos2unix /opt/dolphinscheduler/script/*.sh && \
     dos2unix /opt/dolphinscheduler/bin/*.sh && \
-    dos2unix /opt/zookeeper/bin/*.sh && \
     rm -rf /bin/sh && \
     ln -s /bin/bash /bin/sh && \
     mkdir -p /tmp/xls
 
-#8. remove apk index cache
+#7. remove apk index cache and disable coredup for sudo
-RUN rm -rf /var/cache/apk/*
+RUN rm -rf /var/cache/apk/* && \
+    echo "Set disable_coredump false" >> /etc/sudo.conf
 
-#9. expose port
+#8. expose port
 EXPOSE 2181 2888 3888 5432 5678 1234 12345 50051 8888
 
 ENTRYPOINT ["/sbin/tini", "--", "/root/startup.sh"]

dockerfile/startup.sh

@@ -24,31 +24,6 @@ DOLPHINSCHEDULER_LOGS=${DOLPHINSCHEDULER_HOME}/logs
 
 # start postgresql
 initPostgreSQL() {
-    echo "checking postgresql"
-    if [[ "${POSTGRESQL_HOST}" = "127.0.0.1" || "${POSTGRESQL_HOST}" = "localhost" ]]; then
-        export PGPORT=${POSTGRESQL_PORT}
-
-        echo "start postgresql service"
-        rc-service postgresql restart
-
-        # role if not exists, create
-        flag=$(sudo -u postgres psql -tAc "SELECT 1 FROM pg_roles WHERE rolname='${POSTGRESQL_USERNAME}'")
-        if [ -z "${flag}" ]; then
-            echo "create user"
-            sudo -u postgres psql -tAc "create user ${POSTGRESQL_USERNAME} with password '${POSTGRESQL_PASSWORD}'"
-        fi
-
-        # database if not exists, create
-        flag=$(sudo -u postgres psql -tAc "select 1 from pg_database where datname='dolphinscheduler'")
-        if [ -z "${flag}" ]; then
-            echo "init db"
-            sudo -u postgres psql -tAc "create database dolphinscheduler owner ${POSTGRESQL_USERNAME}"
-        fi
-
-        # grant
-        sudo -u postgres psql -tAc "grant all privileges on database dolphinscheduler to ${POSTGRESQL_USERNAME}"
-    fi
-
     echo "test postgresql service"
     while ! nc -z ${POSTGRESQL_HOST} ${POSTGRESQL_PORT}; do
         counter=$((counter+1))
@@ -73,24 +48,18 @@ initPostgreSQL() {
 
 # start zk
 initZK() {
-    echo -e "checking zookeeper"
+    echo "connect remote zookeeper"
-    if [[ "${ZOOKEEPER_QUORUM}" = "127.0.0.1:2181" || "${ZOOKEEPER_QUORUM}" = "localhost:2181" ]]; then
+    echo "${ZOOKEEPER_QUORUM}" | awk -F ',' 'BEGIN{ i=1 }{ while( i <= NF ){ print $i; i++ } }' | while read line; do
-        echo "start local zookeeper"
+        while ! nc -z ${line%:*} ${line#*:}; do
-        /opt/zookeeper/bin/zkServer.sh restart
+            counter=$((counter+1))
-    else
+            if [ $counter == 30 ]; then
-        echo "connect remote zookeeper"
+                echo "Error: Couldn't connect to zookeeper."
-        echo "${ZOOKEEPER_QUORUM}" | awk -F ',' 'BEGIN{ i=1 }{ while( i <= NF ){ print $i; i++ } }' | while read line; do
+                exit 1
-            while ! nc -z ${line%:*} ${line#*:}; do
+            fi
-                counter=$((counter+1))
+            echo "Trying to connect to zookeeper at ${line}. Attempt $counter."
-                if [ $counter == 30 ]; then
+            sleep 5
-                    echo "Error: Couldn't connect to zookeeper."
-                    exit 1
-                fi
-                echo "Trying to connect to zookeeper at ${line}. Attempt $counter."
-                sleep 5
-            done
         done
-    fi
+    done
 }
 
 # start nginx

dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapperTest.java

@@ -73,6 +73,22 @@ public class ProcessDefinitionMapperTest {
         return processDefinition;
     }
 
+    /**
+     * insert
+     * @return ProcessDefinition
+     */
+    private ProcessDefinition insertTwo(){
+        //insertOne
+        ProcessDefinition processDefinition = new ProcessDefinition();
+        processDefinition.setName("def 2");
+        processDefinition.setProjectId(1010);
+        processDefinition.setUserId(101);
+        processDefinition.setUpdateTime(new Date());
+        processDefinition.setCreateTime(new Date());
+        processDefinitionMapper.insert(processDefinition);
+        return processDefinition;
+    }
+
     /**
      * test update
      */
@@ -177,7 +193,7 @@ public class ProcessDefinitionMapperTest {
     public void testQueryDefinitionListByIdList() {
 
         ProcessDefinition processDefinition = insertOne();
-        ProcessDefinition processDefinition1 = insertOne();
+        ProcessDefinition processDefinition1 = insertTwo();
 
         Integer[] array = new Integer[2];
         array[0] = processDefinition.getId();

sql/upgrade/1.3.0_schema/postgresql/dolphinscheduler_ddl.sql

@@ -368,9 +368,9 @@ DROP FUNCTION dc_dolphin_T_t_ds_error_command_D_worker_group_id();
 delimiter d//
 CREATE OR REPLACE FUNCTION uc_dolphin_T_t_ds_process_definition_A_process_definition_unique() RETURNS void AS $$
 BEGIN
-       IF NOT EXISTS (SELECT 1 FROM information_schema.STATISTICS
+       IF NOT EXISTS (SELECT 1 FROM pg_stat_all_indexes
-          WHERE TABLE_NAME='t_ds_process_definition'
+          WHERE relname='t_ds_process_definition'
-                            AND INDEX_NAME ='process_definition_unique')
+                            AND indexrelname ='process_definition_unique')
       THEN
          ALTER TABLE t_ds_process_definition ADD CONSTRAINT process_definition_unique UNIQUE (name,project_id);
        END IF;