Merge pull request #17 from apache/dev-1.3.0

Dev 1.3.0

.github/workflows/ci_e2e.yml

@@ -44,7 +44,7 @@ jobs:
             ${{ runner.os }}-maven-
       - name: Build Image
         run: |
-          sh ./dockerfile/hooks/build
+          sh ./docker/build/hooks/build
       - name: Docker Run
         run: |
           export VERSION=$(cat $(pwd)/pom.xml | grep '<version>' -m 1 | awk '{print $1}' | sed 's/<version>//' | sed 's/<\/version>//')

.github/workflows/ci_ut.yml

@@ -21,7 +21,6 @@ on:
     branches:
       - dev
 env:
-  DOCKER_DIR: ./docker
   LOG_DIR: /tmp/dolphinscheduler
 
 name: Unit Test
@@ -47,7 +46,11 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-maven-
       - name: Bootstrap database
-        run: cd ${DOCKER_DIR} && docker-compose up -d
+        run: |
+          sed -i "s/: root/: test/g" $(pwd)/docker/docker-swarm/docker-compose.yml
+          docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml create --force-recreate dolphinscheduler-zookeeper dolphinscheduler-postgresql
+          sudo cp $(pwd)/sql/dolphinscheduler-postgre.sql $(docker volume inspect docker-swarm_dolphinscheduler-postgresql-initdb | grep "Mountpoint" | awk -F "\"" '{print $4}')
+          docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml up -d dolphinscheduler-zookeeper dolphinscheduler-postgresql
       - name: Set up JDK 1.8
         uses: actions/setup-java@v1
         with:
@@ -82,6 +85,5 @@ jobs:
       - name: Collect logs
         run: |
           mkdir -p ${LOG_DIR}
-          cd ${DOCKER_DIR}
-          docker-compose logs db > ${LOG_DIR}/db.txt
+          docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml logs dolphinscheduler-postgresql > ${LOG_DIR}/db.txt
         continue-on-error: true

docker/build/Dockerfile

@@ -37,32 +37,19 @@ RUN apk add openjdk8
 ENV JAVA_HOME /usr/lib/jvm/java-1.8-openjdk
 ENV PATH $JAVA_HOME/bin:$PATH
 
-#3. install zk
-RUN cd /opt && \
-    wget https://downloads.apache.org/zookeeper/zookeeper-3.5.7/apache-zookeeper-3.5.7-bin.tar.gz && \
-    tar -zxvf apache-zookeeper-3.5.7-bin.tar.gz && \
-    mv apache-zookeeper-3.5.7-bin zookeeper && \
-    mkdir -p /tmp/zookeeper && \
-    rm -rf ./zookeeper-*tar.gz && \
-    rm -rf /opt/zookeeper/conf/zoo_sample.cfg
-ADD ./conf/zookeeper/zoo.cfg /opt/zookeeper/conf
-ENV ZK_HOME /opt/zookeeper
-ENV PATH $ZK_HOME/bin:$PATH
-
-#4. install pg
-RUN apk add postgresql postgresql-contrib
-
-#5. add dolphinscheduler
+#3. add dolphinscheduler
 ADD ./apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz /opt/
 RUN mv /opt/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin/ /opt/dolphinscheduler/
 ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler
 
-#6. modify nginx
+#4. install pg
+RUN apk add postgresql postgresql-contrib
+#5. modify nginx
 RUN echo "daemon off;" >> /etc/nginx/nginx.conf && \
     rm -rf /etc/nginx/conf.d/*
 ADD ./conf/nginx/dolphinscheduler.conf /etc/nginx/conf.d
 
-#7. add configuration and modify permissions and set soft links
+#6. add configuration and modify permissions and set soft links
 ADD ./checkpoint.sh /root/checkpoint.sh
 ADD ./startup-init-conf.sh /root/startup-init-conf.sh
 ADD ./startup.sh /root/startup.sh
@@ -75,22 +62,21 @@ RUN chmod +x /root/checkpoint.sh && \
     chmod +x /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh && \
     chmod +x /opt/dolphinscheduler/script/*.sh && \
     chmod +x /opt/dolphinscheduler/bin/*.sh && \
-    chmod +x /opt/zookeeper/bin/*.sh && \
     dos2unix /root/checkpoint.sh && \
     dos2unix /root/startup-init-conf.sh && \
     dos2unix /root/startup.sh && \
     dos2unix /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh && \
     dos2unix /opt/dolphinscheduler/script/*.sh && \
     dos2unix /opt/dolphinscheduler/bin/*.sh && \
-    dos2unix /opt/zookeeper/bin/*.sh && \
     rm -rf /bin/sh && \
     ln -s /bin/bash /bin/sh && \
     mkdir -p /tmp/xls
 
-#8. remove apk index cache
-RUN rm -rf /var/cache/apk/*
+#7. remove apk index cache and disable coredup for sudo
+RUN rm -rf /var/cache/apk/* && \
+    echo "Set disable_coredump false" >> /etc/sudo.conf
 
-#9. expose port
+#8. expose port
 EXPOSE 2181 2888 3888 5432 5678 1234 12345 50051 8888
 
-ENTRYPOINT ["/sbin/tini", "--", "/root/startup.sh"]
+ENTRYPOINT ["/sbin/tini", "--", "/root/startup.sh"]

docker/build/README.md

@@ -109,16 +109,16 @@ In Unix-Like, Example:
 
 ```bash
 $ cd path/incubator-dolphinscheduler
-$ sh ./dockerfile/hooks/build
+$ sh ./docker/build/hooks/build
 ```
 
 In Windows, Example:
 
 ```bat
-c:\incubator-dolphinscheduler>.\dockerfile\hooks\build.bat
+c:\incubator-dolphinscheduler>.\docker\build\hooks\build.bat
 ```
 
-Please read `./dockerfile/hooks/build` `./dockerfile/hooks/build.bat` script files if you don't understand
+Please read `./docker/build/hooks/build` `./docker/build/hooks/build.bat` script files if you don't understand
 
 ## Environment Variables
 

docker/build/README_zh_CN.md

@@ -109,16 +109,16 @@ dolphinscheduler frontend
 
 ```bash
 $ cd path/incubator-dolphinscheduler
-$ sh ./dockerfile/hooks/build
+$ sh ./docker/build/hooks/build
 ```
 
 Windows系统, 如下:
 
 ```bat
-c:\incubator-dolphinscheduler>.\dockerfile\hooks\build.bat
+c:\incubator-dolphinscheduler>.\docker\build\hooks\build.bat
 ```
 
-如果你不理解这些脚本 `./dockerfile/hooks/build` `./dockerfile/hooks/build.bat`，请阅读里面的内容。
+如果你不理解这些脚本 `./docker/build/hooks/build` `./docker/build/hooks/build.bat`，请阅读里面的内容。
 
 ## 环境变量
 

docker/build/hooks/build

@@ -42,12 +42,12 @@ echo -e "Current Directory is $(pwd)\n"
 echo -e "mvn -B clean compile package -Prelease -Dmaven.test.skip=true"
 mvn -B clean compile package -Prelease -Dmaven.test.skip=true
 
-# mv dolphinscheduler-bin.tar.gz file to dockerfile directory
-echo -e "mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz $(pwd)/dockerfile/\n"
-mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz $(pwd)/dockerfile/
+# mv dolphinscheduler-bin.tar.gz file to docker/build directory
+echo -e "mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz $(pwd)/docker/build/\n"
+mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz $(pwd)/docker/build/
 
 # docker build
-echo -e "docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/dockerfile/\n"
-sudo docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/dockerfile/
+echo -e "docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/docker/build/\n"
+sudo docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/docker/build/
 
 echo "------ dolphinscheduler end   - build -------"

docker/build/hooks/build.bat

@@ -38,13 +38,13 @@ echo "call mvn clean compile package -Prelease"
 call mvn clean compile package -Prelease -DskipTests=true
 if "%errorlevel%"=="1" goto :mvnFailed
 
-:: move dolphinscheduler-bin.tar.gz file to dockerfile directory
-echo "move %cd%\dolphinscheduler-dist\target\apache-dolphinscheduler-incubating-%VERSION%-SNAPSHOT-dolphinscheduler-bin.tar.gz %cd%\dockerfile\"
-move %cd%\dolphinscheduler-dist\target\apache-dolphinscheduler-incubating-%VERSION%-SNAPSHOT-dolphinscheduler-bin.tar.gz %cd%\dockerfile\
+:: move dolphinscheduler-bin.tar.gz file to docker/build directory
+echo "move %cd%\dolphinscheduler-dist\target\apache-dolphinscheduler-incubating-%VERSION%-SNAPSHOT-dolphinscheduler-bin.tar.gz %cd%\docker\build\"
+move %cd%\dolphinscheduler-dist\target\apache-dolphinscheduler-incubating-%VERSION%-SNAPSHOT-dolphinscheduler-bin.tar.gz %cd%\docker\build\
 
 :: docker build
-echo "docker build --build-arg VERSION=%VERSION% -t %DOCKER_REPO%:%VERSION% %cd%\dockerfile\"
-docker build --build-arg VERSION=%VERSION% -t %DOCKER_REPO%:%VERSION% %cd%\dockerfile\
+echo "docker build --build-arg VERSION=%VERSION% -t %DOCKER_REPO%:%VERSION% %cd%\docker\build\"
+docker build --build-arg VERSION=%VERSION% -t %DOCKER_REPO%:%VERSION% %cd%\docker\build\
 if "%errorlevel%"=="1" goto :dockerBuildFailed
 
 echo "------ dolphinscheduler end - build -------"
@@ -53,4 +53,4 @@ echo "------ dolphinscheduler end - build -------"
 echo "MAVEN PACKAGE FAILED!"
 
 :dockerBuildFailed
-echo "DOCKER BUILD FAILED!"
+echo "DOCKER BUILD FAILED!"

docker/build/startup.sh

@@ -24,31 +24,6 @@ DOLPHINSCHEDULER_LOGS=${DOLPHINSCHEDULER_HOME}/logs
 
 # start postgresql
 initPostgreSQL() {
-    echo "checking postgresql"
-    if [[ "${POSTGRESQL_HOST}" = "127.0.0.1" || "${POSTGRESQL_HOST}" = "localhost" ]]; then
-        export PGPORT=${POSTGRESQL_PORT}
-
-        echo "start postgresql service"
-        rc-service postgresql restart
-
-        # role if not exists, create
-        flag=$(sudo -u postgres psql -tAc "SELECT 1 FROM pg_roles WHERE rolname='${POSTGRESQL_USERNAME}'")
-        if [ -z "${flag}" ]; then
-            echo "create user"
-            sudo -u postgres psql -tAc "create user ${POSTGRESQL_USERNAME} with password '${POSTGRESQL_PASSWORD}'"
-        fi
-
-        # database if not exists, create
-        flag=$(sudo -u postgres psql -tAc "select 1 from pg_database where datname='dolphinscheduler'")
-        if [ -z "${flag}" ]; then
-            echo "init db"
-            sudo -u postgres psql -tAc "create database dolphinscheduler owner ${POSTGRESQL_USERNAME}"
-        fi
-
-        # grant
-        sudo -u postgres psql -tAc "grant all privileges on database dolphinscheduler to ${POSTGRESQL_USERNAME}"
-    fi
-
     echo "test postgresql service"
     while ! nc -z ${POSTGRESQL_HOST} ${POSTGRESQL_PORT}; do
         counter=$((counter+1))
@@ -73,24 +48,18 @@ initPostgreSQL() {
 
 # start zk
 initZK() {
-    echo -e "checking zookeeper"
-    if [[ "${ZOOKEEPER_QUORUM}" = "127.0.0.1:2181" || "${ZOOKEEPER_QUORUM}" = "localhost:2181" ]]; then
-        echo "start local zookeeper"
-        /opt/zookeeper/bin/zkServer.sh restart
-    else
-        echo "connect remote zookeeper"
-        echo "${ZOOKEEPER_QUORUM}" | awk -F ',' 'BEGIN{ i=1 }{ while( i <= NF ){ print $i; i++ } }' | while read line; do
-            while ! nc -z ${line%:*} ${line#*:}; do
-                counter=$((counter+1))
-                if [ $counter == 30 ]; then
-                    echo "Error: Couldn't connect to zookeeper."
-                    exit 1
-                fi
-                echo "Trying to connect to zookeeper at ${line}. Attempt $counter."
-                sleep 5
-            done
+    echo "connect remote zookeeper"
+    echo "${ZOOKEEPER_QUORUM}" | awk -F ',' 'BEGIN{ i=1 }{ while( i <= NF ){ print $i; i++ } }' | while read line; do
+        while ! nc -z ${line%:*} ${line#*:}; do
+            counter=$((counter+1))
+            if [ $counter == 30 ]; then
+                echo "Error: Couldn't connect to zookeeper."
+                exit 1
+            fi
+            echo "Trying to connect to zookeeper at ${line}. Attempt $counter."
+            sleep 5
         done
-    fi
+    done
 }
 
 # start nginx

docker/docker-compose.yml

@@ -1,40 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-version: '2'
-services:
-  zookeeper:
-    image: zookeeper
-    restart: always
-    container_name: zookeeper
-    ports:
-      - "2181:2181"
-    environment:
-      ZOO_MY_ID: 1
-      ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons
-  db:
-    image: postgres
-    container_name: postgres
-    environment:
-      - POSTGRES_USER=test
-      - POSTGRES_PASSWORD=test
-      - POSTGRES_DB=dolphinscheduler
-    ports:
-      - "5432:5432"
-    volumes:
-      - pgdata:/var/lib/postgresql/data
-      - ./postgres/docker-entrypoint-initdb:/docker-entrypoint-initdb.d
-volumes:
-  pgdata:

docker/docker-swarm/docker-compose.yml

@@ -30,6 +30,7 @@ services:
       POSTGRESQL_DATABASE: dolphinscheduler
     volumes:
     - dolphinscheduler-postgresql:/bitnami/postgresql
+    - dolphinscheduler-postgresql-initdb:/docker-entrypoint-initdb.d
     networks:
     - dolphinscheduler
 
@@ -175,10 +176,10 @@ services:
     image: apache/dolphinscheduler:latest
     container_name: dolphinscheduler-worker
     command: ["worker-server"]
-    ports: 
+    ports:
     - 1234:1234
     - 50051:50051
-    environment: 
+    environment:
       TZ: Asia/Shanghai
       WORKER_EXEC_THREADS: "100"
       WORKER_HEARTBEAT_INTERVAL: "10"
@@ -221,6 +222,7 @@ networks:
 
 volumes:
   dolphinscheduler-postgresql:
+  dolphinscheduler-postgresql-initdb:
   dolphinscheduler-zookeeper:
   dolphinscheduler-worker-data:
   dolphinscheduler-logs:

docker/postgres/docker-entrypoint-initdb/init.sql

@@ -1,762 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-DROP TABLE IF EXISTS QRTZ_FIRED_TRIGGERS;
-DROP TABLE IF EXISTS QRTZ_PAUSED_TRIGGER_GRPS;
-DROP TABLE IF EXISTS QRTZ_SCHEDULER_STATE;
-DROP TABLE IF EXISTS QRTZ_LOCKS;
-DROP TABLE IF EXISTS QRTZ_SIMPLE_TRIGGERS;
-DROP TABLE IF EXISTS QRTZ_SIMPROP_TRIGGERS;
-DROP TABLE IF EXISTS QRTZ_CRON_TRIGGERS;
-DROP TABLE IF EXISTS QRTZ_BLOB_TRIGGERS;
-DROP TABLE IF EXISTS QRTZ_TRIGGERS;
-DROP TABLE IF EXISTS QRTZ_JOB_DETAILS;
-DROP TABLE IF EXISTS QRTZ_CALENDARS;
-
-CREATE TABLE QRTZ_JOB_DETAILS(
-SCHED_NAME character varying(120) NOT NULL,
-JOB_NAME character varying(200) NOT NULL,
-JOB_GROUP character varying(200) NOT NULL,
-DESCRIPTION character varying(250) NULL,
-JOB_CLASS_NAME character varying(250) NOT NULL,
-IS_DURABLE boolean NOT NULL,
-IS_NONCONCURRENT boolean NOT NULL,
-IS_UPDATE_DATA boolean NOT NULL,
-REQUESTS_RECOVERY boolean NOT NULL,
-JOB_DATA bytea NULL);
-alter table QRTZ_JOB_DETAILS add primary key(SCHED_NAME,JOB_NAME,JOB_GROUP);
-
-CREATE TABLE QRTZ_TRIGGERS (
-SCHED_NAME character varying(120) NOT NULL,
-TRIGGER_NAME character varying(200) NOT NULL,
-TRIGGER_GROUP character varying(200) NOT NULL,
-JOB_NAME character varying(200) NOT NULL,
-JOB_GROUP character varying(200) NOT NULL,
-DESCRIPTION character varying(250) NULL,
-NEXT_FIRE_TIME BIGINT NULL,
-PREV_FIRE_TIME BIGINT NULL,
-PRIORITY INTEGER NULL,
-TRIGGER_STATE character varying(16) NOT NULL,
-TRIGGER_TYPE character varying(8) NOT NULL,
-START_TIME BIGINT NOT NULL,
-END_TIME BIGINT NULL,
-CALENDAR_NAME character varying(200) NULL,
-MISFIRE_INSTR SMALLINT NULL,
-JOB_DATA bytea NULL)  ;
-alter table QRTZ_TRIGGERS add primary key(SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP);
-
-CREATE TABLE QRTZ_SIMPLE_TRIGGERS (
-SCHED_NAME character varying(120) NOT NULL,
-TRIGGER_NAME character varying(200) NOT NULL,
-TRIGGER_GROUP character varying(200) NOT NULL,
-REPEAT_COUNT BIGINT NOT NULL,
-REPEAT_INTERVAL BIGINT NOT NULL,
-TIMES_TRIGGERED BIGINT NOT NULL)  ;
-alter table QRTZ_SIMPLE_TRIGGERS add primary key(SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP);
-
-CREATE TABLE QRTZ_CRON_TRIGGERS (
-SCHED_NAME character varying(120) NOT NULL,
-TRIGGER_NAME character varying(200) NOT NULL,
-TRIGGER_GROUP character varying(200) NOT NULL,
-CRON_EXPRESSION character varying(120) NOT NULL,
-TIME_ZONE_ID character varying(80))  ;
-alter table QRTZ_CRON_TRIGGERS add primary key(SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP);
-
-CREATE TABLE QRTZ_SIMPROP_TRIGGERS
-  (
-    SCHED_NAME character varying(120) NOT NULL,
-    TRIGGER_NAME character varying(200) NOT NULL,
-    TRIGGER_GROUP character varying(200) NOT NULL,
-    STR_PROP_1 character varying(512) NULL,
-    STR_PROP_2 character varying(512) NULL,
-    STR_PROP_3 character varying(512) NULL,
-    INT_PROP_1 INT NULL,
-    INT_PROP_2 INT NULL,
-    LONG_PROP_1 BIGINT NULL,
-    LONG_PROP_2 BIGINT NULL,
-    DEC_PROP_1 NUMERIC(13,4) NULL,
-    DEC_PROP_2 NUMERIC(13,4) NULL,
-    BOOL_PROP_1 boolean NULL,
-    BOOL_PROP_2 boolean NULL) ;
-alter table QRTZ_SIMPROP_TRIGGERS add primary key(SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP);
-
-CREATE TABLE QRTZ_BLOB_TRIGGERS (
-SCHED_NAME character varying(120) NOT NULL,
-TRIGGER_NAME character varying(200) NOT NULL,
-TRIGGER_GROUP character varying(200) NOT NULL,
-BLOB_DATA bytea NULL) ;
-alter table QRTZ_BLOB_TRIGGERS add primary key(SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP);
-
-CREATE TABLE QRTZ_CALENDARS (
-SCHED_NAME character varying(120) NOT NULL,
-CALENDAR_NAME character varying(200) NOT NULL,
-CALENDAR bytea NOT NULL)  ;
-alter table QRTZ_CALENDARS add primary key(SCHED_NAME,CALENDAR_NAME);
-
-CREATE TABLE QRTZ_PAUSED_TRIGGER_GRPS (
-SCHED_NAME character varying(120) NOT NULL,
-TRIGGER_GROUP character varying(200) NOT NULL)  ;
-alter table QRTZ_PAUSED_TRIGGER_GRPS add primary key(SCHED_NAME,TRIGGER_GROUP);
-
-CREATE TABLE QRTZ_FIRED_TRIGGERS (
-SCHED_NAME character varying(120) NOT NULL,
-ENTRY_ID character varying(95) NOT NULL,
-TRIGGER_NAME character varying(200) NOT NULL,
-TRIGGER_GROUP character varying(200) NOT NULL,
-INSTANCE_NAME character varying(200) NOT NULL,
-FIRED_TIME BIGINT NOT NULL,
-SCHED_TIME BIGINT NOT NULL,
-PRIORITY INTEGER NOT NULL,
-STATE character varying(16) NOT NULL,
-JOB_NAME character varying(200) NULL,
-JOB_GROUP character varying(200) NULL,
-IS_NONCONCURRENT boolean NULL,
-REQUESTS_RECOVERY boolean NULL)  ;
-alter table QRTZ_FIRED_TRIGGERS add primary key(SCHED_NAME,ENTRY_ID);
-
-CREATE TABLE QRTZ_SCHEDULER_STATE (
-SCHED_NAME character varying(120) NOT NULL,
-INSTANCE_NAME character varying(200) NOT NULL,
-LAST_CHECKIN_TIME BIGINT NOT NULL,
-CHECKIN_INTERVAL BIGINT NOT NULL)  ;
-alter table QRTZ_SCHEDULER_STATE add primary key(SCHED_NAME,INSTANCE_NAME);
-
-CREATE TABLE QRTZ_LOCKS (
-SCHED_NAME character varying(120) NOT NULL,
-LOCK_NAME character varying(40) NOT NULL)  ;
-alter table QRTZ_LOCKS add primary key(SCHED_NAME,LOCK_NAME);
-
-CREATE INDEX IDX_QRTZ_J_REQ_RECOVERY ON QRTZ_JOB_DETAILS(SCHED_NAME,REQUESTS_RECOVERY);
-CREATE INDEX IDX_QRTZ_J_GRP ON QRTZ_JOB_DETAILS(SCHED_NAME,JOB_GROUP);
-
-CREATE INDEX IDX_QRTZ_T_J ON QRTZ_TRIGGERS(SCHED_NAME,JOB_NAME,JOB_GROUP);
-CREATE INDEX IDX_QRTZ_T_JG ON QRTZ_TRIGGERS(SCHED_NAME,JOB_GROUP);
-CREATE INDEX IDX_QRTZ_T_C ON QRTZ_TRIGGERS(SCHED_NAME,CALENDAR_NAME);
-CREATE INDEX IDX_QRTZ_T_G ON QRTZ_TRIGGERS(SCHED_NAME,TRIGGER_GROUP);
-CREATE INDEX IDX_QRTZ_T_STATE ON QRTZ_TRIGGERS(SCHED_NAME,TRIGGER_STATE);
-CREATE INDEX IDX_QRTZ_T_N_STATE ON QRTZ_TRIGGERS(SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP,TRIGGER_STATE);
-CREATE INDEX IDX_QRTZ_T_N_G_STATE ON QRTZ_TRIGGERS(SCHED_NAME,TRIGGER_GROUP,TRIGGER_STATE);
-CREATE INDEX IDX_QRTZ_T_NEXT_FIRE_TIME ON QRTZ_TRIGGERS(SCHED_NAME,NEXT_FIRE_TIME);
-CREATE INDEX IDX_QRTZ_T_NFT_ST ON QRTZ_TRIGGERS(SCHED_NAME,TRIGGER_STATE,NEXT_FIRE_TIME);
-CREATE INDEX IDX_QRTZ_T_NFT_MISFIRE ON QRTZ_TRIGGERS(SCHED_NAME,MISFIRE_INSTR,NEXT_FIRE_TIME);
-CREATE INDEX IDX_QRTZ_T_NFT_ST_MISFIRE ON QRTZ_TRIGGERS(SCHED_NAME,MISFIRE_INSTR,NEXT_FIRE_TIME,TRIGGER_STATE);
-CREATE INDEX IDX_QRTZ_T_NFT_ST_MISFIRE_GRP ON QRTZ_TRIGGERS(SCHED_NAME,MISFIRE_INSTR,NEXT_FIRE_TIME,TRIGGER_GROUP,TRIGGER_STATE);
-
-CREATE INDEX IDX_QRTZ_FT_TRIG_INST_NAME ON QRTZ_FIRED_TRIGGERS(SCHED_NAME,INSTANCE_NAME);
-CREATE INDEX IDX_QRTZ_FT_INST_JOB_REQ_RCVRY ON QRTZ_FIRED_TRIGGERS(SCHED_NAME,INSTANCE_NAME,REQUESTS_RECOVERY);
-CREATE INDEX IDX_QRTZ_FT_J_G ON QRTZ_FIRED_TRIGGERS(SCHED_NAME,JOB_NAME,JOB_GROUP);
-CREATE INDEX IDX_QRTZ_FT_JG ON QRTZ_FIRED_TRIGGERS(SCHED_NAME,JOB_GROUP);
-CREATE INDEX IDX_QRTZ_FT_T_G ON QRTZ_FIRED_TRIGGERS(SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP);
-CREATE INDEX IDX_QRTZ_FT_TG ON QRTZ_FIRED_TRIGGERS(SCHED_NAME,TRIGGER_GROUP);
-
-
---
--- Table structure for table t_ds_access_token
---
-
-DROP TABLE IF EXISTS t_ds_access_token;
-CREATE TABLE t_ds_access_token (
-  id int NOT NULL  ,
-  user_id int DEFAULT NULL ,
-  token varchar(64) DEFAULT NULL ,
-  expire_time timestamp DEFAULT NULL ,
-  create_time timestamp DEFAULT NULL ,
-  update_time timestamp DEFAULT NULL ,
-  PRIMARY KEY (id)
-) ;
-
---
--- Table structure for table t_ds_alert
---
-
-DROP TABLE IF EXISTS t_ds_alert;
-CREATE TABLE t_ds_alert (
-  id int NOT NULL  ,
-  title varchar(64) DEFAULT NULL ,
-  show_type int DEFAULT NULL ,
-  content text ,
-  alert_type int DEFAULT NULL ,
-  alert_status int DEFAULT '0' ,
-  ·log· text ,
-  alertgroup_id int DEFAULT NULL ,
-  receivers text ,
-  receivers_cc text ,
-  create_time timestamp DEFAULT NULL ,
-  update_time timestamp DEFAULT NULL ,
-  PRIMARY KEY (id)
-) ;
---
--- Table structure for table t_ds_alertgroup
---
-
-DROP TABLE IF EXISTS t_ds_alertgroup;
-CREATE TABLE t_ds_alertgroup (
-  id int NOT NULL  ,
-  group_name varchar(255) DEFAULT NULL ,
-  group_type int DEFAULT NULL ,
-  description varchar(255) DEFAULT NULL ,
-  create_time timestamp DEFAULT NULL ,
-  update_time timestamp DEFAULT NULL ,
-  PRIMARY KEY (id)
-) ;
-
---
--- Table structure for table t_ds_command
---
-
-DROP TABLE IF EXISTS t_ds_command;
-CREATE TABLE t_ds_command (
-  id int NOT NULL  ,
-  command_type int DEFAULT NULL ,
-  process_definition_id int DEFAULT NULL ,
-  command_param text ,
-  task_depend_type int DEFAULT NULL ,
-  failure_strategy int DEFAULT '0' ,
-  warning_type int DEFAULT '0' ,
-  warning_group_id int DEFAULT NULL ,
-  schedule_time timestamp DEFAULT NULL ,
-  start_time timestamp DEFAULT NULL ,
-  executor_id int DEFAULT NULL ,
-  dependence varchar(255) DEFAULT NULL ,
-  update_time timestamp DEFAULT NULL ,
-  process_instance_priority int DEFAULT NULL ,
-  worker_group varchar(64),
-  PRIMARY KEY (id)
-) ;
-
---
--- Table structure for table t_ds_datasource
---
-
-DROP TABLE IF EXISTS t_ds_datasource;
-CREATE TABLE t_ds_datasource (
-  id int NOT NULL  ,
-  name varchar(64) NOT NULL ,
-  note varchar(256) DEFAULT NULL ,
-  type int NOT NULL ,
-  user_id int NOT NULL ,
-  connection_params text NOT NULL ,
-  create_time timestamp NOT NULL ,
-  update_time timestamp DEFAULT NULL ,
-  PRIMARY KEY (id)
-) ;
-
---
--- Table structure for table t_ds_error_command
---
-
-DROP TABLE IF EXISTS t_ds_error_command;
-CREATE TABLE t_ds_error_command (
-  id int NOT NULL ,
-  command_type int DEFAULT NULL ,
-  executor_id int DEFAULT NULL ,
-  process_definition_id int DEFAULT NULL ,
-  command_param text ,
-  task_depend_type int DEFAULT NULL ,
-  failure_strategy int DEFAULT '0' ,
-  warning_type int DEFAULT '0' ,
-  warning_group_id int DEFAULT NULL ,
-  schedule_time timestamp DEFAULT NULL ,
-  start_time timestamp DEFAULT NULL ,
-  update_time timestamp DEFAULT NULL ,
-  dependence text ,
-  process_instance_priority int DEFAULT NULL ,
-  worker_group varchar(64),
-  message text ,
-  PRIMARY KEY (id)
-);
---
--- Table structure for table t_ds_master_server
---
-
---
--- Table structure for table t_ds_process_definition
---
-
-DROP TABLE IF EXISTS t_ds_process_definition;
-CREATE TABLE t_ds_process_definition (
-  id int NOT NULL  ,
-  name varchar(255) DEFAULT NULL ,
-  version int DEFAULT NULL ,
-  release_state int DEFAULT NULL ,
-  project_id int DEFAULT NULL ,
-  user_id int DEFAULT NULL ,
-  process_definition_json text ,
-  description text ,
-  global_params text ,
-  flag int DEFAULT NULL ,
-  locations text ,
-  connects text ,
-  receivers text ,
-  receivers_cc text ,
-  create_time timestamp DEFAULT NULL ,
-  timeout int DEFAULT '0' ,
-  tenant_id int NOT NULL DEFAULT '-1' ,
-  update_time timestamp DEFAULT NULL ,
-  modify_by varchar(36) DEFAULT '' ,
-  resource_ids varchar(64),
-  PRIMARY KEY (id)
-) ;
-
-create index process_definition_index on t_ds_process_definition (project_id,id);
-
---
--- Table structure for table t_ds_process_instance
---
-
-DROP TABLE IF EXISTS t_ds_process_instance;
-CREATE TABLE t_ds_process_instance (
-  id int NOT NULL  ,
-  name varchar(255) DEFAULT NULL ,
-  process_definition_id int DEFAULT NULL ,
-  state int DEFAULT NULL ,
-  recovery int DEFAULT NULL ,
-  start_time timestamp DEFAULT NULL ,
-  end_time timestamp DEFAULT NULL ,
-  run_times int DEFAULT NULL ,
-  host varchar(45) DEFAULT NULL ,
-  command_type int DEFAULT NULL ,
-  command_param text ,
-  task_depend_type int DEFAULT NULL ,
-  max_try_times int DEFAULT '0' ,
-  failure_strategy int DEFAULT '0' ,
-  warning_type int DEFAULT '0' ,
-  warning_group_id int DEFAULT NULL ,
-  schedule_time timestamp DEFAULT NULL ,
-  command_start_time timestamp DEFAULT NULL ,
-  global_params text ,
-  process_instance_json text ,
-  flag int DEFAULT '1' ,
-  update_time timestamp NULL ,
-  is_sub_process int DEFAULT '0' ,
-  executor_id int NOT NULL ,
-  locations text ,
-  connects text ,
-  history_cmd text ,
-  dependence_schedule_times text ,
-  process_instance_priority int DEFAULT NULL ,
-  worker_group varchar(64) ,
-  timeout int DEFAULT '0' ,
-  tenant_id int NOT NULL DEFAULT '-1' ,
-  PRIMARY KEY (id)
-) ;
-  create index process_instance_index on t_ds_process_instance (process_definition_id,id);
-  create index start_time_index on t_ds_process_instance (start_time);
-
---
--- Table structure for table t_ds_project
---
-
-DROP TABLE IF EXISTS t_ds_project;
-CREATE TABLE t_ds_project (
-  id int NOT NULL  ,
-  name varchar(100) DEFAULT NULL ,
-  description varchar(200) DEFAULT NULL ,
-  user_id int DEFAULT NULL ,
-  flag int DEFAULT '1' ,
-  create_time timestamp DEFAULT CURRENT_TIMESTAMP ,
-  update_time timestamp DEFAULT CURRENT_TIMESTAMP ,
-  PRIMARY KEY (id)
-) ;
-  create index user_id_index on t_ds_project (user_id);
-
---
--- Table structure for table t_ds_queue
---
-
-DROP TABLE IF EXISTS t_ds_queue;
-CREATE TABLE t_ds_queue (
-  id int NOT NULL  ,
-  queue_name varchar(64) DEFAULT NULL ,
-  queue varchar(64) DEFAULT NULL ,
-  create_time timestamp DEFAULT NULL ,
-  update_time timestamp DEFAULT NULL ,
-  PRIMARY KEY (id)
-);
-
-
---
--- Table structure for table t_ds_relation_datasource_user
---
-
-DROP TABLE IF EXISTS t_ds_relation_datasource_user;
-CREATE TABLE t_ds_relation_datasource_user (
-  id int NOT NULL  ,
-  user_id int NOT NULL ,
-  datasource_id int DEFAULT NULL ,
-  perm int DEFAULT '1' ,
-  create_time timestamp DEFAULT NULL ,
-  update_time timestamp DEFAULT NULL ,
-  PRIMARY KEY (id)
-) ;
-;
-
---
--- Table structure for table t_ds_relation_process_instance
---
-
-DROP TABLE IF EXISTS t_ds_relation_process_instance;
-CREATE TABLE t_ds_relation_process_instance (
-  id int NOT NULL  ,
-  parent_process_instance_id int DEFAULT NULL ,
-  parent_task_instance_id int DEFAULT NULL ,
-  process_instance_id int DEFAULT NULL ,
-  PRIMARY KEY (id)
-) ;
-
-
---
--- Table structure for table t_ds_relation_project_user
---
-
-DROP TABLE IF EXISTS t_ds_relation_project_user;
-CREATE TABLE t_ds_relation_project_user (
-  id int NOT NULL  ,
-  user_id int NOT NULL ,
-  project_id int DEFAULT NULL ,
-  perm int DEFAULT '1' ,
-  create_time timestamp DEFAULT NULL ,
-  update_time timestamp DEFAULT NULL ,
-  PRIMARY KEY (id)
-) ;
-create index relation_project_user_id_index on t_ds_relation_project_user (user_id);
-
---
--- Table structure for table t_ds_relation_resources_user
---
-
-DROP TABLE IF EXISTS t_ds_relation_resources_user;
-CREATE TABLE t_ds_relation_resources_user (
-  id int NOT NULL ,
-  user_id int NOT NULL ,
-  resources_id int DEFAULT NULL ,
-  perm int DEFAULT '1' ,
-  create_time timestamp DEFAULT NULL ,
-  update_time timestamp DEFAULT NULL ,
-  PRIMARY KEY (id)
-) ;
-
---
--- Table structure for table t_ds_relation_udfs_user
---
-
-DROP TABLE IF EXISTS t_ds_relation_udfs_user;
-CREATE TABLE t_ds_relation_udfs_user (
-  id int NOT NULL  ,
-  user_id int NOT NULL ,
-  udf_id int DEFAULT NULL ,
-  perm int DEFAULT '1' ,
-  create_time timestamp DEFAULT NULL ,
-  update_time timestamp DEFAULT NULL ,
-  PRIMARY KEY (id)
-) ;
-;
-
---
--- Table structure for table t_ds_relation_user_alertgroup
---
-
-DROP TABLE IF EXISTS t_ds_relation_user_alertgroup;
-CREATE TABLE t_ds_relation_user_alertgroup (
-  id int NOT NULL,
-  alertgroup_id int DEFAULT NULL,
-  user_id int DEFAULT NULL,
-  create_time timestamp DEFAULT NULL,
-  update_time timestamp DEFAULT NULL,
-  PRIMARY KEY (id)
-);
-
---
--- Table structure for table t_ds_resources
---
-
-DROP TABLE IF EXISTS t_ds_resources;
-CREATE TABLE t_ds_resources (
-  id int NOT NULL  ,
-  alias varchar(64) DEFAULT NULL ,
-  file_name varchar(64) DEFAULT NULL ,
-  description varchar(256) DEFAULT NULL ,
-  user_id int DEFAULT NULL ,
-  type int DEFAULT NULL ,
-  size bigint DEFAULT NULL ,
-  create_time timestamp DEFAULT NULL ,
-  update_time timestamp DEFAULT NULL ,
-  pid int,
-  full_name varchar(64),
-  is_directory int,
-  PRIMARY KEY (id)
-) ;
-
-
---
--- Table structure for table t_ds_schedules
---
-
-DROP TABLE IF EXISTS t_ds_schedules;
-CREATE TABLE t_ds_schedules (
-  id int NOT NULL  ,
-  process_definition_id int NOT NULL ,
-  start_time timestamp NOT NULL ,
-  end_time timestamp NOT NULL ,
-  crontab varchar(256) NOT NULL ,
-  failure_strategy int NOT NULL ,
-  user_id int NOT NULL ,
-  release_state int NOT NULL ,
-  warning_type int NOT NULL ,
-  warning_group_id int DEFAULT NULL ,
-  process_instance_priority int DEFAULT NULL ,
-  worker_group varchar(64),
-  create_time timestamp NOT NULL ,
-  update_time timestamp NOT NULL ,
-  PRIMARY KEY (id)
-);
-
---
--- Table structure for table t_ds_session
---
-
-DROP TABLE IF EXISTS t_ds_session;
-CREATE TABLE t_ds_session (
-  id varchar(64) NOT NULL ,
-  user_id int DEFAULT NULL ,
-  ip varchar(45) DEFAULT NULL ,
-  last_login_time timestamp DEFAULT NULL ,
-  PRIMARY KEY (id)
-);
-
---
--- Table structure for table t_ds_task_instance
---
-
-DROP TABLE IF EXISTS t_ds_task_instance;
-CREATE TABLE t_ds_task_instance (
-  id int NOT NULL  ,
-  name varchar(255) DEFAULT NULL ,
-  task_type varchar(64) DEFAULT NULL ,
-  process_definition_id int DEFAULT NULL ,
-  process_instance_id int DEFAULT NULL ,
-  task_json text ,
-  state int DEFAULT NULL ,
-  submit_time timestamp DEFAULT NULL ,
-  start_time timestamp DEFAULT NULL ,
-  end_time timestamp DEFAULT NULL ,
-  host varchar(45) DEFAULT NULL ,
-  execute_path varchar(200) DEFAULT NULL ,
-  log_path varchar(200) DEFAULT NULL ,
-  alert_flag int DEFAULT NULL ,
-  retry_times int DEFAULT '0' ,
-  pid int DEFAULT NULL ,
-  app_link varchar(255) DEFAULT NULL ,
-  flag int DEFAULT '1' ,
-  retry_interval int DEFAULT NULL ,
-  max_retry_times int DEFAULT NULL ,
-  task_instance_priority int DEFAULT NULL ,
-   worker_group varchar(64),
-  executor_id int DEFAULT NULL ,
-  PRIMARY KEY (id)
-) ;
-
---
--- Table structure for table t_ds_tenant
---
-
-DROP TABLE IF EXISTS t_ds_tenant;
-CREATE TABLE t_ds_tenant (
-  id int NOT NULL  ,
-  tenant_code varchar(64) DEFAULT NULL ,
-  tenant_name varchar(64) DEFAULT NULL ,
-  description varchar(256) DEFAULT NULL ,
-  queue_id int DEFAULT NULL ,
-  create_time timestamp DEFAULT NULL ,
-  update_time timestamp DEFAULT NULL ,
-  PRIMARY KEY (id)
-) ;
-
---
--- Table structure for table t_ds_udfs
---
-
-DROP TABLE IF EXISTS t_ds_udfs;
-CREATE TABLE t_ds_udfs (
-  id int NOT NULL  ,
-  user_id int NOT NULL ,
-  func_name varchar(100) NOT NULL ,
-  class_name varchar(255) NOT NULL ,
-  type int NOT NULL ,
-  arg_types varchar(255) DEFAULT NULL ,
-  database varchar(255) DEFAULT NULL ,
-  description varchar(255) DEFAULT NULL ,
-  resource_id int NOT NULL ,
-  resource_name varchar(255) NOT NULL ,
-  create_time timestamp NOT NULL ,
-  update_time timestamp NOT NULL ,
-  PRIMARY KEY (id)
-) ;
-
---
--- Table structure for table t_ds_user
---
-
-DROP TABLE IF EXISTS t_ds_user;
-CREATE TABLE t_ds_user (
-  id int NOT NULL  ,
-  user_name varchar(64) DEFAULT NULL ,
-  user_password varchar(64) DEFAULT NULL ,
-  user_type int DEFAULT NULL ,
-  email varchar(64) DEFAULT NULL ,
-  phone varchar(11) DEFAULT NULL ,
-  tenant_id int DEFAULT NULL ,
-  create_time timestamp DEFAULT NULL ,
-  update_time timestamp DEFAULT NULL ,
-  queue varchar(64) DEFAULT NULL ,
-  PRIMARY KEY (id)
-);
-
---
--- Table structure for table t_ds_version
---
-
-DROP TABLE IF EXISTS t_ds_version;
-CREATE TABLE t_ds_version (
-  id int NOT NULL ,
-  version varchar(200) NOT NULL,
-  PRIMARY KEY (id)
-) ;
-create index version_index on t_ds_version(version);
-
---
--- Table structure for table t_ds_worker_group
---
-
-DROP TABLE IF EXISTS t_ds_worker_group;
-CREATE TABLE t_ds_worker_group (
-  id bigint NOT NULL  ,
-  name varchar(256) DEFAULT NULL ,
-  ip_list varchar(256) DEFAULT NULL ,
-  create_time timestamp DEFAULT NULL ,
-  update_time timestamp DEFAULT NULL ,
-  PRIMARY KEY (id)
-) ;
-
---
--- Table structure for table t_ds_worker_server
---
-
-DROP TABLE IF EXISTS t_ds_worker_server;
-CREATE TABLE t_ds_worker_server (
-  id int NOT NULL  ,
-  host varchar(45) DEFAULT NULL ,
-  port int DEFAULT NULL ,
-  zk_directory varchar(64)   DEFAULT NULL ,
-  res_info varchar(255) DEFAULT NULL ,
-  create_time timestamp DEFAULT NULL ,
-  last_heartbeat_time timestamp DEFAULT NULL ,
-  PRIMARY KEY (id)
-) ;
-
-
-DROP SEQUENCE IF EXISTS t_ds_access_token_id_sequence;
-CREATE SEQUENCE  t_ds_access_token_id_sequence;
-ALTER TABLE t_ds_access_token ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_access_token_id_sequence');
-DROP SEQUENCE IF EXISTS t_ds_alert_id_sequence;
-CREATE SEQUENCE  t_ds_alert_id_sequence;
-ALTER TABLE t_ds_alert ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_alert_id_sequence');
-DROP SEQUENCE IF EXISTS t_ds_alertgroup_id_sequence;
-CREATE SEQUENCE  t_ds_alertgroup_id_sequence;
-ALTER TABLE t_ds_alertgroup ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_alertgroup_id_sequence');
-
-DROP SEQUENCE IF EXISTS t_ds_command_id_sequence;
-CREATE SEQUENCE  t_ds_command_id_sequence;
-ALTER TABLE t_ds_command ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_command_id_sequence');
-DROP SEQUENCE IF EXISTS t_ds_datasource_id_sequence;
-CREATE SEQUENCE  t_ds_datasource_id_sequence;
-ALTER TABLE t_ds_datasource ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_datasource_id_sequence');
-DROP SEQUENCE IF EXISTS t_ds_process_definition_id_sequence;
-CREATE SEQUENCE  t_ds_process_definition_id_sequence;
-ALTER TABLE t_ds_process_definition ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_process_definition_id_sequence');
-DROP SEQUENCE IF EXISTS t_ds_process_instance_id_sequence;
-CREATE SEQUENCE  t_ds_process_instance_id_sequence;
-ALTER TABLE t_ds_process_instance ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_process_instance_id_sequence');
-DROP SEQUENCE IF EXISTS t_ds_project_id_sequence;
-CREATE SEQUENCE  t_ds_project_id_sequence;
-ALTER TABLE t_ds_project ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_project_id_sequence');
-DROP SEQUENCE IF EXISTS t_ds_queue_id_sequence;
-CREATE SEQUENCE  t_ds_queue_id_sequence;
-ALTER TABLE t_ds_queue ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_queue_id_sequence');
-
-DROP SEQUENCE IF EXISTS t_ds_relation_datasource_user_id_sequence;
-CREATE SEQUENCE  t_ds_relation_datasource_user_id_sequence;
-ALTER TABLE t_ds_relation_datasource_user ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_relation_datasource_user_id_sequence');
-DROP SEQUENCE IF EXISTS t_ds_relation_process_instance_id_sequence;
-CREATE SEQUENCE  t_ds_relation_process_instance_id_sequence;
-ALTER TABLE t_ds_relation_process_instance ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_relation_process_instance_id_sequence');
-DROP SEQUENCE IF EXISTS t_ds_relation_project_user_id_sequence;
-CREATE SEQUENCE  t_ds_relation_project_user_id_sequence;
-ALTER TABLE t_ds_relation_project_user ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_relation_project_user_id_sequence');
-DROP SEQUENCE IF EXISTS t_ds_relation_resources_user_id_sequence;
-CREATE SEQUENCE  t_ds_relation_resources_user_id_sequence;
-ALTER TABLE t_ds_relation_resources_user ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_relation_resources_user_id_sequence');
-DROP SEQUENCE IF EXISTS t_ds_relation_udfs_user_id_sequence;
-CREATE SEQUENCE  t_ds_relation_udfs_user_id_sequence;
-ALTER TABLE t_ds_relation_udfs_user ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_relation_udfs_user_id_sequence');
-DROP SEQUENCE IF EXISTS t_ds_relation_user_alertgroup_id_sequence;
-CREATE SEQUENCE  t_ds_relation_user_alertgroup_id_sequence;
-ALTER TABLE t_ds_relation_user_alertgroup ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_relation_user_alertgroup_id_sequence');
-
-DROP SEQUENCE IF EXISTS t_ds_resources_id_sequence;
-CREATE SEQUENCE  t_ds_resources_id_sequence;
-ALTER TABLE t_ds_resources ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_resources_id_sequence');
-DROP SEQUENCE IF EXISTS t_ds_schedules_id_sequence;
-CREATE SEQUENCE  t_ds_schedules_id_sequence;
-ALTER TABLE t_ds_schedules ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_schedules_id_sequence');
-DROP SEQUENCE IF EXISTS t_ds_task_instance_id_sequence;
-CREATE SEQUENCE  t_ds_task_instance_id_sequence;
-ALTER TABLE t_ds_task_instance ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_task_instance_id_sequence');
-DROP SEQUENCE IF EXISTS t_ds_tenant_id_sequence;
-CREATE SEQUENCE  t_ds_tenant_id_sequence;
-ALTER TABLE t_ds_tenant ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_tenant_id_sequence');
-DROP SEQUENCE IF EXISTS t_ds_udfs_id_sequence;
-CREATE SEQUENCE  t_ds_udfs_id_sequence;
-ALTER TABLE t_ds_udfs ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_udfs_id_sequence');
-DROP SEQUENCE IF EXISTS t_ds_user_id_sequence;
-CREATE SEQUENCE  t_ds_user_id_sequence;
-ALTER TABLE t_ds_user ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_user_id_sequence');
-
-DROP SEQUENCE IF EXISTS t_ds_version_id_sequence;
-CREATE SEQUENCE  t_ds_version_id_sequence;
-ALTER TABLE t_ds_version ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_version_id_sequence');
-
-DROP SEQUENCE IF EXISTS t_ds_worker_group_id_sequence;
-CREATE SEQUENCE  t_ds_worker_group_id_sequence;
-ALTER TABLE t_ds_worker_group ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_worker_group_id_sequence');
-DROP SEQUENCE IF EXISTS t_ds_worker_server_id_sequence;
-CREATE SEQUENCE  t_ds_worker_server_id_sequence;
-ALTER TABLE t_ds_worker_server ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_worker_server_id_sequence');
-
-
--- Records of t_ds_user?user : admin , password : dolphinscheduler123
-INSERT INTO t_ds_user(user_name,user_password,user_type,email,phone,tenant_id,create_time,update_time) VALUES ('admin', '7ad2410b2f4c074479a8937a28a22b8f', '0', 'xxx@qq.com', 'xx', '0', '2018-03-27 15:48:50', '2018-10-24 17:40:22');
-
--- Records of t_ds_alertgroup，dolphinscheduler warning group
-INSERT INTO t_ds_alertgroup(group_name,group_type,description,create_time,update_time)  VALUES ('dolphinscheduler warning group', '0', 'dolphinscheduler warning group','2018-11-29 10:20:39', '2018-11-29 10:20:39');
-INSERT INTO t_ds_relation_user_alertgroup(alertgroup_id,user_id,create_time,update_time) VALUES ( '1', '1', '2018-11-29 10:22:33', '2018-11-29 10:22:33');
-
--- Records of t_ds_queue,default queue name : default
-INSERT INTO t_ds_queue(queue_name,queue,create_time,update_time) VALUES ('default', 'default','2018-11-29 10:22:33', '2018-11-29 10:22:33');
-
--- Records of t_ds_queue,default queue name : default
-INSERT INTO t_ds_version(version) VALUES ('1.3.0');

dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapperTest.java

@@ -73,6 +73,22 @@ public class ProcessDefinitionMapperTest {
         return processDefinition;
     }
 
+    /**
+     * insert
+     * @return ProcessDefinition
+     */
+    private ProcessDefinition insertTwo(){
+        //insertOne
+        ProcessDefinition processDefinition = new ProcessDefinition();
+        processDefinition.setName("def 2");
+        processDefinition.setProjectId(1010);
+        processDefinition.setUserId(101);
+        processDefinition.setUpdateTime(new Date());
+        processDefinition.setCreateTime(new Date());
+        processDefinitionMapper.insert(processDefinition);
+        return processDefinition;
+    }
+
     /**
      * test update
      */
@@ -177,7 +193,7 @@ public class ProcessDefinitionMapperTest {
     public void testQueryDefinitionListByIdList() {
 
         ProcessDefinition processDefinition = insertOne();
-        ProcessDefinition processDefinition1 = insertOne();
+        ProcessDefinition processDefinition1 = insertTwo();
 
         Integer[] array = new Integer[2];
         array[0] = processDefinition.getId();

dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/RemoveTaskLogRequestCommandTest.java

@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.remote;
+
+import junit.framework.Assert;
+import org.apache.dolphinscheduler.remote.command.Command;
+import org.apache.dolphinscheduler.remote.command.log.RemoveTaskLogRequestCommand;
+import org.apache.dolphinscheduler.remote.command.log.RemoveTaskLogResponseCommand;
+import org.junit.Test;
+
+public class RemoveTaskLogRequestCommandTest {
+
+    @Test
+    public void testConvert2Command(){
+        RemoveTaskLogResponseCommand removeTaskLogResponseCommand = new RemoveTaskLogResponseCommand();
+        removeTaskLogResponseCommand.setStatus(true);
+        Command command = removeTaskLogResponseCommand.convert2Command(122);
+        Assert.assertNotNull(command);
+    }
+}

dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/RemoveTaskLogResponseCommandTest.java

@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.remote;
+
+import junit.framework.Assert;
+import org.apache.dolphinscheduler.remote.command.Command;
+import org.apache.dolphinscheduler.remote.command.log.RemoveTaskLogRequestCommand;
+import org.junit.Test;
+
+public class RemoveTaskLogResponseCommandTest {
+
+    @Test
+    public void testConvert2Command(){
+        RemoveTaskLogRequestCommand removeTaskLogRequestCommand = new RemoveTaskLogRequestCommand();
+        removeTaskLogRequestCommand.setPath("/opt/zhangsan");
+        Command command = removeTaskLogRequestCommand.convert2Command();
+        Assert.assertNotNull(command);
+    }
+}

dolphinscheduler-server/src/main/resources/config/install_config.conf

@@ -130,7 +130,8 @@ masters="ds1,ds2"
 
 # run worker machine
 # note: need to write the worker group name of each worker, the default value is "default"
-workersGroup=(["ds1"]="default" ["ds2"]="default" ["ds3"]="default" ["ds4"]="default" ["ds5"]="default")
+
+declare -A workersGroup=(["ds1"]="default" ["ds2"]="default" ["ds3"]="default" ["ds4"]="default" ["ds5"]="default")
 
 # run alert machine
 # note: list of machine hostnames for deploying alert server
@@ -141,4 +142,4 @@ alertServer="ds3"
 apiServers="ds1"
 
 # whether to start monitoring self-starting scripts
-monitorServerState="false"
+monitorServerState="false"

dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerServerTest.java

@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.log;
+
+import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.service.log.LogClientService;
+import org.junit.Test;
+
+public class LoggerServerTest {
+
+
+    @Test
+    public void testRollViewLog(){
+        LoggerServer loggerServer = new LoggerServer();
+        loggerServer.start();
+
+        LogClientService logClientService = new LogClientService();
+        logClientService.rollViewLog("localhost", Constants.RPC_PORT,"/opt/demo.txt",0,1000);
+
+        try {
+            Thread.sleep(5000);
+        } catch (InterruptedException e) {
+
+        }
+
+        loggerServer.stop();
+        logClientService.close();
+    }
+
+    @Test
+    public void testRemoveTaskLog(){
+        LoggerServer loggerServer = new LoggerServer();
+        loggerServer.start();
+
+        LogClientService logClientService = new LogClientService();
+        logClientService.removeTaskLog("localhost", Constants.RPC_PORT,"/opt/zhangsan");
+
+        try {
+            Thread.sleep(5000);
+        } catch (InterruptedException e) {
+
+        }
+
+        loggerServer.stop();
+        logClientService.close();
+    }
+}

dolphinscheduler-service/pom.xml

@@ -72,5 +72,27 @@
             <groupId>org.quartz-scheduler</groupId>
             <artifactId>quartz-jobs</artifactId>
         </dependency>
+
+        <dependency>
+            <groupId>org.powermock</groupId>
+            <artifactId>powermock-module-junit4</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.powermock</groupId>
+            <artifactId>powermock-api-mockito2</artifactId>
+            <scope>test</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.mockito</groupId>
+                    <artifactId>mockito-core</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-core</artifactId>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 </project>

pom.xml

@@ -777,9 +777,12 @@
                         <include>**/dao/mapper/CommandMapperTest.java</include>
                         <include>**/dao/mapper/ConnectionFactoryTest.java</include>
                         <include>**/dao/mapper/DataSourceMapperTest.java</include>
+                        <include>**/remote/RemoveTaskLogResponseCommandTest.java</include>
+                        <include>**/remote/RemoveTaskLogRequestCommandTest.java</include>
                         <include>**/remote/FastJsonSerializerTest.java</include>
                         <include>**/remote/NettyRemotingClientTest.java</include>
                         <include>**/remote/ResponseFutureTest.java</include>
+                        <include>**/server/log/LoggerServerTest.java</include>
                         <include>**/server/log/MasterLogFilterTest.java</include>
                         <include>**/server/log/SensitiveDataConverterTest.java</include>
                         <include>**/server/log/TaskLogDiscriminatorTest.java</include>

script/scp-hosts.sh

@@ -40,8 +40,8 @@ do
   for dsDir in bin conf lib script sql ui install.sh
   do
     # if worker in workersGroup
-    if [[ "${map[${host}]}" ]] && [[ "${dsDir}" -eq "conf" ]]; then
-      sed -i ${txt} "s#worker.group.*#worker.group=${map[${host}]}#g" $workDir/../conf/worker.properties
+    if [[ "${workersGroup[${host}]}" ]] && [[ "${dsDir}" == "conf" ]]; then
+      sed -i ${txt} "s#worker.group.*#worker.group=${workersGroup[${host}]}#g" ${dsDir}/worker.properties
     fi
 
     echo "start to scp $dsDir to $host/$installPath"
@@ -49,4 +49,4 @@ do
   done
 
   echo "scp dirs to $host/$installPath complete"
-done
+done

script/start-all.sh

@@ -43,5 +43,4 @@ for apiServer in ${apiServersHost[@]}
 do
   echo "$apiServer worker server is starting"
   ssh -p $sshPort $apiServer  "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start api-server;"
-done
-
+done

script/stop-all.sh

@@ -43,5 +43,4 @@ for apiServer in ${apiServersHost[@]}
 do
   echo "$apiServer worker server is stopping"
   ssh -p $sshPort $apiServer  "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop api-server;"
-done
-
+done

sql/upgrade/1.3.0_schema/postgresql/dolphinscheduler_ddl.sql

@@ -368,9 +368,9 @@ DROP FUNCTION dc_dolphin_T_t_ds_error_command_D_worker_group_id();
 delimiter d//
 CREATE OR REPLACE FUNCTION uc_dolphin_T_t_ds_process_definition_A_process_definition_unique() RETURNS void AS $$
 BEGIN
-       IF NOT EXISTS (SELECT 1 FROM information_schema.STATISTICS
-          WHERE TABLE_NAME='t_ds_process_definition'
-                            AND INDEX_NAME ='process_definition_unique')
+       IF NOT EXISTS (SELECT 1 FROM pg_stat_all_indexes
+          WHERE relname='t_ds_process_definition'
+                            AND indexrelname ='process_definition_unique')
       THEN
          ALTER TABLE t_ds_process_definition ADD CONSTRAINT process_definition_unique UNIQUE (name,project_id);
        END IF;