diff --git a/.github/workflows/ci_e2e.yml b/.github/workflows/ci_e2e.yml index c52f0e07d1..58069a7259 100644 --- a/.github/workflows/ci_e2e.yml +++ b/.github/workflows/ci_e2e.yml @@ -44,7 +44,7 @@ jobs: ${{ runner.os }}-maven- - name: Build Image run: | - sh ./dockerfile/hooks/build + sh ./docker/build/hooks/build - name: Docker Run run: | export VERSION=$(cat $(pwd)/pom.xml | grep '' -m 1 | awk '{print $1}' | sed 's///' | sed 's/<\/version>//') diff --git a/.github/workflows/ci_ut.yml b/.github/workflows/ci_ut.yml index 6d0b2a2a1d..55f1259d2b 100644 --- a/.github/workflows/ci_ut.yml +++ b/.github/workflows/ci_ut.yml @@ -21,7 +21,6 @@ on: branches: - dev env: - DOCKER_DIR: ./docker LOG_DIR: /tmp/dolphinscheduler name: Unit Test @@ -47,7 +46,11 @@ jobs: restore-keys: | ${{ runner.os }}-maven- - name: Bootstrap database - run: cd ${DOCKER_DIR} && docker-compose up -d + run: | + sed -i "s/: root/: test/g" $(pwd)/docker/docker-swarm/docker-compose.yml + docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml create --force-recreate dolphinscheduler-zookeeper dolphinscheduler-postgresql + sudo cp $(pwd)/sql/dolphinscheduler-postgre.sql $(docker volume inspect docker-swarm_dolphinscheduler-postgresql-initdb | grep "Mountpoint" | awk -F "\"" '{print $4}') + docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml up -d dolphinscheduler-zookeeper dolphinscheduler-postgresql - name: Set up JDK 1.8 uses: actions/setup-java@v1 with: @@ -82,6 +85,5 @@ jobs: - name: Collect logs run: | mkdir -p ${LOG_DIR} - cd ${DOCKER_DIR} - docker-compose logs db > ${LOG_DIR}/db.txt + docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml logs dolphinscheduler-postgresql > ${LOG_DIR}/db.txt continue-on-error: true diff --git a/dockerfile/Dockerfile b/docker/build/Dockerfile similarity index 79% rename from dockerfile/Dockerfile rename to docker/build/Dockerfile index 53e362b631..bb69347797 100644 --- a/dockerfile/Dockerfile +++ b/docker/build/Dockerfile @@ -37,32 +37,19 @@ RUN apk add openjdk8 ENV JAVA_HOME /usr/lib/jvm/java-1.8-openjdk ENV PATH $JAVA_HOME/bin:$PATH -#3. install zk -RUN cd /opt && \ - wget https://downloads.apache.org/zookeeper/zookeeper-3.5.7/apache-zookeeper-3.5.7-bin.tar.gz && \ - tar -zxvf apache-zookeeper-3.5.7-bin.tar.gz && \ - mv apache-zookeeper-3.5.7-bin zookeeper && \ - mkdir -p /tmp/zookeeper && \ - rm -rf ./zookeeper-*tar.gz && \ - rm -rf /opt/zookeeper/conf/zoo_sample.cfg -ADD ./conf/zookeeper/zoo.cfg /opt/zookeeper/conf -ENV ZK_HOME /opt/zookeeper -ENV PATH $ZK_HOME/bin:$PATH - -#4. install pg -RUN apk add postgresql postgresql-contrib - -#5. add dolphinscheduler +#3. add dolphinscheduler ADD ./apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz /opt/ RUN mv /opt/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin/ /opt/dolphinscheduler/ ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler -#6. modify nginx +#4. install pg +RUN apk add postgresql postgresql-contrib +#5. modify nginx RUN echo "daemon off;" >> /etc/nginx/nginx.conf && \ rm -rf /etc/nginx/conf.d/* ADD ./conf/nginx/dolphinscheduler.conf /etc/nginx/conf.d -#7. add configuration and modify permissions and set soft links +#6. add configuration and modify permissions and set soft links ADD ./checkpoint.sh /root/checkpoint.sh ADD ./startup-init-conf.sh /root/startup-init-conf.sh ADD ./startup.sh /root/startup.sh @@ -75,22 +62,21 @@ RUN chmod +x /root/checkpoint.sh && \ chmod +x /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh && \ chmod +x /opt/dolphinscheduler/script/*.sh && \ chmod +x /opt/dolphinscheduler/bin/*.sh && \ - chmod +x /opt/zookeeper/bin/*.sh && \ dos2unix /root/checkpoint.sh && \ dos2unix /root/startup-init-conf.sh && \ dos2unix /root/startup.sh && \ dos2unix /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh && \ dos2unix /opt/dolphinscheduler/script/*.sh && \ dos2unix /opt/dolphinscheduler/bin/*.sh && \ - dos2unix /opt/zookeeper/bin/*.sh && \ rm -rf /bin/sh && \ ln -s /bin/bash /bin/sh && \ mkdir -p /tmp/xls -#8. remove apk index cache -RUN rm -rf /var/cache/apk/* +#7. remove apk index cache and disable coredup for sudo +RUN rm -rf /var/cache/apk/* && \ + echo "Set disable_coredump false" >> /etc/sudo.conf -#9. expose port +#8. expose port EXPOSE 2181 2888 3888 5432 5678 1234 12345 50051 8888 -ENTRYPOINT ["/sbin/tini", "--", "/root/startup.sh"] \ No newline at end of file +ENTRYPOINT ["/sbin/tini", "--", "/root/startup.sh"] diff --git a/dockerfile/README.md b/docker/build/README.md similarity index 98% rename from dockerfile/README.md rename to docker/build/README.md index b407f57d3b..5725ec56ce 100644 --- a/dockerfile/README.md +++ b/docker/build/README.md @@ -109,16 +109,16 @@ In Unix-Like, Example: ```bash $ cd path/incubator-dolphinscheduler -$ sh ./dockerfile/hooks/build +$ sh ./docker/build/hooks/build ``` In Windows, Example: ```bat -c:\incubator-dolphinscheduler>.\dockerfile\hooks\build.bat +c:\incubator-dolphinscheduler>.\docker\build\hooks\build.bat ``` -Please read `./dockerfile/hooks/build` `./dockerfile/hooks/build.bat` script files if you don't understand +Please read `./docker/build/hooks/build` `./docker/build/hooks/build.bat` script files if you don't understand ## Environment Variables diff --git a/dockerfile/README_zh_CN.md b/docker/build/README_zh_CN.md similarity index 97% rename from dockerfile/README_zh_CN.md rename to docker/build/README_zh_CN.md index 187261581d..0605bef547 100644 --- a/dockerfile/README_zh_CN.md +++ b/docker/build/README_zh_CN.md @@ -109,16 +109,16 @@ dolphinscheduler frontend ```bash $ cd path/incubator-dolphinscheduler -$ sh ./dockerfile/hooks/build +$ sh ./docker/build/hooks/build ``` Windows系统, 如下: ```bat -c:\incubator-dolphinscheduler>.\dockerfile\hooks\build.bat +c:\incubator-dolphinscheduler>.\docker\build\hooks\build.bat ``` -如果你不理解这些脚本 `./dockerfile/hooks/build` `./dockerfile/hooks/build.bat`，请阅读里面的内容。 +如果你不理解这些脚本 `./docker/build/hooks/build` `./docker/build/hooks/build.bat`，请阅读里面的内容。 ## 环境变量 diff --git a/dockerfile/checkpoint.sh b/docker/build/checkpoint.sh similarity index 100% rename from dockerfile/checkpoint.sh rename to docker/build/checkpoint.sh diff --git a/dockerfile/conf/dolphinscheduler/alert.properties.tpl b/docker/build/conf/dolphinscheduler/alert.properties.tpl similarity index 100% rename from dockerfile/conf/dolphinscheduler/alert.properties.tpl rename to docker/build/conf/dolphinscheduler/alert.properties.tpl diff --git a/dockerfile/conf/dolphinscheduler/application-api.properties.tpl b/docker/build/conf/dolphinscheduler/application-api.properties.tpl similarity index 100% rename from dockerfile/conf/dolphinscheduler/application-api.properties.tpl rename to docker/build/conf/dolphinscheduler/application-api.properties.tpl diff --git a/dockerfile/conf/dolphinscheduler/common.properties.tpl b/docker/build/conf/dolphinscheduler/common.properties.tpl similarity index 100% rename from dockerfile/conf/dolphinscheduler/common.properties.tpl rename to docker/build/conf/dolphinscheduler/common.properties.tpl diff --git a/dockerfile/conf/dolphinscheduler/datasource.properties.tpl b/docker/build/conf/dolphinscheduler/datasource.properties.tpl similarity index 100% rename from dockerfile/conf/dolphinscheduler/datasource.properties.tpl rename to docker/build/conf/dolphinscheduler/datasource.properties.tpl diff --git a/dockerfile/conf/dolphinscheduler/env/dolphinscheduler_env.sh b/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh similarity index 100% rename from dockerfile/conf/dolphinscheduler/env/dolphinscheduler_env.sh rename to docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh diff --git a/dockerfile/conf/dolphinscheduler/logback/logback-alert.xml b/docker/build/conf/dolphinscheduler/logback/logback-alert.xml similarity index 100% rename from dockerfile/conf/dolphinscheduler/logback/logback-alert.xml rename to docker/build/conf/dolphinscheduler/logback/logback-alert.xml diff --git a/dockerfile/conf/dolphinscheduler/logback/logback-api.xml b/docker/build/conf/dolphinscheduler/logback/logback-api.xml similarity index 100% rename from dockerfile/conf/dolphinscheduler/logback/logback-api.xml rename to docker/build/conf/dolphinscheduler/logback/logback-api.xml diff --git a/dockerfile/conf/dolphinscheduler/logback/logback-master.xml b/docker/build/conf/dolphinscheduler/logback/logback-master.xml similarity index 100% rename from dockerfile/conf/dolphinscheduler/logback/logback-master.xml rename to docker/build/conf/dolphinscheduler/logback/logback-master.xml diff --git a/dockerfile/conf/dolphinscheduler/logback/logback-worker.xml b/docker/build/conf/dolphinscheduler/logback/logback-worker.xml similarity index 100% rename from dockerfile/conf/dolphinscheduler/logback/logback-worker.xml rename to docker/build/conf/dolphinscheduler/logback/logback-worker.xml diff --git a/dockerfile/conf/dolphinscheduler/master.properties.tpl b/docker/build/conf/dolphinscheduler/master.properties.tpl similarity index 100% rename from dockerfile/conf/dolphinscheduler/master.properties.tpl rename to docker/build/conf/dolphinscheduler/master.properties.tpl diff --git a/dockerfile/conf/dolphinscheduler/quartz.properties.tpl b/docker/build/conf/dolphinscheduler/quartz.properties.tpl similarity index 100% rename from dockerfile/conf/dolphinscheduler/quartz.properties.tpl rename to docker/build/conf/dolphinscheduler/quartz.properties.tpl diff --git a/dockerfile/conf/dolphinscheduler/worker.properties.tpl b/docker/build/conf/dolphinscheduler/worker.properties.tpl similarity index 100% rename from dockerfile/conf/dolphinscheduler/worker.properties.tpl rename to docker/build/conf/dolphinscheduler/worker.properties.tpl diff --git a/dockerfile/conf/dolphinscheduler/zookeeper.properties.tpl b/docker/build/conf/dolphinscheduler/zookeeper.properties.tpl similarity index 100% rename from dockerfile/conf/dolphinscheduler/zookeeper.properties.tpl rename to docker/build/conf/dolphinscheduler/zookeeper.properties.tpl diff --git a/dockerfile/conf/nginx/dolphinscheduler.conf b/docker/build/conf/nginx/dolphinscheduler.conf similarity index 100% rename from dockerfile/conf/nginx/dolphinscheduler.conf rename to docker/build/conf/nginx/dolphinscheduler.conf diff --git a/dockerfile/conf/zookeeper/zoo.cfg b/docker/build/conf/zookeeper/zoo.cfg similarity index 100% rename from dockerfile/conf/zookeeper/zoo.cfg rename to docker/build/conf/zookeeper/zoo.cfg diff --git a/dockerfile/hooks/build b/docker/build/hooks/build similarity index 89% rename from dockerfile/hooks/build rename to docker/build/hooks/build index 956881b628..3276f167c4 100644 --- a/dockerfile/hooks/build +++ b/docker/build/hooks/build @@ -42,12 +42,12 @@ echo -e "Current Directory is $(pwd)\n" echo -e "mvn -B clean compile package -Prelease -Dmaven.test.skip=true" mvn -B clean compile package -Prelease -Dmaven.test.skip=true -# mv dolphinscheduler-bin.tar.gz file to dockerfile directory -echo -e "mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz $(pwd)/dockerfile/\n" -mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz $(pwd)/dockerfile/ +# mv dolphinscheduler-bin.tar.gz file to docker/build directory +echo -e "mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz $(pwd)/docker/build/\n" +mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz $(pwd)/docker/build/ # docker build -echo -e "docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/dockerfile/\n" -sudo docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/dockerfile/ +echo -e "docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/docker/build/\n" +sudo docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/docker/build/ echo "------ dolphinscheduler end - build -------" diff --git a/dockerfile/hooks/build.bat b/docker/build/hooks/build.bat similarity index 91% rename from dockerfile/hooks/build.bat rename to docker/build/hooks/build.bat index b15c7b00df..fd7c9b55ba 100644 --- a/dockerfile/hooks/build.bat +++ b/docker/build/hooks/build.bat @@ -38,13 +38,13 @@ echo "call mvn clean compile package -Prelease" call mvn clean compile package -Prelease -DskipTests=true if "%errorlevel%"=="1" goto :mvnFailed -:: move dolphinscheduler-bin.tar.gz file to dockerfile directory -echo "move %cd%\dolphinscheduler-dist\target\apache-dolphinscheduler-incubating-%VERSION%-SNAPSHOT-dolphinscheduler-bin.tar.gz %cd%\dockerfile\" -move %cd%\dolphinscheduler-dist\target\apache-dolphinscheduler-incubating-%VERSION%-SNAPSHOT-dolphinscheduler-bin.tar.gz %cd%\dockerfile\ +:: move dolphinscheduler-bin.tar.gz file to docker/build directory +echo "move %cd%\dolphinscheduler-dist\target\apache-dolphinscheduler-incubating-%VERSION%-SNAPSHOT-dolphinscheduler-bin.tar.gz %cd%\docker\build\" +move %cd%\dolphinscheduler-dist\target\apache-dolphinscheduler-incubating-%VERSION%-SNAPSHOT-dolphinscheduler-bin.tar.gz %cd%\docker\build\ :: docker build -echo "docker build --build-arg VERSION=%VERSION% -t %DOCKER_REPO%:%VERSION% %cd%\dockerfile\" -docker build --build-arg VERSION=%VERSION% -t %DOCKER_REPO%:%VERSION% %cd%\dockerfile\ +echo "docker build --build-arg VERSION=%VERSION% -t %DOCKER_REPO%:%VERSION% %cd%\docker\build\" +docker build --build-arg VERSION=%VERSION% -t %DOCKER_REPO%:%VERSION% %cd%\docker\build\ if "%errorlevel%"=="1" goto :dockerBuildFailed echo "------ dolphinscheduler end - build -------" @@ -53,4 +53,4 @@ echo "------ dolphinscheduler end - build -------" echo "MAVEN PACKAGE FAILED!" :dockerBuildFailed -echo "DOCKER BUILD FAILED!" \ No newline at end of file +echo "DOCKER BUILD FAILED!" diff --git a/dockerfile/hooks/push b/docker/build/hooks/push similarity index 100% rename from dockerfile/hooks/push rename to docker/build/hooks/push diff --git a/dockerfile/hooks/push.bat b/docker/build/hooks/push.bat similarity index 100% rename from dockerfile/hooks/push.bat rename to docker/build/hooks/push.bat diff --git a/dockerfile/startup-init-conf.sh b/docker/build/startup-init-conf.sh similarity index 100% rename from dockerfile/startup-init-conf.sh rename to docker/build/startup-init-conf.sh diff --git a/dockerfile/startup.sh b/docker/build/startup.sh similarity index 75% rename from dockerfile/startup.sh rename to docker/build/startup.sh index 71ab506777..00bb9cdc52 100644 --- a/dockerfile/startup.sh +++ b/docker/build/startup.sh @@ -24,31 +24,6 @@ DOLPHINSCHEDULER_LOGS=${DOLPHINSCHEDULER_HOME}/logs # start postgresql initPostgreSQL() { - echo "checking postgresql" - if [[ "${POSTGRESQL_HOST}" = "127.0.0.1" || "${POSTGRESQL_HOST}" = "localhost" ]]; then - export PGPORT=${POSTGRESQL_PORT} - - echo "start postgresql service" - rc-service postgresql restart - - # role if not exists, create - flag=$(sudo -u postgres psql -tAc "SELECT 1 FROM pg_roles WHERE rolname='${POSTGRESQL_USERNAME}'") - if [ -z "${flag}" ]; then - echo "create user" - sudo -u postgres psql -tAc "create user ${POSTGRESQL_USERNAME} with password '${POSTGRESQL_PASSWORD}'" - fi - - # database if not exists, create - flag=$(sudo -u postgres psql -tAc "select 1 from pg_database where datname='dolphinscheduler'") - if [ -z "${flag}" ]; then - echo "init db" - sudo -u postgres psql -tAc "create database dolphinscheduler owner ${POSTGRESQL_USERNAME}" - fi - - # grant - sudo -u postgres psql -tAc "grant all privileges on database dolphinscheduler to ${POSTGRESQL_USERNAME}" - fi - echo "test postgresql service" while ! nc -z ${POSTGRESQL_HOST} ${POSTGRESQL_PORT}; do counter=$((counter+1)) @@ -73,24 +48,18 @@ initPostgreSQL() { # start zk initZK() { - echo -e "checking zookeeper" - if [[ "${ZOOKEEPER_QUORUM}" = "127.0.0.1:2181" || "${ZOOKEEPER_QUORUM}" = "localhost:2181" ]]; then - echo "start local zookeeper" - /opt/zookeeper/bin/zkServer.sh restart - else - echo "connect remote zookeeper" - echo "${ZOOKEEPER_QUORUM}" | awk -F ',' 'BEGIN{ i=1 }{ while( i <= NF ){ print $i; i++ } }' | while read line; do - while ! nc -z ${line%:*} ${line#*:}; do - counter=$((counter+1)) - if [ $counter == 30 ]; then - echo "Error: Couldn't connect to zookeeper." - exit 1 - fi - echo "Trying to connect to zookeeper at ${line}. Attempt $counter." - sleep 5 - done + echo "connect remote zookeeper" + echo "${ZOOKEEPER_QUORUM}" | awk -F ',' 'BEGIN{ i=1 }{ while( i <= NF ){ print $i; i++ } }' | while read line; do + while ! nc -z ${line%:*} ${line#*:}; do + counter=$((counter+1)) + if [ $counter == 30 ]; then + echo "Error: Couldn't connect to zookeeper." + exit 1 + fi + echo "Trying to connect to zookeeper at ${line}. Attempt $counter." + sleep 5 done - fi + done } # start nginx diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml deleted file mode 100644 index 26ffb5b2b5..0000000000 --- a/docker/docker-compose.yml +++ /dev/null @@ -1,40 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -version: '2' -services: - zookeeper: - image: zookeeper - restart: always - container_name: zookeeper - ports: - - "2181:2181" - environment: - ZOO_MY_ID: 1 - ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons - db: - image: postgres - container_name: postgres - environment: - - POSTGRES_USER=test - - POSTGRES_PASSWORD=test - - POSTGRES_DB=dolphinscheduler - ports: - - "5432:5432" - volumes: - - pgdata:/var/lib/postgresql/data - - ./postgres/docker-entrypoint-initdb:/docker-entrypoint-initdb.d -volumes: - pgdata: diff --git a/docker/docker-swarm/docker-compose.yml b/docker/docker-swarm/docker-compose.yml index b564eda842..ee8be2570d 100644 --- a/docker/docker-swarm/docker-compose.yml +++ b/docker/docker-swarm/docker-compose.yml @@ -30,6 +30,7 @@ services: POSTGRESQL_DATABASE: dolphinscheduler volumes: - dolphinscheduler-postgresql:/bitnami/postgresql + - dolphinscheduler-postgresql-initdb:/docker-entrypoint-initdb.d networks: - dolphinscheduler @@ -175,10 +176,10 @@ services: image: apache/dolphinscheduler:latest container_name: dolphinscheduler-worker command: ["worker-server"] - ports: + ports: - 1234:1234 - 50051:50051 - environment: + environment: TZ: Asia/Shanghai WORKER_EXEC_THREADS: "100" WORKER_HEARTBEAT_INTERVAL: "10" @@ -221,6 +222,7 @@ networks: volumes: dolphinscheduler-postgresql: + dolphinscheduler-postgresql-initdb: dolphinscheduler-zookeeper: dolphinscheduler-worker-data: dolphinscheduler-logs: diff --git a/docker/postgres/docker-entrypoint-initdb/init.sql b/docker/postgres/docker-entrypoint-initdb/init.sql deleted file mode 100755 index e9cdc68fb1..0000000000 --- a/docker/postgres/docker-entrypoint-initdb/init.sql +++ /dev/null @@ -1,762 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -DROP TABLE IF EXISTS QRTZ_FIRED_TRIGGERS; -DROP TABLE IF EXISTS QRTZ_PAUSED_TRIGGER_GRPS; -DROP TABLE IF EXISTS QRTZ_SCHEDULER_STATE; -DROP TABLE IF EXISTS QRTZ_LOCKS; -DROP TABLE IF EXISTS QRTZ_SIMPLE_TRIGGERS; -DROP TABLE IF EXISTS QRTZ_SIMPROP_TRIGGERS; -DROP TABLE IF EXISTS QRTZ_CRON_TRIGGERS; -DROP TABLE IF EXISTS QRTZ_BLOB_TRIGGERS; -DROP TABLE IF EXISTS QRTZ_TRIGGERS; -DROP TABLE IF EXISTS QRTZ_JOB_DETAILS; -DROP TABLE IF EXISTS QRTZ_CALENDARS; - -CREATE TABLE QRTZ_JOB_DETAILS( -SCHED_NAME character varying(120) NOT NULL, -JOB_NAME character varying(200) NOT NULL, -JOB_GROUP character varying(200) NOT NULL, -DESCRIPTION character varying(250) NULL, -JOB_CLASS_NAME character varying(250) NOT NULL, -IS_DURABLE boolean NOT NULL, -IS_NONCONCURRENT boolean NOT NULL, -IS_UPDATE_DATA boolean NOT NULL, -REQUESTS_RECOVERY boolean NOT NULL, -JOB_DATA bytea NULL); -alter table QRTZ_JOB_DETAILS add primary key(SCHED_NAME,JOB_NAME,JOB_GROUP); - -CREATE TABLE QRTZ_TRIGGERS ( -SCHED_NAME character varying(120) NOT NULL, -TRIGGER_NAME character varying(200) NOT NULL, -TRIGGER_GROUP character varying(200) NOT NULL, -JOB_NAME character varying(200) NOT NULL, -JOB_GROUP character varying(200) NOT NULL, -DESCRIPTION character varying(250) NULL, -NEXT_FIRE_TIME BIGINT NULL, -PREV_FIRE_TIME BIGINT NULL, -PRIORITY INTEGER NULL, -TRIGGER_STATE character varying(16) NOT NULL, -TRIGGER_TYPE character varying(8) NOT NULL, -START_TIME BIGINT NOT NULL, -END_TIME BIGINT NULL, -CALENDAR_NAME character varying(200) NULL, -MISFIRE_INSTR SMALLINT NULL, -JOB_DATA bytea NULL) ; -alter table QRTZ_TRIGGERS add primary key(SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP); - -CREATE TABLE QRTZ_SIMPLE_TRIGGERS ( -SCHED_NAME character varying(120) NOT NULL, -TRIGGER_NAME character varying(200) NOT NULL, -TRIGGER_GROUP character varying(200) NOT NULL, -REPEAT_COUNT BIGINT NOT NULL, -REPEAT_INTERVAL BIGINT NOT NULL, -TIMES_TRIGGERED BIGINT NOT NULL) ; -alter table QRTZ_SIMPLE_TRIGGERS add primary key(SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP); - -CREATE TABLE QRTZ_CRON_TRIGGERS ( -SCHED_NAME character varying(120) NOT NULL, -TRIGGER_NAME character varying(200) NOT NULL, -TRIGGER_GROUP character varying(200) NOT NULL, -CRON_EXPRESSION character varying(120) NOT NULL, -TIME_ZONE_ID character varying(80)) ; -alter table QRTZ_CRON_TRIGGERS add primary key(SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP); - -CREATE TABLE QRTZ_SIMPROP_TRIGGERS - ( - SCHED_NAME character varying(120) NOT NULL, - TRIGGER_NAME character varying(200) NOT NULL, - TRIGGER_GROUP character varying(200) NOT NULL, - STR_PROP_1 character varying(512) NULL, - STR_PROP_2 character varying(512) NULL, - STR_PROP_3 character varying(512) NULL, - INT_PROP_1 INT NULL, - INT_PROP_2 INT NULL, - LONG_PROP_1 BIGINT NULL, - LONG_PROP_2 BIGINT NULL, - DEC_PROP_1 NUMERIC(13,4) NULL, - DEC_PROP_2 NUMERIC(13,4) NULL, - BOOL_PROP_1 boolean NULL, - BOOL_PROP_2 boolean NULL) ; -alter table QRTZ_SIMPROP_TRIGGERS add primary key(SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP); - -CREATE TABLE QRTZ_BLOB_TRIGGERS ( -SCHED_NAME character varying(120) NOT NULL, -TRIGGER_NAME character varying(200) NOT NULL, -TRIGGER_GROUP character varying(200) NOT NULL, -BLOB_DATA bytea NULL) ; -alter table QRTZ_BLOB_TRIGGERS add primary key(SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP); - -CREATE TABLE QRTZ_CALENDARS ( -SCHED_NAME character varying(120) NOT NULL, -CALENDAR_NAME character varying(200) NOT NULL, -CALENDAR bytea NOT NULL) ; -alter table QRTZ_CALENDARS add primary key(SCHED_NAME,CALENDAR_NAME); - -CREATE TABLE QRTZ_PAUSED_TRIGGER_GRPS ( -SCHED_NAME character varying(120) NOT NULL, -TRIGGER_GROUP character varying(200) NOT NULL) ; -alter table QRTZ_PAUSED_TRIGGER_GRPS add primary key(SCHED_NAME,TRIGGER_GROUP); - -CREATE TABLE QRTZ_FIRED_TRIGGERS ( -SCHED_NAME character varying(120) NOT NULL, -ENTRY_ID character varying(95) NOT NULL, -TRIGGER_NAME character varying(200) NOT NULL, -TRIGGER_GROUP character varying(200) NOT NULL, -INSTANCE_NAME character varying(200) NOT NULL, -FIRED_TIME BIGINT NOT NULL, -SCHED_TIME BIGINT NOT NULL, -PRIORITY INTEGER NOT NULL, -STATE character varying(16) NOT NULL, -JOB_NAME character varying(200) NULL, -JOB_GROUP character varying(200) NULL, -IS_NONCONCURRENT boolean NULL, -REQUESTS_RECOVERY boolean NULL) ; -alter table QRTZ_FIRED_TRIGGERS add primary key(SCHED_NAME,ENTRY_ID); - -CREATE TABLE QRTZ_SCHEDULER_STATE ( -SCHED_NAME character varying(120) NOT NULL, -INSTANCE_NAME character varying(200) NOT NULL, -LAST_CHECKIN_TIME BIGINT NOT NULL, -CHECKIN_INTERVAL BIGINT NOT NULL) ; -alter table QRTZ_SCHEDULER_STATE add primary key(SCHED_NAME,INSTANCE_NAME); - -CREATE TABLE QRTZ_LOCKS ( -SCHED_NAME character varying(120) NOT NULL, -LOCK_NAME character varying(40) NOT NULL) ; -alter table QRTZ_LOCKS add primary key(SCHED_NAME,LOCK_NAME); - -CREATE INDEX IDX_QRTZ_J_REQ_RECOVERY ON QRTZ_JOB_DETAILS(SCHED_NAME,REQUESTS_RECOVERY); -CREATE INDEX IDX_QRTZ_J_GRP ON QRTZ_JOB_DETAILS(SCHED_NAME,JOB_GROUP); - -CREATE INDEX IDX_QRTZ_T_J ON QRTZ_TRIGGERS(SCHED_NAME,JOB_NAME,JOB_GROUP); -CREATE INDEX IDX_QRTZ_T_JG ON QRTZ_TRIGGERS(SCHED_NAME,JOB_GROUP); -CREATE INDEX IDX_QRTZ_T_C ON QRTZ_TRIGGERS(SCHED_NAME,CALENDAR_NAME); -CREATE INDEX IDX_QRTZ_T_G ON QRTZ_TRIGGERS(SCHED_NAME,TRIGGER_GROUP); -CREATE INDEX IDX_QRTZ_T_STATE ON QRTZ_TRIGGERS(SCHED_NAME,TRIGGER_STATE); -CREATE INDEX IDX_QRTZ_T_N_STATE ON QRTZ_TRIGGERS(SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP,TRIGGER_STATE); -CREATE INDEX IDX_QRTZ_T_N_G_STATE ON QRTZ_TRIGGERS(SCHED_NAME,TRIGGER_GROUP,TRIGGER_STATE); -CREATE INDEX IDX_QRTZ_T_NEXT_FIRE_TIME ON QRTZ_TRIGGERS(SCHED_NAME,NEXT_FIRE_TIME); -CREATE INDEX IDX_QRTZ_T_NFT_ST ON QRTZ_TRIGGERS(SCHED_NAME,TRIGGER_STATE,NEXT_FIRE_TIME); -CREATE INDEX IDX_QRTZ_T_NFT_MISFIRE ON QRTZ_TRIGGERS(SCHED_NAME,MISFIRE_INSTR,NEXT_FIRE_TIME); -CREATE INDEX IDX_QRTZ_T_NFT_ST_MISFIRE ON QRTZ_TRIGGERS(SCHED_NAME,MISFIRE_INSTR,NEXT_FIRE_TIME,TRIGGER_STATE); -CREATE INDEX IDX_QRTZ_T_NFT_ST_MISFIRE_GRP ON QRTZ_TRIGGERS(SCHED_NAME,MISFIRE_INSTR,NEXT_FIRE_TIME,TRIGGER_GROUP,TRIGGER_STATE); - -CREATE INDEX IDX_QRTZ_FT_TRIG_INST_NAME ON QRTZ_FIRED_TRIGGERS(SCHED_NAME,INSTANCE_NAME); -CREATE INDEX IDX_QRTZ_FT_INST_JOB_REQ_RCVRY ON QRTZ_FIRED_TRIGGERS(SCHED_NAME,INSTANCE_NAME,REQUESTS_RECOVERY); -CREATE INDEX IDX_QRTZ_FT_J_G ON QRTZ_FIRED_TRIGGERS(SCHED_NAME,JOB_NAME,JOB_GROUP); -CREATE INDEX IDX_QRTZ_FT_JG ON QRTZ_FIRED_TRIGGERS(SCHED_NAME,JOB_GROUP); -CREATE INDEX IDX_QRTZ_FT_T_G ON QRTZ_FIRED_TRIGGERS(SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP); -CREATE INDEX IDX_QRTZ_FT_TG ON QRTZ_FIRED_TRIGGERS(SCHED_NAME,TRIGGER_GROUP); - - --- --- Table structure for table t_ds_access_token --- - -DROP TABLE IF EXISTS t_ds_access_token; -CREATE TABLE t_ds_access_token ( - id int NOT NULL , - user_id int DEFAULT NULL , - token varchar(64) DEFAULT NULL , - expire_time timestamp DEFAULT NULL , - create_time timestamp DEFAULT NULL , - update_time timestamp DEFAULT NULL , - PRIMARY KEY (id) -) ; - --- --- Table structure for table t_ds_alert --- - -DROP TABLE IF EXISTS t_ds_alert; -CREATE TABLE t_ds_alert ( - id int NOT NULL , - title varchar(64) DEFAULT NULL , - show_type int DEFAULT NULL , - content text , - alert_type int DEFAULT NULL , - alert_status int DEFAULT '0' , - ·log· text , - alertgroup_id int DEFAULT NULL , - receivers text , - receivers_cc text , - create_time timestamp DEFAULT NULL , - update_time timestamp DEFAULT NULL , - PRIMARY KEY (id) -) ; --- --- Table structure for table t_ds_alertgroup --- - -DROP TABLE IF EXISTS t_ds_alertgroup; -CREATE TABLE t_ds_alertgroup ( - id int NOT NULL , - group_name varchar(255) DEFAULT NULL , - group_type int DEFAULT NULL , - description varchar(255) DEFAULT NULL , - create_time timestamp DEFAULT NULL , - update_time timestamp DEFAULT NULL , - PRIMARY KEY (id) -) ; - --- --- Table structure for table t_ds_command --- - -DROP TABLE IF EXISTS t_ds_command; -CREATE TABLE t_ds_command ( - id int NOT NULL , - command_type int DEFAULT NULL , - process_definition_id int DEFAULT NULL , - command_param text , - task_depend_type int DEFAULT NULL , - failure_strategy int DEFAULT '0' , - warning_type int DEFAULT '0' , - warning_group_id int DEFAULT NULL , - schedule_time timestamp DEFAULT NULL , - start_time timestamp DEFAULT NULL , - executor_id int DEFAULT NULL , - dependence varchar(255) DEFAULT NULL , - update_time timestamp DEFAULT NULL , - process_instance_priority int DEFAULT NULL , - worker_group varchar(64), - PRIMARY KEY (id) -) ; - --- --- Table structure for table t_ds_datasource --- - -DROP TABLE IF EXISTS t_ds_datasource; -CREATE TABLE t_ds_datasource ( - id int NOT NULL , - name varchar(64) NOT NULL , - note varchar(256) DEFAULT NULL , - type int NOT NULL , - user_id int NOT NULL , - connection_params text NOT NULL , - create_time timestamp NOT NULL , - update_time timestamp DEFAULT NULL , - PRIMARY KEY (id) -) ; - --- --- Table structure for table t_ds_error_command --- - -DROP TABLE IF EXISTS t_ds_error_command; -CREATE TABLE t_ds_error_command ( - id int NOT NULL , - command_type int DEFAULT NULL , - executor_id int DEFAULT NULL , - process_definition_id int DEFAULT NULL , - command_param text , - task_depend_type int DEFAULT NULL , - failure_strategy int DEFAULT '0' , - warning_type int DEFAULT '0' , - warning_group_id int DEFAULT NULL , - schedule_time timestamp DEFAULT NULL , - start_time timestamp DEFAULT NULL , - update_time timestamp DEFAULT NULL , - dependence text , - process_instance_priority int DEFAULT NULL , - worker_group varchar(64), - message text , - PRIMARY KEY (id) -); --- --- Table structure for table t_ds_master_server --- - --- --- Table structure for table t_ds_process_definition --- - -DROP TABLE IF EXISTS t_ds_process_definition; -CREATE TABLE t_ds_process_definition ( - id int NOT NULL , - name varchar(255) DEFAULT NULL , - version int DEFAULT NULL , - release_state int DEFAULT NULL , - project_id int DEFAULT NULL , - user_id int DEFAULT NULL , - process_definition_json text , - description text , - global_params text , - flag int DEFAULT NULL , - locations text , - connects text , - receivers text , - receivers_cc text , - create_time timestamp DEFAULT NULL , - timeout int DEFAULT '0' , - tenant_id int NOT NULL DEFAULT '-1' , - update_time timestamp DEFAULT NULL , - modify_by varchar(36) DEFAULT '' , - resource_ids varchar(64), - PRIMARY KEY (id) -) ; - -create index process_definition_index on t_ds_process_definition (project_id,id); - --- --- Table structure for table t_ds_process_instance --- - -DROP TABLE IF EXISTS t_ds_process_instance; -CREATE TABLE t_ds_process_instance ( - id int NOT NULL , - name varchar(255) DEFAULT NULL , - process_definition_id int DEFAULT NULL , - state int DEFAULT NULL , - recovery int DEFAULT NULL , - start_time timestamp DEFAULT NULL , - end_time timestamp DEFAULT NULL , - run_times int DEFAULT NULL , - host varchar(45) DEFAULT NULL , - command_type int DEFAULT NULL , - command_param text , - task_depend_type int DEFAULT NULL , - max_try_times int DEFAULT '0' , - failure_strategy int DEFAULT '0' , - warning_type int DEFAULT '0' , - warning_group_id int DEFAULT NULL , - schedule_time timestamp DEFAULT NULL , - command_start_time timestamp DEFAULT NULL , - global_params text , - process_instance_json text , - flag int DEFAULT '1' , - update_time timestamp NULL , - is_sub_process int DEFAULT '0' , - executor_id int NOT NULL , - locations text , - connects text , - history_cmd text , - dependence_schedule_times text , - process_instance_priority int DEFAULT NULL , - worker_group varchar(64) , - timeout int DEFAULT '0' , - tenant_id int NOT NULL DEFAULT '-1' , - PRIMARY KEY (id) -) ; - create index process_instance_index on t_ds_process_instance (process_definition_id,id); - create index start_time_index on t_ds_process_instance (start_time); - --- --- Table structure for table t_ds_project --- - -DROP TABLE IF EXISTS t_ds_project; -CREATE TABLE t_ds_project ( - id int NOT NULL , - name varchar(100) DEFAULT NULL , - description varchar(200) DEFAULT NULL , - user_id int DEFAULT NULL , - flag int DEFAULT '1' , - create_time timestamp DEFAULT CURRENT_TIMESTAMP , - update_time timestamp DEFAULT CURRENT_TIMESTAMP , - PRIMARY KEY (id) -) ; - create index user_id_index on t_ds_project (user_id); - --- --- Table structure for table t_ds_queue --- - -DROP TABLE IF EXISTS t_ds_queue; -CREATE TABLE t_ds_queue ( - id int NOT NULL , - queue_name varchar(64) DEFAULT NULL , - queue varchar(64) DEFAULT NULL , - create_time timestamp DEFAULT NULL , - update_time timestamp DEFAULT NULL , - PRIMARY KEY (id) -); - - --- --- Table structure for table t_ds_relation_datasource_user --- - -DROP TABLE IF EXISTS t_ds_relation_datasource_user; -CREATE TABLE t_ds_relation_datasource_user ( - id int NOT NULL , - user_id int NOT NULL , - datasource_id int DEFAULT NULL , - perm int DEFAULT '1' , - create_time timestamp DEFAULT NULL , - update_time timestamp DEFAULT NULL , - PRIMARY KEY (id) -) ; -; - --- --- Table structure for table t_ds_relation_process_instance --- - -DROP TABLE IF EXISTS t_ds_relation_process_instance; -CREATE TABLE t_ds_relation_process_instance ( - id int NOT NULL , - parent_process_instance_id int DEFAULT NULL , - parent_task_instance_id int DEFAULT NULL , - process_instance_id int DEFAULT NULL , - PRIMARY KEY (id) -) ; - - --- --- Table structure for table t_ds_relation_project_user --- - -DROP TABLE IF EXISTS t_ds_relation_project_user; -CREATE TABLE t_ds_relation_project_user ( - id int NOT NULL , - user_id int NOT NULL , - project_id int DEFAULT NULL , - perm int DEFAULT '1' , - create_time timestamp DEFAULT NULL , - update_time timestamp DEFAULT NULL , - PRIMARY KEY (id) -) ; -create index relation_project_user_id_index on t_ds_relation_project_user (user_id); - --- --- Table structure for table t_ds_relation_resources_user --- - -DROP TABLE IF EXISTS t_ds_relation_resources_user; -CREATE TABLE t_ds_relation_resources_user ( - id int NOT NULL , - user_id int NOT NULL , - resources_id int DEFAULT NULL , - perm int DEFAULT '1' , - create_time timestamp DEFAULT NULL , - update_time timestamp DEFAULT NULL , - PRIMARY KEY (id) -) ; - --- --- Table structure for table t_ds_relation_udfs_user --- - -DROP TABLE IF EXISTS t_ds_relation_udfs_user; -CREATE TABLE t_ds_relation_udfs_user ( - id int NOT NULL , - user_id int NOT NULL , - udf_id int DEFAULT NULL , - perm int DEFAULT '1' , - create_time timestamp DEFAULT NULL , - update_time timestamp DEFAULT NULL , - PRIMARY KEY (id) -) ; -; - --- --- Table structure for table t_ds_relation_user_alertgroup --- - -DROP TABLE IF EXISTS t_ds_relation_user_alertgroup; -CREATE TABLE t_ds_relation_user_alertgroup ( - id int NOT NULL, - alertgroup_id int DEFAULT NULL, - user_id int DEFAULT NULL, - create_time timestamp DEFAULT NULL, - update_time timestamp DEFAULT NULL, - PRIMARY KEY (id) -); - --- --- Table structure for table t_ds_resources --- - -DROP TABLE IF EXISTS t_ds_resources; -CREATE TABLE t_ds_resources ( - id int NOT NULL , - alias varchar(64) DEFAULT NULL , - file_name varchar(64) DEFAULT NULL , - description varchar(256) DEFAULT NULL , - user_id int DEFAULT NULL , - type int DEFAULT NULL , - size bigint DEFAULT NULL , - create_time timestamp DEFAULT NULL , - update_time timestamp DEFAULT NULL , - pid int, - full_name varchar(64), - is_directory int, - PRIMARY KEY (id) -) ; - - --- --- Table structure for table t_ds_schedules --- - -DROP TABLE IF EXISTS t_ds_schedules; -CREATE TABLE t_ds_schedules ( - id int NOT NULL , - process_definition_id int NOT NULL , - start_time timestamp NOT NULL , - end_time timestamp NOT NULL , - crontab varchar(256) NOT NULL , - failure_strategy int NOT NULL , - user_id int NOT NULL , - release_state int NOT NULL , - warning_type int NOT NULL , - warning_group_id int DEFAULT NULL , - process_instance_priority int DEFAULT NULL , - worker_group varchar(64), - create_time timestamp NOT NULL , - update_time timestamp NOT NULL , - PRIMARY KEY (id) -); - --- --- Table structure for table t_ds_session --- - -DROP TABLE IF EXISTS t_ds_session; -CREATE TABLE t_ds_session ( - id varchar(64) NOT NULL , - user_id int DEFAULT NULL , - ip varchar(45) DEFAULT NULL , - last_login_time timestamp DEFAULT NULL , - PRIMARY KEY (id) -); - --- --- Table structure for table t_ds_task_instance --- - -DROP TABLE IF EXISTS t_ds_task_instance; -CREATE TABLE t_ds_task_instance ( - id int NOT NULL , - name varchar(255) DEFAULT NULL , - task_type varchar(64) DEFAULT NULL , - process_definition_id int DEFAULT NULL , - process_instance_id int DEFAULT NULL , - task_json text , - state int DEFAULT NULL , - submit_time timestamp DEFAULT NULL , - start_time timestamp DEFAULT NULL , - end_time timestamp DEFAULT NULL , - host varchar(45) DEFAULT NULL , - execute_path varchar(200) DEFAULT NULL , - log_path varchar(200) DEFAULT NULL , - alert_flag int DEFAULT NULL , - retry_times int DEFAULT '0' , - pid int DEFAULT NULL , - app_link varchar(255) DEFAULT NULL , - flag int DEFAULT '1' , - retry_interval int DEFAULT NULL , - max_retry_times int DEFAULT NULL , - task_instance_priority int DEFAULT NULL , - worker_group varchar(64), - executor_id int DEFAULT NULL , - PRIMARY KEY (id) -) ; - --- --- Table structure for table t_ds_tenant --- - -DROP TABLE IF EXISTS t_ds_tenant; -CREATE TABLE t_ds_tenant ( - id int NOT NULL , - tenant_code varchar(64) DEFAULT NULL , - tenant_name varchar(64) DEFAULT NULL , - description varchar(256) DEFAULT NULL , - queue_id int DEFAULT NULL , - create_time timestamp DEFAULT NULL , - update_time timestamp DEFAULT NULL , - PRIMARY KEY (id) -) ; - --- --- Table structure for table t_ds_udfs --- - -DROP TABLE IF EXISTS t_ds_udfs; -CREATE TABLE t_ds_udfs ( - id int NOT NULL , - user_id int NOT NULL , - func_name varchar(100) NOT NULL , - class_name varchar(255) NOT NULL , - type int NOT NULL , - arg_types varchar(255) DEFAULT NULL , - database varchar(255) DEFAULT NULL , - description varchar(255) DEFAULT NULL , - resource_id int NOT NULL , - resource_name varchar(255) NOT NULL , - create_time timestamp NOT NULL , - update_time timestamp NOT NULL , - PRIMARY KEY (id) -) ; - --- --- Table structure for table t_ds_user --- - -DROP TABLE IF EXISTS t_ds_user; -CREATE TABLE t_ds_user ( - id int NOT NULL , - user_name varchar(64) DEFAULT NULL , - user_password varchar(64) DEFAULT NULL , - user_type int DEFAULT NULL , - email varchar(64) DEFAULT NULL , - phone varchar(11) DEFAULT NULL , - tenant_id int DEFAULT NULL , - create_time timestamp DEFAULT NULL , - update_time timestamp DEFAULT NULL , - queue varchar(64) DEFAULT NULL , - PRIMARY KEY (id) -); - --- --- Table structure for table t_ds_version --- - -DROP TABLE IF EXISTS t_ds_version; -CREATE TABLE t_ds_version ( - id int NOT NULL , - version varchar(200) NOT NULL, - PRIMARY KEY (id) -) ; -create index version_index on t_ds_version(version); - --- --- Table structure for table t_ds_worker_group --- - -DROP TABLE IF EXISTS t_ds_worker_group; -CREATE TABLE t_ds_worker_group ( - id bigint NOT NULL , - name varchar(256) DEFAULT NULL , - ip_list varchar(256) DEFAULT NULL , - create_time timestamp DEFAULT NULL , - update_time timestamp DEFAULT NULL , - PRIMARY KEY (id) -) ; - --- --- Table structure for table t_ds_worker_server --- - -DROP TABLE IF EXISTS t_ds_worker_server; -CREATE TABLE t_ds_worker_server ( - id int NOT NULL , - host varchar(45) DEFAULT NULL , - port int DEFAULT NULL , - zk_directory varchar(64) DEFAULT NULL , - res_info varchar(255) DEFAULT NULL , - create_time timestamp DEFAULT NULL , - last_heartbeat_time timestamp DEFAULT NULL , - PRIMARY KEY (id) -) ; - - -DROP SEQUENCE IF EXISTS t_ds_access_token_id_sequence; -CREATE SEQUENCE t_ds_access_token_id_sequence; -ALTER TABLE t_ds_access_token ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_access_token_id_sequence'); -DROP SEQUENCE IF EXISTS t_ds_alert_id_sequence; -CREATE SEQUENCE t_ds_alert_id_sequence; -ALTER TABLE t_ds_alert ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_alert_id_sequence'); -DROP SEQUENCE IF EXISTS t_ds_alertgroup_id_sequence; -CREATE SEQUENCE t_ds_alertgroup_id_sequence; -ALTER TABLE t_ds_alertgroup ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_alertgroup_id_sequence'); - -DROP SEQUENCE IF EXISTS t_ds_command_id_sequence; -CREATE SEQUENCE t_ds_command_id_sequence; -ALTER TABLE t_ds_command ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_command_id_sequence'); -DROP SEQUENCE IF EXISTS t_ds_datasource_id_sequence; -CREATE SEQUENCE t_ds_datasource_id_sequence; -ALTER TABLE t_ds_datasource ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_datasource_id_sequence'); -DROP SEQUENCE IF EXISTS t_ds_process_definition_id_sequence; -CREATE SEQUENCE t_ds_process_definition_id_sequence; -ALTER TABLE t_ds_process_definition ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_process_definition_id_sequence'); -DROP SEQUENCE IF EXISTS t_ds_process_instance_id_sequence; -CREATE SEQUENCE t_ds_process_instance_id_sequence; -ALTER TABLE t_ds_process_instance ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_process_instance_id_sequence'); -DROP SEQUENCE IF EXISTS t_ds_project_id_sequence; -CREATE SEQUENCE t_ds_project_id_sequence; -ALTER TABLE t_ds_project ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_project_id_sequence'); -DROP SEQUENCE IF EXISTS t_ds_queue_id_sequence; -CREATE SEQUENCE t_ds_queue_id_sequence; -ALTER TABLE t_ds_queue ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_queue_id_sequence'); - -DROP SEQUENCE IF EXISTS t_ds_relation_datasource_user_id_sequence; -CREATE SEQUENCE t_ds_relation_datasource_user_id_sequence; -ALTER TABLE t_ds_relation_datasource_user ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_relation_datasource_user_id_sequence'); -DROP SEQUENCE IF EXISTS t_ds_relation_process_instance_id_sequence; -CREATE SEQUENCE t_ds_relation_process_instance_id_sequence; -ALTER TABLE t_ds_relation_process_instance ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_relation_process_instance_id_sequence'); -DROP SEQUENCE IF EXISTS t_ds_relation_project_user_id_sequence; -CREATE SEQUENCE t_ds_relation_project_user_id_sequence; -ALTER TABLE t_ds_relation_project_user ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_relation_project_user_id_sequence'); -DROP SEQUENCE IF EXISTS t_ds_relation_resources_user_id_sequence; -CREATE SEQUENCE t_ds_relation_resources_user_id_sequence; -ALTER TABLE t_ds_relation_resources_user ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_relation_resources_user_id_sequence'); -DROP SEQUENCE IF EXISTS t_ds_relation_udfs_user_id_sequence; -CREATE SEQUENCE t_ds_relation_udfs_user_id_sequence; -ALTER TABLE t_ds_relation_udfs_user ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_relation_udfs_user_id_sequence'); -DROP SEQUENCE IF EXISTS t_ds_relation_user_alertgroup_id_sequence; -CREATE SEQUENCE t_ds_relation_user_alertgroup_id_sequence; -ALTER TABLE t_ds_relation_user_alertgroup ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_relation_user_alertgroup_id_sequence'); - -DROP SEQUENCE IF EXISTS t_ds_resources_id_sequence; -CREATE SEQUENCE t_ds_resources_id_sequence; -ALTER TABLE t_ds_resources ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_resources_id_sequence'); -DROP SEQUENCE IF EXISTS t_ds_schedules_id_sequence; -CREATE SEQUENCE t_ds_schedules_id_sequence; -ALTER TABLE t_ds_schedules ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_schedules_id_sequence'); -DROP SEQUENCE IF EXISTS t_ds_task_instance_id_sequence; -CREATE SEQUENCE t_ds_task_instance_id_sequence; -ALTER TABLE t_ds_task_instance ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_task_instance_id_sequence'); -DROP SEQUENCE IF EXISTS t_ds_tenant_id_sequence; -CREATE SEQUENCE t_ds_tenant_id_sequence; -ALTER TABLE t_ds_tenant ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_tenant_id_sequence'); -DROP SEQUENCE IF EXISTS t_ds_udfs_id_sequence; -CREATE SEQUENCE t_ds_udfs_id_sequence; -ALTER TABLE t_ds_udfs ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_udfs_id_sequence'); -DROP SEQUENCE IF EXISTS t_ds_user_id_sequence; -CREATE SEQUENCE t_ds_user_id_sequence; -ALTER TABLE t_ds_user ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_user_id_sequence'); - -DROP SEQUENCE IF EXISTS t_ds_version_id_sequence; -CREATE SEQUENCE t_ds_version_id_sequence; -ALTER TABLE t_ds_version ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_version_id_sequence'); - -DROP SEQUENCE IF EXISTS t_ds_worker_group_id_sequence; -CREATE SEQUENCE t_ds_worker_group_id_sequence; -ALTER TABLE t_ds_worker_group ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_worker_group_id_sequence'); -DROP SEQUENCE IF EXISTS t_ds_worker_server_id_sequence; -CREATE SEQUENCE t_ds_worker_server_id_sequence; -ALTER TABLE t_ds_worker_server ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_worker_server_id_sequence'); - - --- Records of t_ds_user?user : admin , password : dolphinscheduler123 -INSERT INTO t_ds_user(user_name,user_password,user_type,email,phone,tenant_id,create_time,update_time) VALUES ('admin', '7ad2410b2f4c074479a8937a28a22b8f', '0', 'xxx@qq.com', 'xx', '0', '2018-03-27 15:48:50', '2018-10-24 17:40:22'); - --- Records of t_ds_alertgroup，dolphinscheduler warning group -INSERT INTO t_ds_alertgroup(group_name,group_type,description,create_time,update_time) VALUES ('dolphinscheduler warning group', '0', 'dolphinscheduler warning group','2018-11-29 10:20:39', '2018-11-29 10:20:39'); -INSERT INTO t_ds_relation_user_alertgroup(alertgroup_id,user_id,create_time,update_time) VALUES ( '1', '1', '2018-11-29 10:22:33', '2018-11-29 10:22:33'); - --- Records of t_ds_queue,default queue name : default -INSERT INTO t_ds_queue(queue_name,queue,create_time,update_time) VALUES ('default', 'default','2018-11-29 10:22:33', '2018-11-29 10:22:33'); - --- Records of t_ds_queue,default queue name : default -INSERT INTO t_ds_version(version) VALUES ('1.3.0'); \ No newline at end of file diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapperTest.java index 9dafbe138c..e92ec54540 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapperTest.java @@ -73,6 +73,22 @@ public class ProcessDefinitionMapperTest { return processDefinition; } + /** + * insert + * @return ProcessDefinition + */ + private ProcessDefinition insertTwo(){ + //insertOne + ProcessDefinition processDefinition = new ProcessDefinition(); + processDefinition.setName("def 2"); + processDefinition.setProjectId(1010); + processDefinition.setUserId(101); + processDefinition.setUpdateTime(new Date()); + processDefinition.setCreateTime(new Date()); + processDefinitionMapper.insert(processDefinition); + return processDefinition; + } + /** * test update */ @@ -177,7 +193,7 @@ public class ProcessDefinitionMapperTest { public void testQueryDefinitionListByIdList() { ProcessDefinition processDefinition = insertOne(); - ProcessDefinition processDefinition1 = insertOne(); + ProcessDefinition processDefinition1 = insertTwo(); Integer[] array = new Integer[2]; array[0] = processDefinition.getId(); diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/RemoveTaskLogRequestCommandTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/RemoveTaskLogRequestCommandTest.java new file mode 100644 index 0000000000..37c21064c4 --- /dev/null +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/RemoveTaskLogRequestCommandTest.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.remote; + +import junit.framework.Assert; +import org.apache.dolphinscheduler.remote.command.Command; +import org.apache.dolphinscheduler.remote.command.log.RemoveTaskLogRequestCommand; +import org.apache.dolphinscheduler.remote.command.log.RemoveTaskLogResponseCommand; +import org.junit.Test; + +public class RemoveTaskLogRequestCommandTest { + + @Test + public void testConvert2Command(){ + RemoveTaskLogResponseCommand removeTaskLogResponseCommand = new RemoveTaskLogResponseCommand(); + removeTaskLogResponseCommand.setStatus(true); + Command command = removeTaskLogResponseCommand.convert2Command(122); + Assert.assertNotNull(command); + } +} diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/RemoveTaskLogResponseCommandTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/RemoveTaskLogResponseCommandTest.java new file mode 100644 index 0000000000..aab0ad336f --- /dev/null +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/RemoveTaskLogResponseCommandTest.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.remote; + +import junit.framework.Assert; +import org.apache.dolphinscheduler.remote.command.Command; +import org.apache.dolphinscheduler.remote.command.log.RemoveTaskLogRequestCommand; +import org.junit.Test; + +public class RemoveTaskLogResponseCommandTest { + + @Test + public void testConvert2Command(){ + RemoveTaskLogRequestCommand removeTaskLogRequestCommand = new RemoveTaskLogRequestCommand(); + removeTaskLogRequestCommand.setPath("/opt/zhangsan"); + Command command = removeTaskLogRequestCommand.convert2Command(); + Assert.assertNotNull(command); + } +} diff --git a/dolphinscheduler-server/src/main/resources/config/install_config.conf b/dolphinscheduler-server/src/main/resources/config/install_config.conf index f6b09b9cb8..6e24bd9f89 100644 --- a/dolphinscheduler-server/src/main/resources/config/install_config.conf +++ b/dolphinscheduler-server/src/main/resources/config/install_config.conf @@ -130,7 +130,8 @@ masters="ds1,ds2" # run worker machine # note: need to write the worker group name of each worker, the default value is "default" -workersGroup=(["ds1"]="default" ["ds2"]="default" ["ds3"]="default" ["ds4"]="default" ["ds5"]="default") + +declare -A workersGroup=(["ds1"]="default" ["ds2"]="default" ["ds3"]="default" ["ds4"]="default" ["ds5"]="default") # run alert machine # note: list of machine hostnames for deploying alert server @@ -141,4 +142,4 @@ alertServer="ds3" apiServers="ds1" # whether to start monitoring self-starting scripts -monitorServerState="false" +monitorServerState="false" \ No newline at end of file diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerServerTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerServerTest.java new file mode 100644 index 0000000000..0da88746f5 --- /dev/null +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerServerTest.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.log; + +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.service.log.LogClientService; +import org.junit.Test; + +public class LoggerServerTest { + + + @Test + public void testRollViewLog(){ + LoggerServer loggerServer = new LoggerServer(); + loggerServer.start(); + + LogClientService logClientService = new LogClientService(); + logClientService.rollViewLog("localhost", Constants.RPC_PORT,"/opt/demo.txt",0,1000); + + try { + Thread.sleep(5000); + } catch (InterruptedException e) { + + } + + loggerServer.stop(); + logClientService.close(); + } + + @Test + public void testRemoveTaskLog(){ + LoggerServer loggerServer = new LoggerServer(); + loggerServer.start(); + + LogClientService logClientService = new LogClientService(); + logClientService.removeTaskLog("localhost", Constants.RPC_PORT,"/opt/zhangsan"); + + try { + Thread.sleep(5000); + } catch (InterruptedException e) { + + } + + loggerServer.stop(); + logClientService.close(); + } +} diff --git a/dolphinscheduler-service/pom.xml b/dolphinscheduler-service/pom.xml index c150e834b9..62b0954afe 100644 --- a/dolphinscheduler-service/pom.xml +++ b/dolphinscheduler-service/pom.xml @@ -72,5 +72,27 @@ org.quartz-scheduler quartz-jobs + + + org.powermock + powermock-module-junit4 + test + + + org.powermock + powermock-api-mockito2 + test + + + org.mockito + mockito-core + + + + + org.mockito + mockito-core + test + diff --git a/pom.xml b/pom.xml index 7e291d2c4e..13a4c33eeb 100644 --- a/pom.xml +++ b/pom.xml @@ -777,9 +777,12 @@ **/dao/mapper/CommandMapperTest.java **/dao/mapper/ConnectionFactoryTest.java **/dao/mapper/DataSourceMapperTest.java + **/remote/RemoveTaskLogResponseCommandTest.java + **/remote/RemoveTaskLogRequestCommandTest.java **/remote/FastJsonSerializerTest.java **/remote/NettyRemotingClientTest.java **/remote/ResponseFutureTest.java + **/server/log/LoggerServerTest.java **/server/log/MasterLogFilterTest.java **/server/log/SensitiveDataConverterTest.java **/server/log/TaskLogDiscriminatorTest.java diff --git a/script/scp-hosts.sh b/script/scp-hosts.sh index adee7d76c7..6e85566d4b 100644 --- a/script/scp-hosts.sh +++ b/script/scp-hosts.sh @@ -40,8 +40,8 @@ do for dsDir in bin conf lib script sql ui install.sh do # if worker in workersGroup - if [[ "${map[${host}]}" ]] && [[ "${dsDir}" -eq "conf" ]]; then - sed -i ${txt} "s#worker.group.*#worker.group=${map[${host}]}#g" $workDir/../conf/worker.properties + if [[ "${workersGroup[${host}]}" ]] && [[ "${dsDir}" == "conf" ]]; then + sed -i ${txt} "s#worker.group.*#worker.group=${workersGroup[${host}]}#g" ${dsDir}/worker.properties fi echo "start to scp $dsDir to $host/$installPath" @@ -49,4 +49,4 @@ do done echo "scp dirs to $host/$installPath complete" -done +done \ No newline at end of file diff --git a/script/start-all.sh b/script/start-all.sh index 5579a7d87e..dda46f0c06 100644 --- a/script/start-all.sh +++ b/script/start-all.sh @@ -43,5 +43,4 @@ for apiServer in ${apiServersHost[@]} do echo "$apiServer worker server is starting" ssh -p $sshPort $apiServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start api-server;" -done - +done \ No newline at end of file diff --git a/script/stop-all.sh b/script/stop-all.sh index e4ccf75a29..e2afd087a9 100644 --- a/script/stop-all.sh +++ b/script/stop-all.sh @@ -43,5 +43,4 @@ for apiServer in ${apiServersHost[@]} do echo "$apiServer worker server is stopping" ssh -p $sshPort $apiServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop api-server;" -done - +done \ No newline at end of file diff --git a/sql/upgrade/1.3.0_schema/postgresql/dolphinscheduler_ddl.sql b/sql/upgrade/1.3.0_schema/postgresql/dolphinscheduler_ddl.sql index 61cbfed4fe..16a49a897e 100644 --- a/sql/upgrade/1.3.0_schema/postgresql/dolphinscheduler_ddl.sql +++ b/sql/upgrade/1.3.0_schema/postgresql/dolphinscheduler_ddl.sql @@ -368,9 +368,9 @@ DROP FUNCTION dc_dolphin_T_t_ds_error_command_D_worker_group_id(); delimiter d// CREATE OR REPLACE FUNCTION uc_dolphin_T_t_ds_process_definition_A_process_definition_unique() RETURNS void AS $$ BEGIN - IF NOT EXISTS (SELECT 1 FROM information_schema.STATISTICS - WHERE TABLE_NAME='t_ds_process_definition' - AND INDEX_NAME ='process_definition_unique') + IF NOT EXISTS (SELECT 1 FROM pg_stat_all_indexes + WHERE relname='t_ds_process_definition' + AND indexrelname ='process_definition_unique') THEN ALTER TABLE t_ds_process_definition ADD CONSTRAINT process_definition_unique UNIQUE (name,project_id); END IF;