DolphinScheduler/.github/workflows/backend.yml

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

name: Backend

on:
  push:
    branches:
      - dev
    paths:
      - '.github/workflows/backend.yml'
      - 'package.xml'
      - 'pom.xml'
      - 'dolphinscheduler-alert/**'
      - 'dolphinscheduler-api/**'
      - 'dolphinscheduler-common/**'
      - 'dolphinscheduler-dao/**'
      - 'dolphinscheduler-rpc/**'
  pull_request:

concurrency:
  group: backend-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  paths-filter:
    name: Backend-Path-Filter
    runs-on: ubuntu-latest
    outputs:
      not-ignore: ${{ steps.filter.outputs.not-ignore }}
      db-schema: ${{ steps.filter.outputs.db-schema }}
    steps:
      - uses: actions/checkout@v2
      - uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721
        id: filter
        with:
          filters: |
            not-ignore:
              - '!(docs/**)'
            db-schema:
              - 'dolphinscheduler-dao/src/main/resources/sql/**'            
  build:
    name: Backend-Build
    needs: paths-filter
    if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }}
    runs-on: ubuntu-latest
    strategy:
      matrix:
        java: [ '8', '11' ]
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: true
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v2
        with:
          java-version: ${{ matrix.java }}
          distribution: 'adopt'
      - name: Sanity Check
        uses: ./.github/actions/sanity-check
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
      - uses: actions/cache@v3
        with:
          path: ~/.m2/repository
          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-backend
          restore-keys: ${{ runner.os }}-maven-
      - name: Build and Package on ${{ matrix.java }}
        run: |
          ./mvnw -B clean install \
                 -Prelease \
                 -Dmaven.test.skip=true \
                 -Dspotless.skip=true \
                 -Dhttp.keepAlive=false \
                 -Dmaven.wagon.http.pool=false \
                 -Dmaven.wagon.httpconnectionManager.ttlSeconds=120          
      - name: Check dependency license
        run: tools/dependencies/check-LICENSE.sh
      - uses: actions/upload-artifact@v2
        if: ${{ matrix.java == '8' }}
        name: Upload Binary Package
        with:
          name: binary-package-${{ matrix.java }}
          path: ./dolphinscheduler-dist/target/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz
          retention-days: 1
  cluster-test:
    name: ${{ matrix.case.name }}
    needs: build
    runs-on: ubuntu-latest
    timeout-minutes: 20
    strategy:
      matrix:
        case:
          - name: cluster-test-mysql
            script: .github/workflows/cluster-test/mysql/start-job.sh
          - name: cluster-test-postgresql
            script: .github/workflows/cluster-test/postgresql/start-job.sh
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: true
      - uses: actions/download-artifact@v2
        name: Download Binary Package
        with:
          # Only run cluster test on jdk8
          name: binary-package-8
          path: ./
      - name: Running cluster test
        run: |
                    /bin/bash ${{ matrix.case.script }}
  schema-check:
    runs-on: ubuntu-latest
    if: ${{ (needs.paths-filter.outputs.db-schema == 'true') || (github.event_name == 'push') }}
    timeout-minutes: 20
    needs: build
    services:
      mysql:
        image: mysql:5.7
        env:
          MYSQL_ROOT_PASSWORD: mysql
          MYSQL_DATABASE: dolphinscheduler_dev
        ports:
          - 3306:3306
        options: --name=mysql --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
      postgres:
        image: postgres:15
        env:
          POSTGRES_PASSWORD: postgres
          POSTGRES_DB: dolphinscheduler_dev
        ports:
          - 5432:5432
        options: --name=postgres --health-cmd=pg_isready --health-interval=10s --health-timeout=5s --health-retries=5
    strategy:
      fail-fast: false
      matrix:
        db: ["mysql", "postgresql"]
        version: ["2.0.9", "3.0.6", "3.1.8"]
    steps:
      - name: Set up JDK 8
        uses: actions/setup-java@v2
        with:
          java-version: 8
          distribution: 'adopt'
      - name: Install Atlas and Create Dir
        run: |
          mkdir -p dolphinscheduler/dev dolphinscheduler/${{ matrix.version }}
          curl -sSf https://atlasgo.sh | sh          
      - name: Download Tarball
        uses: actions/download-artifact@v2
        with:
          name: binary-package-8
          path: dolphinscheduler/dev
      - name: Set Env
        run: |
          VERSION=${{ matrix.version }}
          echo "DATABASE_VERSION=${VERSION//\./}" >> $GITHUB_ENV          
      - name: Prepare
        run: |
          wget https://dlcdn.apache.org/dolphinscheduler/${{ matrix.version }}/apache-dolphinscheduler-${{ matrix.version }}-bin.tar.gz -P dolphinscheduler/${{ matrix.version }}
          tar -xzf dolphinscheduler/${{ matrix.version }}/apache-dolphinscheduler-${{ matrix.version }}-bin.tar.gz -C dolphinscheduler/${{ matrix.version }} --strip-components 1
          tar -xzf dolphinscheduler/dev/apache-dolphinscheduler-*-bin.tar.gz -C dolphinscheduler/dev --strip-components 1

          if [[ ${{ matrix.db }} == "mysql" ]]; then
            MYSQL_JDBC_URL="https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar"
            MYSQL_JDBC_JAR="mysql-connector-java-8.0.16.jar"
            wget ${MYSQL_JDBC_URL} -O /tmp/${MYSQL_JDBC_JAR}
            for base_dir in dolphinscheduler/dev dolphinscheduler/${{ matrix.version }}; do
              if [[ $base_dir == *"dolphinscheduler/2"* ]]; then
                cp /tmp/${MYSQL_JDBC_JAR} ${base_dir}/lib
              else
                for d in alert-server api-server master-server worker-server tools; do
                  cp /tmp/${MYSQL_JDBC_JAR} ${base_dir}/${d}/libs
                done
              fi
            done
            docker exec -i mysql mysql -uroot -pmysql -e "create database dolphinscheduler_${{ env.DATABASE_VERSION }}"; 
          else
            docker exec -i postgres psql -U postgres -c "create database dolphinscheduler_${{ env.DATABASE_VERSION }};"
          fi          
      - name: Check
        run: |
          if [[ $DATABASE_VERSION -lt 300 ]]; then
            chmod +x dolphinscheduler/dev/tools/bin/upgrade-schema.sh dolphinscheduler/${{ matrix.version }}/script/create-dolphinscheduler.sh
          else
            chmod +x dolphinscheduler/dev/tools/bin/upgrade-schema.sh dolphinscheduler/${{ matrix.version }}/tools/bin/upgrade-schema.sh
          fi
          if [[ ${{ matrix.db }} == "mysql" ]]; then
            export DATABASE="mysql"
            export SPRING_DATASOURCE_DRIVER_CLASS_NAME="com.mysql.cj.jdbc.Driver"
            export SPRING_DATASOURCE_URL="jdbc:mysql://127.0.0.1:3306/dolphinscheduler_dev?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true&useSSL=false"
            export SPRING_DATASOURCE_USERNAME="root"
            export SPRING_DATASOURCE_PASSWORD="mysql"
            bash dolphinscheduler/dev/tools/bin/upgrade-schema.sh

            export SPRING_DATASOURCE_URL="jdbc:mysql://127.0.0.1:3306/dolphinscheduler_${{ env.DATABASE_VERSION }}?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true&useSSL=false"
            if [[ $DATABASE_VERSION -lt 300 ]]; then
              bash dolphinscheduler/${{ matrix.version }}/script/create-dolphinscheduler.sh
            else
              bash dolphinscheduler/${{ matrix.version }}/tools/bin/upgrade-schema.sh
            fi
            bash dolphinscheduler/dev/tools/bin/upgrade-schema.sh

            atlas_result=$(atlas schema diff \
              --from "mysql://root:mysql@127.0.0.1:3306/dolphinscheduler_${{ env.DATABASE_VERSION }}" \
              --to "mysql://root:mysql@127.0.0.1:3306/dolphinscheduler_dev")
            if [[ ${atlas_result} != *"Schemas are synced"* ]]; then
              echo "================================================================================================"
              echo "                                !!!!! For Contributors !!!!!"
              echo "================================================================================================"
              echo "Database schema not sync, please add below change in the latest version of dolphinscheduler-dao/src/main/resources/sql/upgrade directory"
              echo "${atlas_result}"
              exit 1
            fi
          else
            export DATABASE="postgresql"
            export SPRING_DATASOURCE_DRIVER_CLASS_NAME="org.postgresql.Driver"
            export SPRING_DATASOURCE_URL="jdbc:postgresql://127.0.0.1:5432/dolphinscheduler_dev"
            export SPRING_DATASOURCE_USERNAME="postgres"
            export SPRING_DATASOURCE_PASSWORD="postgres"
            bash dolphinscheduler/dev/tools/bin/upgrade-schema.sh
          
            export SPRING_DATASOURCE_URL="jdbc:postgresql://127.0.0.1:5432/dolphinscheduler_${{ env.DATABASE_VERSION }}"
            if [[ $DATABASE_VERSION -lt 300 ]]; then
              bash dolphinscheduler/${{ matrix.version }}/script/create-dolphinscheduler.sh
            else
              bash dolphinscheduler/${{ matrix.version }}/tools/bin/upgrade-schema.sh
            fi
            bash dolphinscheduler/dev/tools/bin/upgrade-schema.sh
          
            atlas_result=$(atlas schema diff \
              --from "postgres://postgres:postgres@127.0.0.1:5432/dolphinscheduler_${{ env.DATABASE_VERSION }}?search_path=public&sslmode=disable" \
              --to "postgres://postgres:postgres@127.0.0.1:5432/dolphinscheduler_dev?search_path=public&sslmode=disable")
            if [[ ${atlas_result} != *"Schemas are synced"* ]]; then
              echo "================================================================================================"
              echo "                                !!!!! For Contributors !!!!!"
              echo "================================================================================================"
              echo "Database schema not sync, please add below change in the latest version in dolphinscheduler-dao/src/main/resources/sql/upgrade directory"
              echo "${atlas_result}"
              exit 1
            fi
          fi          
  result:
    name: Build
    runs-on: ubuntu-latest
    timeout-minutes: 30
    needs: [ build, paths-filter, cluster-test, schema-check ]
    if: always()
    steps:
      - name: Status
        run: |
          if [[ ${{ needs.paths-filter.outputs.not-ignore }} == 'false' && ${{ needs.paths-filter.outputs.db-schema }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then
            echo "Skip Build!"
            exit 0
          fi
          if [[ ${{ needs.build.result }} != 'success' || ${{ needs.cluster-test.result }} != 'success' ]]; then
            echo "Build Failed!"
            exit -1
          fi