From 1b629bb03dcf1dd00292c16d3d059090e00bef3d Mon Sep 17 00:00:00 2001
From: xiangzihao <460888207@qq.com>
Date: Fri, 6 Sep 2024 11:05:16 +0800
Subject: [PATCH] [Chore] Improve the robustness of CI (#16592)

---
 .github/workflows/api-test.yml                |  4 +-
 .github/workflows/backend.yml                 | 49 +++++++++++++++----
 .github/workflows/e2e.yml                     |  4 +-
 .github/workflows/publish-docker.yaml         |  2 +-
 .../workflows/schema-check/mysql/start-job.sh |  6 ++-
 .../schema-check/postgresql/start-job.sh      |  6 ++-
 .github/workflows/unit-test.yml               |  2 +-
 7 files changed, 55 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/api-test.yml b/.github/workflows/api-test.yml
index 90a2e9aa17..d3194dac23 100644
--- a/.github/workflows/api-test.yml
+++ b/.github/workflows/api-test.yml
@@ -62,7 +62,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-api-test
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-backend
           restore-keys: ${{ runner.os }}-maven-
       - name: Build Image
         run: |
@@ -122,7 +122,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-api-test
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-backend
           restore-keys: ${{ runner.os }}-maven-
       - uses: actions/download-artifact@v4
         name: Download Docker Images
diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
index 0e98226e2a..ab336eefea 100644
--- a/.github/workflows/backend.yml
+++ b/.github/workflows/backend.yml
@@ -21,15 +21,8 @@ on:
   push:
     branches:
       - dev
-    paths:
-      - '.github/workflows/backend.yml'
-      - 'package.xml'
-      - 'pom.xml'
-      - 'dolphinscheduler-alert/**'
-      - 'dolphinscheduler-api/**'
-      - 'dolphinscheduler-common/**'
-      - 'dolphinscheduler-dao/**'
-      - 'dolphinscheduler-rpc/**'
+      - '[0-9]+.[0-9]+.[0-9]+-prepare'
+      - '[0-9]+.[0-9]+.[0-9]+-release'
   pull_request:
 
 concurrency:
@@ -131,9 +124,39 @@ jobs:
       - name: Running cluster test
         run: |
           /bin/bash ${{ matrix.case.script }}
+  schema-check-prepare:
+    name: schema-check-prepare-${{ matrix.version }}
+    needs: paths-filter
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        version: [ "3.1.9", "3.2.0" ]
+    steps:
+      - uses: actions/cache/restore@v4
+        name: Download Binary Package Cache
+        with:
+          path: /tmp/${{ matrix.version }}
+          key: apache-dolphinscheduler-${{ matrix.version }}-bin
+          restore-keys: apache-dolphinscheduler-${{ matrix.version }}-bin
+      - name: Download Binary Package if not cached
+        run: |
+          if [ ! -f /tmp/${{ matrix.version }}/apache-dolphinscheduler-${{ matrix.version }}-bin.tar.gz ]; then
+            echo "Binary package not found in cache, downloading..."
+            mkdir -p /tmp/${{ matrix.version }}
+            wget https://mirrors.tuna.tsinghua.edu.cn/apache/dolphinscheduler/${{ matrix.version }}/apache-dolphinscheduler-${{ matrix.version }}-bin.tar.gz -P /tmp/${{ matrix.version }}
+          else
+            echo "Binary package found in cache, skipping download."
+          fi
+      - uses: actions/cache/save@v4
+        name: Upload Binary Package Cache
+        with:
+          path: /tmp/${{ matrix.version }}
+          key: apache-dolphinscheduler-${{ matrix.version }}-bin
   schema-check:
     name: ${{ matrix.case.name }}-${{ matrix.version }}
-    needs: build
+    needs: [ build, schema-check-prepare]
     runs-on: ubuntu-latest
     timeout-minutes: 20
     strategy:
@@ -158,6 +181,12 @@ jobs:
         with:
           name: binary-package-8
           path: ds_schema_check_test/dev
+      - uses: actions/cache/restore@v4
+        name: Download Binary Package Cache
+        with:
+          path: /tmp/${{ matrix.version }}
+          key: apache-dolphinscheduler-${{ matrix.version }}-bin
+          restore-keys: apache-dolphinscheduler-${{ matrix.version }}-bin
       - name: Running Schema Check
         run: |
           /bin/bash ${{ matrix.case.script }} ${{ matrix.version }}
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index a24cf820f8..cf7e7c5e2e 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -64,7 +64,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-backend
           restore-keys: ${{ runner.os }}-maven-
       - name: Build Image
         run: |
@@ -151,7 +151,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-backend
           restore-keys: ${{ runner.os }}-maven-
       - uses: actions/download-artifact@v4
         name: Download Docker Images
diff --git a/.github/workflows/publish-docker.yaml b/.github/workflows/publish-docker.yaml
index 1c67bf6d86..19eaab9f25 100644
--- a/.github/workflows/publish-docker.yaml
+++ b/.github/workflows/publish-docker.yaml
@@ -51,7 +51,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-docker
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-backend
           restore-keys: ${{ runner.os }}-maven-
       - name: Set environment variables
         run: |
diff --git a/.github/workflows/schema-check/mysql/start-job.sh b/.github/workflows/schema-check/mysql/start-job.sh
index 4ca8ee4810..852c7f8a0e 100644
--- a/.github/workflows/schema-check/mysql/start-job.sh
+++ b/.github/workflows/schema-check/mysql/start-job.sh
@@ -25,7 +25,11 @@ mkdir -p ds_schema_check_test/dev ds_schema_check_test/${DS_VERSION}
 curl -sSf https://atlasgo.sh | sh
 
 # Preparing the environment
-wget https://archive.apache.org/dist/dolphinscheduler/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz -P ds_schema_check_test/${DS_VERSION}
+if [ ! -f /tmp/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz ]; then
+  wget https://archive.apache.org/dist/dolphinscheduler/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz -P ds_schema_check_test/${DS_VERSION}
+else
+  mv /tmp/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz ds_schema_check_test/${DS_VERSION}
+fi
 tar -xzf ds_schema_check_test/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz -C ds_schema_check_test/${DS_VERSION} --strip-components 1
 tar -xzf ds_schema_check_test/dev/apache-dolphinscheduler-*-bin.tar.gz -C ds_schema_check_test/dev --strip-components 1
 
diff --git a/.github/workflows/schema-check/postgresql/start-job.sh b/.github/workflows/schema-check/postgresql/start-job.sh
index 2d71794fe6..26308923e2 100644
--- a/.github/workflows/schema-check/postgresql/start-job.sh
+++ b/.github/workflows/schema-check/postgresql/start-job.sh
@@ -25,7 +25,11 @@ mkdir -p ds_schema_check_test/dev ds_schema_check_test/${DS_VERSION}
 curl -sSf https://atlasgo.sh | sh
 
 # Preparing the environment
-wget https://archive.apache.org/dist/dolphinscheduler/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz -P ds_schema_check_test/${DS_VERSION}
+if [ ! -f /tmp/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz ]; then
+  wget https://archive.apache.org/dist/dolphinscheduler/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz -P ds_schema_check_test/${DS_VERSION}
+else
+  mv /tmp/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz ds_schema_check_test/${DS_VERSION}
+fi
 tar -xzf ds_schema_check_test/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz -C ds_schema_check_test/${DS_VERSION} --strip-components 1
 tar -xzf ds_schema_check_test/dev/apache-dolphinscheduler-*-bin.tar.gz -C ds_schema_check_test/dev --strip-components 1
 
diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml
index c0bc86dbf0..273b934f5c 100644
--- a/.github/workflows/unit-test.yml
+++ b/.github/workflows/unit-test.yml
@@ -74,7 +74,7 @@ jobs:
       - uses: actions/cache@v4
         with:
           path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-unit-test
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-backend
           restore-keys: ${{ runner.os }}-maven-
       - name: Run Unit tests
         run: ./mvnw clean verify -B -Dmaven.test.skip=false -Dspotless.skip=true -DskipUT=false