[Chore] Improve the robustness of CI (#16592)

.github/workflows/api-test.yml

@@ -62,7 +62,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-api-test
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-backend
           restore-keys: ${{ runner.os }}-maven-
       - name: Build Image
         run: |
@@ -122,7 +122,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-api-test
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-backend
           restore-keys: ${{ runner.os }}-maven-
       - uses: actions/download-artifact@v4
         name: Download Docker Images

.github/workflows/backend.yml

@@ -21,15 +21,8 @@ on:
   push:
     branches:
       - dev
-    paths:
-      - '.github/workflows/backend.yml'
-      - 'package.xml'
-      - 'pom.xml'
-      - 'dolphinscheduler-alert/**'
-      - 'dolphinscheduler-api/**'
-      - 'dolphinscheduler-common/**'
-      - 'dolphinscheduler-dao/**'
-      - 'dolphinscheduler-rpc/**'
+      - '[0-9]+.[0-9]+.[0-9]+-prepare'
+      - '[0-9]+.[0-9]+.[0-9]+-release'
   pull_request:
 
 concurrency:
@@ -131,9 +124,39 @@ jobs:
       - name: Running cluster test
         run: |
           /bin/bash ${{ matrix.case.script }}
+  schema-check-prepare:
+    name: schema-check-prepare-${{ matrix.version }}
+    needs: paths-filter
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        version: [ "3.1.9", "3.2.0" ]
+    steps:
+      - uses: actions/cache/restore@v4
+        name: Download Binary Package Cache
+        with:
+          path: /tmp/${{ matrix.version }}
+          key: apache-dolphinscheduler-${{ matrix.version }}-bin
+          restore-keys: apache-dolphinscheduler-${{ matrix.version }}-bin
+      - name: Download Binary Package if not cached
+        run: |
+          if [ ! -f /tmp/${{ matrix.version }}/apache-dolphinscheduler-${{ matrix.version }}-bin.tar.gz ]; then
+            echo "Binary package not found in cache, downloading..."
+            mkdir -p /tmp/${{ matrix.version }}
+            wget https://mirrors.tuna.tsinghua.edu.cn/apache/dolphinscheduler/${{ matrix.version }}/apache-dolphinscheduler-${{ matrix.version }}-bin.tar.gz -P /tmp/${{ matrix.version }}
+          else
+            echo "Binary package found in cache, skipping download."
+          fi
+      - uses: actions/cache/save@v4
+        name: Upload Binary Package Cache
+        with:
+          path: /tmp/${{ matrix.version }}
+          key: apache-dolphinscheduler-${{ matrix.version }}-bin
   schema-check:
     name: ${{ matrix.case.name }}-${{ matrix.version }}
-    needs: build
+    needs: [ build, schema-check-prepare]
     runs-on: ubuntu-latest
     timeout-minutes: 20
     strategy:
@@ -158,6 +181,12 @@ jobs:
         with:
           name: binary-package-8
           path: ds_schema_check_test/dev
+      - uses: actions/cache/restore@v4
+        name: Download Binary Package Cache
+        with:
+          path: /tmp/${{ matrix.version }}
+          key: apache-dolphinscheduler-${{ matrix.version }}-bin
+          restore-keys: apache-dolphinscheduler-${{ matrix.version }}-bin
       - name: Running Schema Check
         run: |
           /bin/bash ${{ matrix.case.script }} ${{ matrix.version }}

.github/workflows/e2e.yml

@@ -64,7 +64,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-backend
           restore-keys: ${{ runner.os }}-maven-
       - name: Build Image
         run: |
@@ -151,7 +151,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-backend
           restore-keys: ${{ runner.os }}-maven-
       - uses: actions/download-artifact@v4
         name: Download Docker Images

.github/workflows/publish-docker.yaml

@@ -51,7 +51,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-docker
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-backend
           restore-keys: ${{ runner.os }}-maven-
       - name: Set environment variables
         run: |

.github/workflows/schema-check/mysql/start-job.sh

@@ -25,7 +25,11 @@ mkdir -p ds_schema_check_test/dev ds_schema_check_test/${DS_VERSION}
 curl -sSf https://atlasgo.sh | sh
 
 # Preparing the environment
-wget https://archive.apache.org/dist/dolphinscheduler/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz -P ds_schema_check_test/${DS_VERSION}
+if [ ! -f /tmp/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz ]; then
+  wget https://archive.apache.org/dist/dolphinscheduler/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz -P ds_schema_check_test/${DS_VERSION}
+else
+  mv /tmp/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz ds_schema_check_test/${DS_VERSION}
+fi
 tar -xzf ds_schema_check_test/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz -C ds_schema_check_test/${DS_VERSION} --strip-components 1
 tar -xzf ds_schema_check_test/dev/apache-dolphinscheduler-*-bin.tar.gz -C ds_schema_check_test/dev --strip-components 1
 

.github/workflows/schema-check/postgresql/start-job.sh

@@ -25,7 +25,11 @@ mkdir -p ds_schema_check_test/dev ds_schema_check_test/${DS_VERSION}
 curl -sSf https://atlasgo.sh | sh
 
 # Preparing the environment
-wget https://archive.apache.org/dist/dolphinscheduler/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz -P ds_schema_check_test/${DS_VERSION}
+if [ ! -f /tmp/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz ]; then
+  wget https://archive.apache.org/dist/dolphinscheduler/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz -P ds_schema_check_test/${DS_VERSION}
+else
+  mv /tmp/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz ds_schema_check_test/${DS_VERSION}
+fi
 tar -xzf ds_schema_check_test/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz -C ds_schema_check_test/${DS_VERSION} --strip-components 1
 tar -xzf ds_schema_check_test/dev/apache-dolphinscheduler-*-bin.tar.gz -C ds_schema_check_test/dev --strip-components 1
 

.github/workflows/unit-test.yml

@@ -74,7 +74,7 @@ jobs:
       - uses: actions/cache@v4
         with:
           path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-unit-test
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-backend
           restore-keys: ${{ runner.os }}-maven-
       - name: Run Unit tests
         run: ./mvnw clean verify -B -Dmaven.test.skip=false -Dspotless.skip=true -DskipUT=false