Merge branch 'dev' of https://github.com/apache/incubator-dolphinscheduler into dev

.github/workflows/ci_e2e.yml

@@ -49,7 +49,8 @@ jobs:
       - name: Docker Run
         run: |
           VERSION=`cat $(pwd)/pom.xml| grep "SNAPSHOT</version>" | awk -F "-SNAPSHOT" '{print $1}' | awk -F ">" '{print $2}'`
-          docker run -dit -e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test -p 8888:8888 dolphinscheduler:$VERSION all
+          mkdir -p /tmp/logs
+          docker run -dit -e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test -v /tmp/logs:/opt/dolphinscheduler/logs -p 8888:8888 dolphinscheduler:$VERSION all
       - name: Check Server Status
         run: sh ./dockerfile/hooks/check
       - name: Prepare e2e env
@@ -65,7 +66,10 @@ jobs:
       - name: Run e2e Test
         run: cd ./e2e && mvn -B clean test
       - name: Collect logs
-        run: |
-          mkdir -p ${LOG_DIR}
-          docker logs dolphinscheduler > ${LOG_DIR}/dolphinscheduler.txt
-        continue-on-error: true
+        if: failure()
+        uses: actions/upload-artifact@v1
+        with:
+          name: dslogs
+          path: /tmp/logs
+
+

.github/workflows/ci_ut.yml

@@ -15,7 +15,11 @@
 # limitations under the License.
 #
 
-on: ["pull_request", "push"]
+on:
+  pull_request:
+  push:
+    branches:
+      - dev
 env:
   DOCKER_DIR: ./docker
   LOG_DIR: /tmp/dolphinscheduler
@@ -58,7 +62,6 @@ jobs:
           export MAVEN_OPTS='-Dmaven.repo.local=.m2/repository -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC -XX:-UseGCOverheadLimit -Xmx3g'
           mvn test -B -Dmaven.test.skip=false
       - name: Upload coverage report to codecov
-        if: github.event_name == 'pull_request'
         run: |
           CODECOV_TOKEN="09c2663f-b091-4258-8a47-c981827eb29a" bash <(curl -s https://codecov.io/bash)
       - name: Run SonarCloud Analysis

dockerfile/hooks/check

@@ -16,7 +16,7 @@
 # limitations under the License.
 #
 echo "------ dolphinscheduler check - server - status -------"
-sleep 20
+sleep 60
 server_num=$(docker top `docker container list | grep '/sbin/tini' | awk '{print $1}'`| grep java | grep "dolphinscheduler" | awk -F 'classpath ' '{print $2}' | awk '{print $2}' | sort | uniq -c | wc -l)
 if [ $server_num -eq 5 ]
 then
@@ -25,3 +25,11 @@ else
     echo "Server start failed "$server_num
     exit 1
 fi
+ready=`curl http://127.0.0.1:8888/dolphinscheduler/login -d 'userName=admin&userPassword=dolphinscheduler123' -v | grep "login success" | wc -l`
+if [ $ready -eq 1 ]
+then
+    echo "Servers is ready"
+else
+    echo "Servers is not ready"
+    exit 1
+fi

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java

@@ -425,13 +425,13 @@ public class OSUtils {
    * @return check memory and cpu usage
    */
   public static Boolean checkResource(double systemCpuLoad, double systemReservedMemory){
-    // judging usage
+    // system load average
     double loadAverage = OSUtils.loadAverage();
-    //
+    // system available physical memory
     double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize();
 
     if(loadAverage > systemCpuLoad || availablePhysicalMemorySize < systemReservedMemory){
-      logger.warn("load or availablePhysicalMemorySize(G) is too high, it's availablePhysicalMemorySize(G):{},loadAvg:{}", availablePhysicalMemorySize , loadAverage);
+      logger.warn("load is too high or availablePhysicalMemorySize(G) is too low, it's availablePhysicalMemorySize(G):{},loadAvg:{}", availablePhysicalMemorySize , loadAverage);
       return false;
     }else{
       return true;

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32.java

@@ -32,6 +32,7 @@ import java.util.concurrent.TimeUnit;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import static com.sun.jna.platform.win32.WinBase.INVALID_HANDLE_VALUE;
 import static com.sun.jna.platform.win32.WinBase.STILL_ACTIVE;
 import static java.util.Objects.requireNonNull;
 
@@ -742,7 +743,9 @@ public class ProcessImplForWin32 extends Process {
     }
 
     private static void closeHandle(WinNT.HANDLE handle) {
-        Kernel32Util.closeHandle(handle);
+        if (!handle.equals(INVALID_HANDLE_VALUE)) {
+            Kernel32Util.closeHandle(handle);
+        }
     }
 
     /**

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java

@@ -34,7 +34,7 @@ public class WorkerConfig {
     @Value("${worker.max.cpuload.avg}")
     private int workerMaxCpuloadAvg;
 
-    @Value("${master.reserved.memory}")
+    @Value("${worker.reserved.memory}")
     private double workerReservedMemory;
 
     public int getWorkerExecThreads() {

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java

@@ -363,20 +363,20 @@ public class SqlTask extends AbstractTask {
         // is the timeout set
         boolean timeoutFlag = taskProps.getTaskTimeoutStrategy() == TaskTimeoutStrategy.FAILED ||
                 taskProps.getTaskTimeoutStrategy() == TaskTimeoutStrategy.WARNFAILED;
-        try (PreparedStatement  stmt = connection.prepareStatement(sqlBinds.getSql())) {
-            if(timeoutFlag){
-                stmt.setQueryTimeout(taskProps.getTaskTimeout());
-            }
-            Map<Integer, Property> params = sqlBinds.getParamsMap();
-            if(params != null) {
-                for (Map.Entry<Integer, Property> entry : params.entrySet()) {
-                    Property prop = entry.getValue();
-                    ParameterUtils.setInParameter(entry.getKey(), stmt, prop.getType(), prop.getValue());
-                }
+        // prepare statement
+        PreparedStatement stmt = connection.prepareStatement(sqlBinds.getSql());
+        if(timeoutFlag){
+            stmt.setQueryTimeout(taskProps.getTaskTimeout());
+        }
+        Map<Integer, Property> params = sqlBinds.getParamsMap();
+        if(params != null) {
+            for (Map.Entry<Integer, Property> entry : params.entrySet()) {
+                Property prop = entry.getValue();
+                ParameterUtils.setInParameter(entry.getKey(), stmt, prop.getType(), prop.getValue());
             }
-            logger.info("prepare statement replace sql : {} ", stmt);
-            return stmt;
         }
+        logger.info("prepare statement replace sql : {} ", stmt);
+        return stmt;
     }
 
     /**