diff --git a/.github/workflows/ci_e2e.yml b/.github/workflows/ci_e2e.yml index fe818d0668..7ec204e145 100644 --- a/.github/workflows/ci_e2e.yml +++ b/.github/workflows/ci_e2e.yml @@ -49,7 +49,8 @@ jobs: - name: Docker Run run: | VERSION=`cat $(pwd)/pom.xml| grep "SNAPSHOT" | awk -F "-SNAPSHOT" '{print $1}' | awk -F ">" '{print $2}'` - docker run -dit -e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test -p 8888:8888 dolphinscheduler:$VERSION all + mkdir -p /tmp/logs + docker run -dit -e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test -v /tmp/logs:/opt/dolphinscheduler/logs -p 8888:8888 dolphinscheduler:$VERSION all - name: Check Server Status run: sh ./dockerfile/hooks/check - name: Prepare e2e env @@ -65,7 +66,10 @@ jobs: - name: Run e2e Test run: cd ./e2e && mvn -B clean test - name: Collect logs - run: | - mkdir -p ${LOG_DIR} - docker logs dolphinscheduler > ${LOG_DIR}/dolphinscheduler.txt - continue-on-error: true + if: failure() + uses: actions/upload-artifact@v1 + with: + name: dslogs + path: /tmp/logs + + diff --git a/.github/workflows/ci_ut.yml b/.github/workflows/ci_ut.yml index 95dfed8119..8d75bae163 100644 --- a/.github/workflows/ci_ut.yml +++ b/.github/workflows/ci_ut.yml @@ -15,7 +15,11 @@ # limitations under the License. # -on: ["pull_request", "push"] +on: + pull_request: + push: + branches: + - dev env: DOCKER_DIR: ./docker LOG_DIR: /tmp/dolphinscheduler @@ -58,7 +62,6 @@ jobs: export MAVEN_OPTS='-Dmaven.repo.local=.m2/repository -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC -XX:-UseGCOverheadLimit -Xmx3g' mvn test -B -Dmaven.test.skip=false - name: Upload coverage report to codecov - if: github.event_name == 'pull_request' run: | CODECOV_TOKEN="09c2663f-b091-4258-8a47-c981827eb29a" bash <(curl -s https://codecov.io/bash) - name: Run SonarCloud Analysis diff --git a/dockerfile/hooks/check b/dockerfile/hooks/check index 21f3f4db12..fdb1902311 100644 --- a/dockerfile/hooks/check +++ b/dockerfile/hooks/check @@ -16,7 +16,7 @@ # limitations under the License. # echo "------ dolphinscheduler check - server - status -------" -sleep 20 +sleep 60 server_num=$(docker top `docker container list | grep '/sbin/tini' | awk '{print $1}'`| grep java | grep "dolphinscheduler" | awk -F 'classpath ' '{print $2}' | awk '{print $2}' | sort | uniq -c | wc -l) if [ $server_num -eq 5 ] then @@ -25,3 +25,11 @@ else echo "Server start failed "$server_num exit 1 fi +ready=`curl http://127.0.0.1:8888/dolphinscheduler/login -d 'userName=admin&userPassword=dolphinscheduler123' -v | grep "login success" | wc -l` +if [ $ready -eq 1 ] +then + echo "Servers is ready" +else + echo "Servers is not ready" + exit 1 +fi diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java index b011c0bc4e..a0fea8d33c 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java @@ -425,13 +425,13 @@ public class OSUtils { * @return check memory and cpu usage */ public static Boolean checkResource(double systemCpuLoad, double systemReservedMemory){ - // judging usage + // system load average double loadAverage = OSUtils.loadAverage(); - // + // system available physical memory double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize(); if(loadAverage > systemCpuLoad || availablePhysicalMemorySize < systemReservedMemory){ - logger.warn("load or availablePhysicalMemorySize(G) is too high, it's availablePhysicalMemorySize(G):{},loadAvg:{}", availablePhysicalMemorySize , loadAverage); + logger.warn("load is too high or availablePhysicalMemorySize(G) is too low, it's availablePhysicalMemorySize(G):{},loadAvg:{}", availablePhysicalMemorySize , loadAverage); return false; }else{ return true; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32.java index 9f2716a096..bef5bb0ae2 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32.java @@ -32,6 +32,7 @@ import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; +import static com.sun.jna.platform.win32.WinBase.INVALID_HANDLE_VALUE; import static com.sun.jna.platform.win32.WinBase.STILL_ACTIVE; import static java.util.Objects.requireNonNull; @@ -742,7 +743,9 @@ public class ProcessImplForWin32 extends Process { } private static void closeHandle(WinNT.HANDLE handle) { - Kernel32Util.closeHandle(handle); + if (!handle.equals(INVALID_HANDLE_VALUE)) { + Kernel32Util.closeHandle(handle); + } } /** diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java index c4d4b61af5..aaaf5c7805 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java @@ -34,7 +34,7 @@ public class WorkerConfig { @Value("${worker.max.cpuload.avg}") private int workerMaxCpuloadAvg; - @Value("${master.reserved.memory}") + @Value("${worker.reserved.memory}") private double workerReservedMemory; public int getWorkerExecThreads() { diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java index 71ab56b528..f7070b1d38 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java @@ -363,20 +363,20 @@ public class SqlTask extends AbstractTask { // is the timeout set boolean timeoutFlag = taskProps.getTaskTimeoutStrategy() == TaskTimeoutStrategy.FAILED || taskProps.getTaskTimeoutStrategy() == TaskTimeoutStrategy.WARNFAILED; - try (PreparedStatement stmt = connection.prepareStatement(sqlBinds.getSql())) { - if(timeoutFlag){ - stmt.setQueryTimeout(taskProps.getTaskTimeout()); - } - Map params = sqlBinds.getParamsMap(); - if(params != null) { - for (Map.Entry entry : params.entrySet()) { - Property prop = entry.getValue(); - ParameterUtils.setInParameter(entry.getKey(), stmt, prop.getType(), prop.getValue()); - } + // prepare statement + PreparedStatement stmt = connection.prepareStatement(sqlBinds.getSql()); + if(timeoutFlag){ + stmt.setQueryTimeout(taskProps.getTaskTimeout()); + } + Map params = sqlBinds.getParamsMap(); + if(params != null) { + for (Map.Entry entry : params.entrySet()) { + Property prop = entry.getValue(); + ParameterUtils.setInParameter(entry.getKey(), stmt, prop.getType(), prop.getValue()); } - logger.info("prepare statement replace sql : {} ", stmt); - return stmt; } + logger.info("prepare statement replace sql : {} ", stmt); + return stmt; } /**