diff --git a/README.md b/README.md index bc7dba684f..93fb7a40b7 100644 --- a/README.md +++ b/README.md @@ -58,25 +58,25 @@ Overload processing: Task queue mechanism, the number of schedulable tasks on a ### Document -- Backend deployment documentation +- Backend deployment documentation -- Front-end deployment documentation +- Front-end deployment documentation -- [**User manual**](https://dolphinscheduler.apache.org/en-us/docs/user_doc/system-manual.html?_blank "System manual") +- [**User manual**](https://dolphinscheduler.apache.org/en-us/docs/1.2.0/user_doc/system-manual.html?_blank "System manual") -- [**Upgrade document**](https://dolphinscheduler.apache.org/en-us/docs/release/upgrade.html?_blank "Upgrade document") +- [**Upgrade document**](https://dolphinscheduler.apache.org/en-us/docs/1.2.0/release/upgrade.html?_blank "Upgrade document") - Online Demo -More documentation please refer to [DolphinScheduler online documentation] +More documentation please refer to [DolphinScheduler online documentation] ### Recent R&D plan Work plan of Dolphin Scheduler: [R&D plan](https://github.com/apache/incubator-dolphinscheduler/projects/1), Under the `In Develop` card is what is currently being developed, TODO card is to be done (including feature ideas) -### How to contribute code +### How to contribute -Welcome to participate in contributing code, please refer to the process of submitting the code: -[[How to contribute code](https://github.com/apache/incubator-dolphinscheduler/issues/310)] +Welcome to participate in contributing, please refer to the process of submitting the code: +[[How to contribute](https://dolphinscheduler.apache.org/en-us/docs/development/contribute.html)] ### How to Build diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/SecurityConfig.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/SecurityConfig.java index 8e1f1ac1f8..823c9bdfba 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/SecurityConfig.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/SecurityConfig.java @@ -16,7 +16,7 @@ */ package org.apache.dolphinscheduler.api.security; -import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java index c4cec68683..e5b9daf6f0 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java @@ -141,7 +141,7 @@ public class MasterServer implements IStoppable { } catch (SchedulerException e1) { logger.error("QuartzExecutors shutdown failed : " + e1.getMessage(), e1); } - logger.error("start Quartz failed : " + e.getMessage(), e); + logger.error("start Quartz failed", e); } @@ -183,7 +183,7 @@ public class MasterServer implements IStoppable { //thread sleep 3 seconds for thread quitely stop Thread.sleep(3000L); }catch (Exception e){ - logger.warn("thread sleep exception:" + e.getMessage(), e); + logger.warn("thread sleep exception ", e); } try { heartbeatMasterService.shutdownNow(); @@ -228,7 +228,7 @@ public class MasterServer implements IStoppable { } catch (Exception e) { - logger.error("master server stop exception : " + e.getMessage(), e); + logger.error("master server stop exception ", e); System.exit(-1); } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java index 5c96757072..c1552c4621 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java @@ -140,7 +140,7 @@ public class MasterBaseTaskExecThread implements Callable { } Thread.sleep(commitRetryInterval); } catch (Exception e) { - logger.error("task commit to mysql and queue failed : " + e.getMessage(),e); + logger.error("task commit to mysql and queue failed",e); } retryTimes += 1; } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java index ade4738884..a91f8c17e6 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java @@ -170,7 +170,7 @@ public class MasterExecThread implements Runnable { executeProcess(); } }catch (Exception e){ - logger.error("master exec thread exception: " + e.getMessage(), e); + logger.error("master exec thread exception", e); logger.error("process execute failed, process id:{}", processInstance.getId()); processInstance.setState(ExecutionStatus.FAILURE); processInstance.setEndTime(new Date()); @@ -355,7 +355,7 @@ public class MasterExecThread implements Runnable { try { FileUtils.deleteDirectory(new File(execLocalPath)); } catch (IOException e) { - logger.error("delete exec dir failed : " + e.getMessage(), e); + logger.error("delete exec dir failed ", e); } } } @@ -733,7 +733,7 @@ public class MasterExecThread implements Runnable { Date endTime = DateUtils.getScheduleDate(cmdParam.get(CMDPARAM_COMPLEMENT_DATA_END_DATE)); return processInstance.getScheduleTime().equals(endTime); } catch (Exception e) { - logger.error("complement end failed : " + e.getMessage(),e); + logger.error("complement end failed ",e); return false; } } @@ -1005,7 +1005,7 @@ public class MasterExecThread implements Runnable { return task; } }catch (Exception e){ - logger.error("get recovery task instance failed : " + e.getMessage(),e); + logger.error("get recovery task instance failed ",e); } return null; } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerThread.java index 5f594b3fa0..1d34bbd147 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerThread.java @@ -134,7 +134,7 @@ public class MasterSchedulerThread implements Runnable { } } }catch (Exception e){ - logger.error("master scheduler thread exception : " + e.getMessage(),e); + logger.error("master scheduler thread exception",e); }finally{ AbstractZKClient.releaseMutex(mutex); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java index 7d10591e0d..1861e8a908 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java @@ -137,7 +137,7 @@ public class MasterTaskExecThread extends MasterBaseTaskExecThread { processInstance = processDao.findProcessInstanceById(processInstance.getId()); Thread.sleep(Constants.SLEEP_TIME_MILLIS); } catch (Exception e) { - logger.error("exception: "+ e.getMessage(),e); + logger.error("exception",e); if (processInstance != null) { logger.error("wait task quit failed, instance id:{}, task id:{}", processInstance.getId(), taskInstance.getId()); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/quartz/DruidConnectionProvider.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/quartz/DruidConnectionProvider.java index 32932127f0..05100ac374 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/quartz/DruidConnectionProvider.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/quartz/DruidConnectionProvider.java @@ -18,9 +18,9 @@ package org.apache.dolphinscheduler.server.quartz; import com.alibaba.druid.pool.DruidDataSource; import org.quartz.SchedulerException; +import org.quartz.utils.ConnectionProvider; import java.sql.Connection; import java.sql.SQLException; -import org.quartz.utils.ConnectionProvider; /** * druid connection provider @@ -137,7 +137,7 @@ public class DruidConnectionProvider implements ConnectionProvider { datasource.setDriverClassName(this.driver); } catch (Exception e) { try { - throw new SchedulerException("Problem setting driver class name on datasource: " + e.getMessage(), e); + throw new SchedulerException("Problem setting driver class name on datasource", e); } catch (SchedulerException e1) { } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/quartz/QuartzExecutors.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/quartz/QuartzExecutors.java index bb0f79897d..d4ad08faec 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/quartz/QuartzExecutors.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/quartz/QuartzExecutors.java @@ -27,7 +27,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.*; -import java.util.Calendar; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -198,7 +197,7 @@ public class QuartzExecutors { } catch (Exception e) { logger.error("add job failed", e); - throw new RuntimeException("add job failed:"+e.getMessage()); + throw new RuntimeException("add job failed", e); } finally { lock.writeLock().unlock(); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/rpc/LogClient.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/rpc/LogClient.java index 3f12ae75dd..1c6c97b88f 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/rpc/LogClient.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/rpc/LogClient.java @@ -102,7 +102,7 @@ public class LogClient { retStrInfo = blockingStub.rollViewLog(pathParameter); return retStrInfo.getMsg(); } catch (StatusRuntimeException e) { - logger.error("roll view log failed : " + e.getMessage(), e); + logger.error("roll view log failed", e); return null; } } @@ -122,7 +122,7 @@ public class LogClient { retStrInfo = blockingStub.viewLog(pathParameter); return retStrInfo.getMsg(); } catch (StatusRuntimeException e) { - logger.error("view log failed : " + e.getMessage(), e); + logger.error("view log failed", e); return null; } } @@ -142,7 +142,7 @@ public class LogClient { retByteInfo = blockingStub.getLogBytes(pathParameter); return retByteInfo.getData().toByteArray(); } catch (StatusRuntimeException e) { - logger.error("get log bytes failed : " + e.getMessage(), e); + logger.error("get log bytes failed ", e); return null; } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java index 7cba139cec..0b621a9bb0 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java @@ -308,7 +308,7 @@ public class ProcessUtils { Runtime.getRuntime().exec(runCmd); } catch (Exception e) { - logger.error("kill application failed : " + e.getMessage(), e); + logger.error("kill application failed", e); } } } @@ -337,7 +337,7 @@ public class ProcessUtils { killYarnJob(taskInstance); } catch (Exception e) { - logger.error("kill failed : " + e.getMessage(), e); + logger.error("kill task failed", e); } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/FetchTaskThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/FetchTaskThread.java index 60cfd82cdd..536fd0b0ca 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/FetchTaskThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/FetchTaskThread.java @@ -186,6 +186,10 @@ public class FetchTaskThread implements Runnable{ continue; } + if(!checkWorkerGroup(taskInstance, OSUtils.getHost())){ + continue; + } + Tenant tenant = processDao.getTenantForProcess(taskInstance.getProcessInstance().getTenantId(), taskInstance.getProcessDefine().getUserId()); @@ -203,11 +207,6 @@ public class FetchTaskThread implements Runnable{ logger.info("worker fetch taskId : {} from queue ", taskInstId); - - if(!checkWorkerGroup(taskInstance, OSUtils.getHost())){ - continue; - } - // local execute path String execLocalPath = getExecLocalPath();