|
|
|
/*
|
|
|
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
|
|
|
* contributor license agreements. See the NOTICE file distributed with
|
|
|
|
* this work for additional information regarding copyright ownership.
|
|
|
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
|
|
|
* (the "License"); you may not use this file except in compliance with
|
|
|
|
* the License. You may obtain a copy of the License at
|
|
|
|
*
|
|
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
*
|
|
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
* See the License for the specific language governing permissions and
|
|
|
|
* limitations under the License.
|
|
|
|
*/
|
|
|
|
package org.apache.dolphinscheduler.server.master;
|
|
|
|
|
|
|
|
import org.apache.dolphinscheduler.common.Constants;
|
merge dev-db to dev (#1426)
* [dolphinscheduler-1345] [newfeature] Add DB2 Datasource (#1391)
* Fix the problem that the 'queueId' is not present when creating a tenant based on the default queue. (#1409)
* dolphinscheduler-#1403][bug]improve the check rules (#1408)
1. When check failed, we don’t know whitch parameter has is wrong, Because username、password、email and phone were checks together. I refactored the check method ,Now it will return failed msg by each field.
2. The email check regex support [_|\-|\.]?) in createUser.vue, But it do not support in backend server , I fix it, Now they have the same check regex both in frontend and backend
* jcip-annotations define version information and maven-assembly-plugin add groupId (#1413)
* "v-for" add key (#1419)
* [dolphinscheduler-#1397] [bug]Resources can not be previewed or updated (#1406)
When create an resource the name will add the suffix, But When rename the resource there is no suffix add to the name, So When update resource name without suffix just like "test.sh" => "test" , Then the bug reproduced.
To fix this bug i add the logic bellow:
When rename, if the name without suffix then add it ,else use the origin name
* simply server module configs (#1424)
* move updateTaskState into try/catch block in case of exception
* fix NPE
* using conf.getInt instead of getString
* for AbstractZKClient, remove the log, for it will print the same log message in createZNodePath.
for AlertDao, correct the spelling.
* duplicate
* refactor getTaskWorkerGroupId
* add friendly log
* update hearbeat thread num = 1
* fix the bug when worker execute task using queue. and remove checking Tenant user anymore in TaskScheduleThread
* 1. move verifyTaskInstanceIsNull after taskInstance
2. keep verifyTenantIsNull/verifyTaskInstanceIsNull clean and readable
* fix the message
* delete before check to avoid KeeperException$NoNodeException
* fix the message
* check processInstance state before delete tenant
* check processInstance state before delete worker group
* refactor
* merge api constants into common constatns
* update the resource perm
* update the dataSource perm
* fix CheckUtils.checkUserParams method
* update AlertGroupService, extends from BaseService, remove duplicate methods
* refactor
* modify method name
* add hasProjectAndPerm method
* using checkProject instead of getResultStatus
* delete checkAuth method, using hasProjectAndPerm instead.
* correct spelling
* add transactional for deleteWorkerGroupById
* add Transactional for deleteProcessInstanceById method
* change sqlSessionTemplate singleton
* change sqlSessionTemplate singleton and reformat code
* fix unsuitable error message
* update shutdownhook methods
* fix worker log bug
* fix api server debug mode bug
* upgrade zk version
* delete this line ,for zkClient.close() will do the whole thing
* fix master server shutdown error
* degrade zk version and add FourLetterWordMain class
* fix PathChildrenCache not close
* add Transactional for createSession method
* add more message for java-doc
* delete App, let spring manage connectionFactory
* add license
* add class Application for test support
* refactor masterServer and workerServer
* add args
* fix the spring transaction not work bug
* remove author
* delete @Bean annotation
* delete master/worker properties
* updates
* rename application.properties to application-dao.properties
* delete this class
* delete master/worker properties and refactory master/worker
* delete unused imports
* merge
* delete unused config
5 years ago
|
|
|
import org.apache.dolphinscheduler.common.IStoppable;
|
|
|
|
import org.apache.dolphinscheduler.common.thread.Stopper;
|
|
|
|
import org.apache.dolphinscheduler.common.thread.ThreadPoolExecutors;
|
|
|
|
import org.apache.dolphinscheduler.common.thread.ThreadUtils;
|
|
|
|
import org.apache.dolphinscheduler.common.utils.OSUtils;
|
|
|
|
import org.apache.dolphinscheduler.common.utils.StringUtils;
|
merge dev-db to dev (#1426)
* [dolphinscheduler-1345] [newfeature] Add DB2 Datasource (#1391)
* Fix the problem that the 'queueId' is not present when creating a tenant based on the default queue. (#1409)
* dolphinscheduler-#1403][bug]improve the check rules (#1408)
1. When check failed, we don’t know whitch parameter has is wrong, Because username、password、email and phone were checks together. I refactored the check method ,Now it will return failed msg by each field.
2. The email check regex support [_|\-|\.]?) in createUser.vue, But it do not support in backend server , I fix it, Now they have the same check regex both in frontend and backend
* jcip-annotations define version information and maven-assembly-plugin add groupId (#1413)
* "v-for" add key (#1419)
* [dolphinscheduler-#1397] [bug]Resources can not be previewed or updated (#1406)
When create an resource the name will add the suffix, But When rename the resource there is no suffix add to the name, So When update resource name without suffix just like "test.sh" => "test" , Then the bug reproduced.
To fix this bug i add the logic bellow:
When rename, if the name without suffix then add it ,else use the origin name
* simply server module configs (#1424)
* move updateTaskState into try/catch block in case of exception
* fix NPE
* using conf.getInt instead of getString
* for AbstractZKClient, remove the log, for it will print the same log message in createZNodePath.
for AlertDao, correct the spelling.
* duplicate
* refactor getTaskWorkerGroupId
* add friendly log
* update hearbeat thread num = 1
* fix the bug when worker execute task using queue. and remove checking Tenant user anymore in TaskScheduleThread
* 1. move verifyTaskInstanceIsNull after taskInstance
2. keep verifyTenantIsNull/verifyTaskInstanceIsNull clean and readable
* fix the message
* delete before check to avoid KeeperException$NoNodeException
* fix the message
* check processInstance state before delete tenant
* check processInstance state before delete worker group
* refactor
* merge api constants into common constatns
* update the resource perm
* update the dataSource perm
* fix CheckUtils.checkUserParams method
* update AlertGroupService, extends from BaseService, remove duplicate methods
* refactor
* modify method name
* add hasProjectAndPerm method
* using checkProject instead of getResultStatus
* delete checkAuth method, using hasProjectAndPerm instead.
* correct spelling
* add transactional for deleteWorkerGroupById
* add Transactional for deleteProcessInstanceById method
* change sqlSessionTemplate singleton
* change sqlSessionTemplate singleton and reformat code
* fix unsuitable error message
* update shutdownhook methods
* fix worker log bug
* fix api server debug mode bug
* upgrade zk version
* delete this line ,for zkClient.close() will do the whole thing
* fix master server shutdown error
* degrade zk version and add FourLetterWordMain class
* fix PathChildrenCache not close
* add Transactional for createSession method
* add more message for java-doc
* delete App, let spring manage connectionFactory
* add license
* add class Application for test support
* refactor masterServer and workerServer
* add args
* fix the spring transaction not work bug
* remove author
* delete @Bean annotation
* delete master/worker properties
* updates
* rename application.properties to application-dao.properties
* delete this class
* delete master/worker properties and refactory master/worker
* delete unused imports
* merge
* delete unused config
5 years ago
|
|
|
import org.apache.dolphinscheduler.server.master.config.MasterConfig;
|
|
|
|
import org.apache.dolphinscheduler.server.master.runner.MasterSchedulerThread;
|
|
|
|
import org.apache.dolphinscheduler.server.zk.ZKMasterClient;
|
|
|
|
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
|
|
|
|
import org.apache.dolphinscheduler.service.process.ProcessService;
|
|
|
|
import org.apache.dolphinscheduler.service.quartz.ProcessScheduleJob;
|
|
|
|
import org.apache.dolphinscheduler.service.quartz.QuartzExecutors;
|
|
|
|
import org.quartz.SchedulerException;
|
|
|
|
import org.slf4j.Logger;
|
|
|
|
import org.slf4j.LoggerFactory;
|
|
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
|
|
import org.springframework.boot.WebApplicationType;
|
|
|
|
import org.springframework.boot.builder.SpringApplicationBuilder;
|
|
|
|
import org.springframework.context.annotation.ComponentScan;
|
|
|
|
|
|
|
|
import javax.annotation.PostConstruct;
|
|
|
|
import java.util.concurrent.ExecutorService;
|
|
|
|
import java.util.concurrent.ScheduledExecutorService;
|
|
|
|
import java.util.concurrent.TimeUnit;
|
|
|
|
|
|
|
|
/**
|
|
|
|
* master server
|
|
|
|
*/
|
|
|
|
@ComponentScan("org.apache.dolphinscheduler")
|
merge dev-db to dev (#1426)
* [dolphinscheduler-1345] [newfeature] Add DB2 Datasource (#1391)
* Fix the problem that the 'queueId' is not present when creating a tenant based on the default queue. (#1409)
* dolphinscheduler-#1403][bug]improve the check rules (#1408)
1. When check failed, we don’t know whitch parameter has is wrong, Because username、password、email and phone were checks together. I refactored the check method ,Now it will return failed msg by each field.
2. The email check regex support [_|\-|\.]?) in createUser.vue, But it do not support in backend server , I fix it, Now they have the same check regex both in frontend and backend
* jcip-annotations define version information and maven-assembly-plugin add groupId (#1413)
* "v-for" add key (#1419)
* [dolphinscheduler-#1397] [bug]Resources can not be previewed or updated (#1406)
When create an resource the name will add the suffix, But When rename the resource there is no suffix add to the name, So When update resource name without suffix just like "test.sh" => "test" , Then the bug reproduced.
To fix this bug i add the logic bellow:
When rename, if the name without suffix then add it ,else use the origin name
* simply server module configs (#1424)
* move updateTaskState into try/catch block in case of exception
* fix NPE
* using conf.getInt instead of getString
* for AbstractZKClient, remove the log, for it will print the same log message in createZNodePath.
for AlertDao, correct the spelling.
* duplicate
* refactor getTaskWorkerGroupId
* add friendly log
* update hearbeat thread num = 1
* fix the bug when worker execute task using queue. and remove checking Tenant user anymore in TaskScheduleThread
* 1. move verifyTaskInstanceIsNull after taskInstance
2. keep verifyTenantIsNull/verifyTaskInstanceIsNull clean and readable
* fix the message
* delete before check to avoid KeeperException$NoNodeException
* fix the message
* check processInstance state before delete tenant
* check processInstance state before delete worker group
* refactor
* merge api constants into common constatns
* update the resource perm
* update the dataSource perm
* fix CheckUtils.checkUserParams method
* update AlertGroupService, extends from BaseService, remove duplicate methods
* refactor
* modify method name
* add hasProjectAndPerm method
* using checkProject instead of getResultStatus
* delete checkAuth method, using hasProjectAndPerm instead.
* correct spelling
* add transactional for deleteWorkerGroupById
* add Transactional for deleteProcessInstanceById method
* change sqlSessionTemplate singleton
* change sqlSessionTemplate singleton and reformat code
* fix unsuitable error message
* update shutdownhook methods
* fix worker log bug
* fix api server debug mode bug
* upgrade zk version
* delete this line ,for zkClient.close() will do the whole thing
* fix master server shutdown error
* degrade zk version and add FourLetterWordMain class
* fix PathChildrenCache not close
* add Transactional for createSession method
* add more message for java-doc
* delete App, let spring manage connectionFactory
* add license
* add class Application for test support
* refactor masterServer and workerServer
* add args
* fix the spring transaction not work bug
* remove author
* delete @Bean annotation
* delete master/worker properties
* updates
* rename application.properties to application-dao.properties
* delete this class
* delete master/worker properties and refactory master/worker
* delete unused imports
* merge
* delete unused config
5 years ago
|
|
|
public class MasterServer implements IStoppable {
|
|
|
|
|
|
|
|
/**
|
|
|
|
* logger of MasterServer
|
|
|
|
*/
|
|
|
|
private static final Logger logger = LoggerFactory.getLogger(MasterServer.class);
|
|
|
|
|
|
|
|
/**
|
|
|
|
* zk master client
|
|
|
|
*/
|
|
|
|
@Autowired
|
|
|
|
private ZKMasterClient zkMasterClient = null;
|
|
|
|
|
|
|
|
/**
|
|
|
|
* heartbeat thread pool
|
|
|
|
*/
|
|
|
|
private ScheduledExecutorService heartbeatMasterService;
|
|
|
|
|
|
|
|
/**
|
|
|
|
* process service
|
|
|
|
*/
|
|
|
|
@Autowired
|
|
|
|
protected ProcessService processService;
|
|
|
|
|
|
|
|
/**
|
|
|
|
* master exec thread pool
|
|
|
|
*/
|
|
|
|
private ExecutorService masterSchedulerService;
|
|
|
|
|
merge dev-db to dev (#1426)
* [dolphinscheduler-1345] [newfeature] Add DB2 Datasource (#1391)
* Fix the problem that the 'queueId' is not present when creating a tenant based on the default queue. (#1409)
* dolphinscheduler-#1403][bug]improve the check rules (#1408)
1. When check failed, we don’t know whitch parameter has is wrong, Because username、password、email and phone were checks together. I refactored the check method ,Now it will return failed msg by each field.
2. The email check regex support [_|\-|\.]?) in createUser.vue, But it do not support in backend server , I fix it, Now they have the same check regex both in frontend and backend
* jcip-annotations define version information and maven-assembly-plugin add groupId (#1413)
* "v-for" add key (#1419)
* [dolphinscheduler-#1397] [bug]Resources can not be previewed or updated (#1406)
When create an resource the name will add the suffix, But When rename the resource there is no suffix add to the name, So When update resource name without suffix just like "test.sh" => "test" , Then the bug reproduced.
To fix this bug i add the logic bellow:
When rename, if the name without suffix then add it ,else use the origin name
* simply server module configs (#1424)
* move updateTaskState into try/catch block in case of exception
* fix NPE
* using conf.getInt instead of getString
* for AbstractZKClient, remove the log, for it will print the same log message in createZNodePath.
for AlertDao, correct the spelling.
* duplicate
* refactor getTaskWorkerGroupId
* add friendly log
* update hearbeat thread num = 1
* fix the bug when worker execute task using queue. and remove checking Tenant user anymore in TaskScheduleThread
* 1. move verifyTaskInstanceIsNull after taskInstance
2. keep verifyTenantIsNull/verifyTaskInstanceIsNull clean and readable
* fix the message
* delete before check to avoid KeeperException$NoNodeException
* fix the message
* check processInstance state before delete tenant
* check processInstance state before delete worker group
* refactor
* merge api constants into common constatns
* update the resource perm
* update the dataSource perm
* fix CheckUtils.checkUserParams method
* update AlertGroupService, extends from BaseService, remove duplicate methods
* refactor
* modify method name
* add hasProjectAndPerm method
* using checkProject instead of getResultStatus
* delete checkAuth method, using hasProjectAndPerm instead.
* correct spelling
* add transactional for deleteWorkerGroupById
* add Transactional for deleteProcessInstanceById method
* change sqlSessionTemplate singleton
* change sqlSessionTemplate singleton and reformat code
* fix unsuitable error message
* update shutdownhook methods
* fix worker log bug
* fix api server debug mode bug
* upgrade zk version
* delete this line ,for zkClient.close() will do the whole thing
* fix master server shutdown error
* degrade zk version and add FourLetterWordMain class
* fix PathChildrenCache not close
* add Transactional for createSession method
* add more message for java-doc
* delete App, let spring manage connectionFactory
* add license
* add class Application for test support
* refactor masterServer and workerServer
* add args
* fix the spring transaction not work bug
* remove author
* delete @Bean annotation
* delete master/worker properties
* updates
* rename application.properties to application-dao.properties
* delete this class
* delete master/worker properties and refactory master/worker
* delete unused imports
* merge
* delete unused config
5 years ago
|
|
|
/**
|
|
|
|
* master config
|
merge dev-db to dev (#1426)
* [dolphinscheduler-1345] [newfeature] Add DB2 Datasource (#1391)
* Fix the problem that the 'queueId' is not present when creating a tenant based on the default queue. (#1409)
* dolphinscheduler-#1403][bug]improve the check rules (#1408)
1. When check failed, we don’t know whitch parameter has is wrong, Because username、password、email and phone were checks together. I refactored the check method ,Now it will return failed msg by each field.
2. The email check regex support [_|\-|\.]?) in createUser.vue, But it do not support in backend server , I fix it, Now they have the same check regex both in frontend and backend
* jcip-annotations define version information and maven-assembly-plugin add groupId (#1413)
* "v-for" add key (#1419)
* [dolphinscheduler-#1397] [bug]Resources can not be previewed or updated (#1406)
When create an resource the name will add the suffix, But When rename the resource there is no suffix add to the name, So When update resource name without suffix just like "test.sh" => "test" , Then the bug reproduced.
To fix this bug i add the logic bellow:
When rename, if the name without suffix then add it ,else use the origin name
* simply server module configs (#1424)
* move updateTaskState into try/catch block in case of exception
* fix NPE
* using conf.getInt instead of getString
* for AbstractZKClient, remove the log, for it will print the same log message in createZNodePath.
for AlertDao, correct the spelling.
* duplicate
* refactor getTaskWorkerGroupId
* add friendly log
* update hearbeat thread num = 1
* fix the bug when worker execute task using queue. and remove checking Tenant user anymore in TaskScheduleThread
* 1. move verifyTaskInstanceIsNull after taskInstance
2. keep verifyTenantIsNull/verifyTaskInstanceIsNull clean and readable
* fix the message
* delete before check to avoid KeeperException$NoNodeException
* fix the message
* check processInstance state before delete tenant
* check processInstance state before delete worker group
* refactor
* merge api constants into common constatns
* update the resource perm
* update the dataSource perm
* fix CheckUtils.checkUserParams method
* update AlertGroupService, extends from BaseService, remove duplicate methods
* refactor
* modify method name
* add hasProjectAndPerm method
* using checkProject instead of getResultStatus
* delete checkAuth method, using hasProjectAndPerm instead.
* correct spelling
* add transactional for deleteWorkerGroupById
* add Transactional for deleteProcessInstanceById method
* change sqlSessionTemplate singleton
* change sqlSessionTemplate singleton and reformat code
* fix unsuitable error message
* update shutdownhook methods
* fix worker log bug
* fix api server debug mode bug
* upgrade zk version
* delete this line ,for zkClient.close() will do the whole thing
* fix master server shutdown error
* degrade zk version and add FourLetterWordMain class
* fix PathChildrenCache not close
* add Transactional for createSession method
* add more message for java-doc
* delete App, let spring manage connectionFactory
* add license
* add class Application for test support
* refactor masterServer and workerServer
* add args
* fix the spring transaction not work bug
* remove author
* delete @Bean annotation
* delete master/worker properties
* updates
* rename application.properties to application-dao.properties
* delete this class
* delete master/worker properties and refactory master/worker
* delete unused imports
* merge
* delete unused config
5 years ago
|
|
|
*/
|
|
|
|
@Autowired
|
|
|
|
private MasterConfig masterConfig;
|
|
|
|
|
merge dev-db to dev (#1426)
* [dolphinscheduler-1345] [newfeature] Add DB2 Datasource (#1391)
* Fix the problem that the 'queueId' is not present when creating a tenant based on the default queue. (#1409)
* dolphinscheduler-#1403][bug]improve the check rules (#1408)
1. When check failed, we don’t know whitch parameter has is wrong, Because username、password、email and phone were checks together. I refactored the check method ,Now it will return failed msg by each field.
2. The email check regex support [_|\-|\.]?) in createUser.vue, But it do not support in backend server , I fix it, Now they have the same check regex both in frontend and backend
* jcip-annotations define version information and maven-assembly-plugin add groupId (#1413)
* "v-for" add key (#1419)
* [dolphinscheduler-#1397] [bug]Resources can not be previewed or updated (#1406)
When create an resource the name will add the suffix, But When rename the resource there is no suffix add to the name, So When update resource name without suffix just like "test.sh" => "test" , Then the bug reproduced.
To fix this bug i add the logic bellow:
When rename, if the name without suffix then add it ,else use the origin name
* simply server module configs (#1424)
* move updateTaskState into try/catch block in case of exception
* fix NPE
* using conf.getInt instead of getString
* for AbstractZKClient, remove the log, for it will print the same log message in createZNodePath.
for AlertDao, correct the spelling.
* duplicate
* refactor getTaskWorkerGroupId
* add friendly log
* update hearbeat thread num = 1
* fix the bug when worker execute task using queue. and remove checking Tenant user anymore in TaskScheduleThread
* 1. move verifyTaskInstanceIsNull after taskInstance
2. keep verifyTenantIsNull/verifyTaskInstanceIsNull clean and readable
* fix the message
* delete before check to avoid KeeperException$NoNodeException
* fix the message
* check processInstance state before delete tenant
* check processInstance state before delete worker group
* refactor
* merge api constants into common constatns
* update the resource perm
* update the dataSource perm
* fix CheckUtils.checkUserParams method
* update AlertGroupService, extends from BaseService, remove duplicate methods
* refactor
* modify method name
* add hasProjectAndPerm method
* using checkProject instead of getResultStatus
* delete checkAuth method, using hasProjectAndPerm instead.
* correct spelling
* add transactional for deleteWorkerGroupById
* add Transactional for deleteProcessInstanceById method
* change sqlSessionTemplate singleton
* change sqlSessionTemplate singleton and reformat code
* fix unsuitable error message
* update shutdownhook methods
* fix worker log bug
* fix api server debug mode bug
* upgrade zk version
* delete this line ,for zkClient.close() will do the whole thing
* fix master server shutdown error
* degrade zk version and add FourLetterWordMain class
* fix PathChildrenCache not close
* add Transactional for createSession method
* add more message for java-doc
* delete App, let spring manage connectionFactory
* add license
* add class Application for test support
* refactor masterServer and workerServer
* add args
* fix the spring transaction not work bug
* remove author
* delete @Bean annotation
* delete master/worker properties
* updates
* rename application.properties to application-dao.properties
* delete this class
* delete master/worker properties and refactory master/worker
* delete unused imports
* merge
* delete unused config
5 years ago
|
|
|
|
|
|
|
/**
|
|
|
|
* spring application context
|
|
|
|
* only use it for initialization
|
merge dev-db to dev (#1426)
* [dolphinscheduler-1345] [newfeature] Add DB2 Datasource (#1391)
* Fix the problem that the 'queueId' is not present when creating a tenant based on the default queue. (#1409)
* dolphinscheduler-#1403][bug]improve the check rules (#1408)
1. When check failed, we don’t know whitch parameter has is wrong, Because username、password、email and phone were checks together. I refactored the check method ,Now it will return failed msg by each field.
2. The email check regex support [_|\-|\.]?) in createUser.vue, But it do not support in backend server , I fix it, Now they have the same check regex both in frontend and backend
* jcip-annotations define version information and maven-assembly-plugin add groupId (#1413)
* "v-for" add key (#1419)
* [dolphinscheduler-#1397] [bug]Resources can not be previewed or updated (#1406)
When create an resource the name will add the suffix, But When rename the resource there is no suffix add to the name, So When update resource name without suffix just like "test.sh" => "test" , Then the bug reproduced.
To fix this bug i add the logic bellow:
When rename, if the name without suffix then add it ,else use the origin name
* simply server module configs (#1424)
* move updateTaskState into try/catch block in case of exception
* fix NPE
* using conf.getInt instead of getString
* for AbstractZKClient, remove the log, for it will print the same log message in createZNodePath.
for AlertDao, correct the spelling.
* duplicate
* refactor getTaskWorkerGroupId
* add friendly log
* update hearbeat thread num = 1
* fix the bug when worker execute task using queue. and remove checking Tenant user anymore in TaskScheduleThread
* 1. move verifyTaskInstanceIsNull after taskInstance
2. keep verifyTenantIsNull/verifyTaskInstanceIsNull clean and readable
* fix the message
* delete before check to avoid KeeperException$NoNodeException
* fix the message
* check processInstance state before delete tenant
* check processInstance state before delete worker group
* refactor
* merge api constants into common constatns
* update the resource perm
* update the dataSource perm
* fix CheckUtils.checkUserParams method
* update AlertGroupService, extends from BaseService, remove duplicate methods
* refactor
* modify method name
* add hasProjectAndPerm method
* using checkProject instead of getResultStatus
* delete checkAuth method, using hasProjectAndPerm instead.
* correct spelling
* add transactional for deleteWorkerGroupById
* add Transactional for deleteProcessInstanceById method
* change sqlSessionTemplate singleton
* change sqlSessionTemplate singleton and reformat code
* fix unsuitable error message
* update shutdownhook methods
* fix worker log bug
* fix api server debug mode bug
* upgrade zk version
* delete this line ,for zkClient.close() will do the whole thing
* fix master server shutdown error
* degrade zk version and add FourLetterWordMain class
* fix PathChildrenCache not close
* add Transactional for createSession method
* add more message for java-doc
* delete App, let spring manage connectionFactory
* add license
* add class Application for test support
* refactor masterServer and workerServer
* add args
* fix the spring transaction not work bug
* remove author
* delete @Bean annotation
* delete master/worker properties
* updates
* rename application.properties to application-dao.properties
* delete this class
* delete master/worker properties and refactory master/worker
* delete unused imports
* merge
* delete unused config
5 years ago
|
|
|
*/
|
|
|
|
@Autowired
|
|
|
|
private SpringApplicationContext springApplicationContext;
|
merge dev-db to dev (#1426)
* [dolphinscheduler-1345] [newfeature] Add DB2 Datasource (#1391)
* Fix the problem that the 'queueId' is not present when creating a tenant based on the default queue. (#1409)
* dolphinscheduler-#1403][bug]improve the check rules (#1408)
1. When check failed, we don’t know whitch parameter has is wrong, Because username、password、email and phone were checks together. I refactored the check method ,Now it will return failed msg by each field.
2. The email check regex support [_|\-|\.]?) in createUser.vue, But it do not support in backend server , I fix it, Now they have the same check regex both in frontend and backend
* jcip-annotations define version information and maven-assembly-plugin add groupId (#1413)
* "v-for" add key (#1419)
* [dolphinscheduler-#1397] [bug]Resources can not be previewed or updated (#1406)
When create an resource the name will add the suffix, But When rename the resource there is no suffix add to the name, So When update resource name without suffix just like "test.sh" => "test" , Then the bug reproduced.
To fix this bug i add the logic bellow:
When rename, if the name without suffix then add it ,else use the origin name
* simply server module configs (#1424)
* move updateTaskState into try/catch block in case of exception
* fix NPE
* using conf.getInt instead of getString
* for AbstractZKClient, remove the log, for it will print the same log message in createZNodePath.
for AlertDao, correct the spelling.
* duplicate
* refactor getTaskWorkerGroupId
* add friendly log
* update hearbeat thread num = 1
* fix the bug when worker execute task using queue. and remove checking Tenant user anymore in TaskScheduleThread
* 1. move verifyTaskInstanceIsNull after taskInstance
2. keep verifyTenantIsNull/verifyTaskInstanceIsNull clean and readable
* fix the message
* delete before check to avoid KeeperException$NoNodeException
* fix the message
* check processInstance state before delete tenant
* check processInstance state before delete worker group
* refactor
* merge api constants into common constatns
* update the resource perm
* update the dataSource perm
* fix CheckUtils.checkUserParams method
* update AlertGroupService, extends from BaseService, remove duplicate methods
* refactor
* modify method name
* add hasProjectAndPerm method
* using checkProject instead of getResultStatus
* delete checkAuth method, using hasProjectAndPerm instead.
* correct spelling
* add transactional for deleteWorkerGroupById
* add Transactional for deleteProcessInstanceById method
* change sqlSessionTemplate singleton
* change sqlSessionTemplate singleton and reformat code
* fix unsuitable error message
* update shutdownhook methods
* fix worker log bug
* fix api server debug mode bug
* upgrade zk version
* delete this line ,for zkClient.close() will do the whole thing
* fix master server shutdown error
* degrade zk version and add FourLetterWordMain class
* fix PathChildrenCache not close
* add Transactional for createSession method
* add more message for java-doc
* delete App, let spring manage connectionFactory
* add license
* add class Application for test support
* refactor masterServer and workerServer
* add args
* fix the spring transaction not work bug
* remove author
* delete @Bean annotation
* delete master/worker properties
* updates
* rename application.properties to application-dao.properties
* delete this class
* delete master/worker properties and refactory master/worker
* delete unused imports
* merge
* delete unused config
5 years ago
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
* master server startup
|
|
|
|
*
|
|
|
|
* master server not use web service
|
|
|
|
* @param args arguments
|
|
|
|
*/
|
|
|
|
public static void main(String[] args) {
|
|
|
|
Thread.currentThread().setName(Constants.THREAD_NAME_MASTER_SERVER);
|
|
|
|
new SpringApplicationBuilder(MasterServer.class).web(WebApplicationType.NONE).run(args);
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* run master server
|
|
|
|
*/
|
|
|
|
@PostConstruct
|
|
|
|
public void run(){
|
|
|
|
zkMasterClient.init();
|
|
|
|
|
|
|
|
masterSchedulerService = ThreadUtils.newDaemonSingleThreadExecutor("Master-Scheduler-Thread");
|
|
|
|
|
|
|
|
heartbeatMasterService = ThreadUtils.newDaemonThreadScheduledExecutor("Master-Main-Thread",Constants.DEFAULT_MASTER_HEARTBEAT_THREAD_NUM);
|
|
|
|
|
|
|
|
// heartbeat thread implement
|
|
|
|
Runnable heartBeatThread = heartBeatThread();
|
|
|
|
|
|
|
|
zkMasterClient.setStoppable(this);
|
|
|
|
|
|
|
|
// regular heartbeat
|
|
|
|
// delay 5 seconds, send heartbeat every 30 seconds
|
|
|
|
heartbeatMasterService.
|
merge dev-db to dev (#1426)
* [dolphinscheduler-1345] [newfeature] Add DB2 Datasource (#1391)
* Fix the problem that the 'queueId' is not present when creating a tenant based on the default queue. (#1409)
* dolphinscheduler-#1403][bug]improve the check rules (#1408)
1. When check failed, we don’t know whitch parameter has is wrong, Because username、password、email and phone were checks together. I refactored the check method ,Now it will return failed msg by each field.
2. The email check regex support [_|\-|\.]?) in createUser.vue, But it do not support in backend server , I fix it, Now they have the same check regex both in frontend and backend
* jcip-annotations define version information and maven-assembly-plugin add groupId (#1413)
* "v-for" add key (#1419)
* [dolphinscheduler-#1397] [bug]Resources can not be previewed or updated (#1406)
When create an resource the name will add the suffix, But When rename the resource there is no suffix add to the name, So When update resource name without suffix just like "test.sh" => "test" , Then the bug reproduced.
To fix this bug i add the logic bellow:
When rename, if the name without suffix then add it ,else use the origin name
* simply server module configs (#1424)
* move updateTaskState into try/catch block in case of exception
* fix NPE
* using conf.getInt instead of getString
* for AbstractZKClient, remove the log, for it will print the same log message in createZNodePath.
for AlertDao, correct the spelling.
* duplicate
* refactor getTaskWorkerGroupId
* add friendly log
* update hearbeat thread num = 1
* fix the bug when worker execute task using queue. and remove checking Tenant user anymore in TaskScheduleThread
* 1. move verifyTaskInstanceIsNull after taskInstance
2. keep verifyTenantIsNull/verifyTaskInstanceIsNull clean and readable
* fix the message
* delete before check to avoid KeeperException$NoNodeException
* fix the message
* check processInstance state before delete tenant
* check processInstance state before delete worker group
* refactor
* merge api constants into common constatns
* update the resource perm
* update the dataSource perm
* fix CheckUtils.checkUserParams method
* update AlertGroupService, extends from BaseService, remove duplicate methods
* refactor
* modify method name
* add hasProjectAndPerm method
* using checkProject instead of getResultStatus
* delete checkAuth method, using hasProjectAndPerm instead.
* correct spelling
* add transactional for deleteWorkerGroupById
* add Transactional for deleteProcessInstanceById method
* change sqlSessionTemplate singleton
* change sqlSessionTemplate singleton and reformat code
* fix unsuitable error message
* update shutdownhook methods
* fix worker log bug
* fix api server debug mode bug
* upgrade zk version
* delete this line ,for zkClient.close() will do the whole thing
* fix master server shutdown error
* degrade zk version and add FourLetterWordMain class
* fix PathChildrenCache not close
* add Transactional for createSession method
* add more message for java-doc
* delete App, let spring manage connectionFactory
* add license
* add class Application for test support
* refactor masterServer and workerServer
* add args
* fix the spring transaction not work bug
* remove author
* delete @Bean annotation
* delete master/worker properties
* updates
* rename application.properties to application-dao.properties
* delete this class
* delete master/worker properties and refactory master/worker
* delete unused imports
* merge
* delete unused config
5 years ago
|
|
|
scheduleAtFixedRate(heartBeatThread, 5, masterConfig.getMasterHeartbeatInterval(), TimeUnit.SECONDS);
|
|
|
|
|
|
|
|
// master scheduler thread
|
|
|
|
MasterSchedulerThread masterSchedulerThread = new MasterSchedulerThread(
|
|
|
|
zkMasterClient,
|
|
|
|
processService,
|
merge dev-db to dev (#1426)
* [dolphinscheduler-1345] [newfeature] Add DB2 Datasource (#1391)
* Fix the problem that the 'queueId' is not present when creating a tenant based on the default queue. (#1409)
* dolphinscheduler-#1403][bug]improve the check rules (#1408)
1. When check failed, we don’t know whitch parameter has is wrong, Because username、password、email and phone were checks together. I refactored the check method ,Now it will return failed msg by each field.
2. The email check regex support [_|\-|\.]?) in createUser.vue, But it do not support in backend server , I fix it, Now they have the same check regex both in frontend and backend
* jcip-annotations define version information and maven-assembly-plugin add groupId (#1413)
* "v-for" add key (#1419)
* [dolphinscheduler-#1397] [bug]Resources can not be previewed or updated (#1406)
When create an resource the name will add the suffix, But When rename the resource there is no suffix add to the name, So When update resource name without suffix just like "test.sh" => "test" , Then the bug reproduced.
To fix this bug i add the logic bellow:
When rename, if the name without suffix then add it ,else use the origin name
* simply server module configs (#1424)
* move updateTaskState into try/catch block in case of exception
* fix NPE
* using conf.getInt instead of getString
* for AbstractZKClient, remove the log, for it will print the same log message in createZNodePath.
for AlertDao, correct the spelling.
* duplicate
* refactor getTaskWorkerGroupId
* add friendly log
* update hearbeat thread num = 1
* fix the bug when worker execute task using queue. and remove checking Tenant user anymore in TaskScheduleThread
* 1. move verifyTaskInstanceIsNull after taskInstance
2. keep verifyTenantIsNull/verifyTaskInstanceIsNull clean and readable
* fix the message
* delete before check to avoid KeeperException$NoNodeException
* fix the message
* check processInstance state before delete tenant
* check processInstance state before delete worker group
* refactor
* merge api constants into common constatns
* update the resource perm
* update the dataSource perm
* fix CheckUtils.checkUserParams method
* update AlertGroupService, extends from BaseService, remove duplicate methods
* refactor
* modify method name
* add hasProjectAndPerm method
* using checkProject instead of getResultStatus
* delete checkAuth method, using hasProjectAndPerm instead.
* correct spelling
* add transactional for deleteWorkerGroupById
* add Transactional for deleteProcessInstanceById method
* change sqlSessionTemplate singleton
* change sqlSessionTemplate singleton and reformat code
* fix unsuitable error message
* update shutdownhook methods
* fix worker log bug
* fix api server debug mode bug
* upgrade zk version
* delete this line ,for zkClient.close() will do the whole thing
* fix master server shutdown error
* degrade zk version and add FourLetterWordMain class
* fix PathChildrenCache not close
* add Transactional for createSession method
* add more message for java-doc
* delete App, let spring manage connectionFactory
* add license
* add class Application for test support
* refactor masterServer and workerServer
* add args
* fix the spring transaction not work bug
* remove author
* delete @Bean annotation
* delete master/worker properties
* updates
* rename application.properties to application-dao.properties
* delete this class
* delete master/worker properties and refactory master/worker
* delete unused imports
* merge
* delete unused config
5 years ago
|
|
|
masterConfig.getMasterExecThreads());
|
|
|
|
|
|
|
|
// submit master scheduler thread
|
|
|
|
masterSchedulerService.execute(masterSchedulerThread);
|
|
|
|
|
|
|
|
// start QuartzExecutors
|
|
|
|
// what system should do if exception
|
|
|
|
try {
|
|
|
|
logger.info("start Quartz server...");
|
|
|
|
ProcessScheduleJob.init(processService);
|
|
|
|
QuartzExecutors.getInstance().start();
|
|
|
|
} catch (Exception e) {
|
|
|
|
try {
|
|
|
|
QuartzExecutors.getInstance().shutdown();
|
|
|
|
} catch (SchedulerException e1) {
|
|
|
|
logger.error("QuartzExecutors shutdown failed : " + e1.getMessage(), e1);
|
|
|
|
}
|
|
|
|
logger.error("start Quartz failed", e);
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
* register hooks, which are called before the process exits
|
|
|
|
*/
|
|
|
|
Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
|
|
|
|
@Override
|
|
|
|
public void run() {
|
|
|
|
if (zkMasterClient.getActiveMasterNum() <= 1) {
|
|
|
|
zkMasterClient.getAlertDao().sendServerStopedAlert(
|
|
|
|
1, OSUtils.getHost(), "Master-Server");
|
|
|
|
}
|
|
|
|
stop("shutdownhook");
|
|
|
|
}
|
|
|
|
}));
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
* gracefully stop
|
|
|
|
* @param cause why stopping
|
|
|
|
*/
|
|
|
|
@Override
|
|
|
|
public synchronized void stop(String cause) {
|
|
|
|
|
|
|
|
try {
|
|
|
|
//execute only once
|
|
|
|
if(Stopper.isStopped()){
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
logger.info("master server is stopping ..., cause : {}", cause);
|
|
|
|
|
|
|
|
// set stop signal is true
|
|
|
|
Stopper.stop();
|
|
|
|
|
|
|
|
try {
|
|
|
|
//thread sleep 3 seconds for thread quitely stop
|
|
|
|
Thread.sleep(3000L);
|
|
|
|
}catch (Exception e){
|
|
|
|
logger.warn("thread sleep exception ", e);
|
|
|
|
}
|
|
|
|
try {
|
|
|
|
heartbeatMasterService.shutdownNow();
|
|
|
|
}catch (Exception e){
|
|
|
|
logger.warn("heartbeat service stopped exception");
|
|
|
|
}
|
|
|
|
|
|
|
|
logger.info("heartbeat service stopped");
|
|
|
|
|
|
|
|
//close quartz
|
|
|
|
try{
|
|
|
|
QuartzExecutors.getInstance().shutdown();
|
|
|
|
}catch (Exception e){
|
|
|
|
logger.warn("Quartz service stopped exception:{}",e.getMessage());
|
|
|
|
}
|
|
|
|
|
|
|
|
logger.info("Quartz service stopped");
|
|
|
|
|
|
|
|
try {
|
|
|
|
ThreadPoolExecutors.getInstance().shutdown();
|
|
|
|
}catch (Exception e){
|
|
|
|
logger.warn("threadpool service stopped exception:{}",e.getMessage());
|
|
|
|
}
|
|
|
|
|
|
|
|
logger.info("threadpool service stopped");
|
|
|
|
|
|
|
|
try {
|
|
|
|
masterSchedulerService.shutdownNow();
|
|
|
|
}catch (Exception e){
|
|
|
|
logger.warn("master scheduler service stopped exception:{}",e.getMessage());
|
|
|
|
}
|
|
|
|
|
|
|
|
logger.info("master scheduler service stopped");
|
|
|
|
|
|
|
|
try {
|
|
|
|
zkMasterClient.close();
|
|
|
|
}catch (Exception e){
|
|
|
|
logger.warn("zookeeper service stopped exception:{}",e.getMessage());
|
|
|
|
}
|
|
|
|
|
|
|
|
logger.info("zookeeper service stopped");
|
|
|
|
|
|
|
|
|
|
|
|
} catch (Exception e) {
|
|
|
|
logger.error("master server stop exception ", e);
|
|
|
|
System.exit(-1);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
* heartbeat thread implement
|
|
|
|
* @return
|
|
|
|
*/
|
|
|
|
private Runnable heartBeatThread(){
|
|
|
|
logger.info("start master heart beat thread...");
|
|
|
|
Runnable heartBeatThread = new Runnable() {
|
|
|
|
@Override
|
|
|
|
public void run() {
|
|
|
|
if(Stopper.isRunning()) {
|
|
|
|
// send heartbeat to zk
|
|
|
|
if (StringUtils.isBlank(zkMasterClient.getMasterZNode())) {
|
|
|
|
logger.error("master send heartbeat to zk failed: can't find zookeeper path of master server");
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
zkMasterClient.heartBeatForZk(zkMasterClient.getMasterZNode(), Constants.MASTER_PREFIX);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
};
|
|
|
|
return heartBeatThread;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|