diff --git a/.gitignore b/.gitignore index 7a99e2e4b0..7cf1d4d7db 100644 --- a/.gitignore +++ b/.gitignore @@ -4,25 +4,27 @@ .zip .gz .DS_Store +.idea .idea/ -dist/ -all-dependencies.txt -self-modules.txt -third-party-dependencies.txt -**/target/ +.idea/* +.target +.target/ +**/**/target/** +target/* +*/target +*/target/* .settings .nbproject .classpath .project -**/*.iml +*.iml *.ipr *.iws *.tgz .*.swp -.factorypath .vim .tmp -**/node_modules +node_modules npm-debug.log .vscode logs/* @@ -39,10 +41,110 @@ dolphinscheduler-alert/logs/ dolphinscheduler-alert/src/main/resources/alert.properties_bak dolphinscheduler-alert/src/main/resources/logback.xml dolphinscheduler-server/src/main/resources/logback.xml -dolphinscheduler-ui/dist/ +dolphinscheduler-ui/dist dolphinscheduler-ui/node -dolphinscheduler-dao/src/main/resources/dao/data_source.properties +dolphinscheduler-ui/dist/css/common.16ac5d9.css +dolphinscheduler-ui/dist/css/home/index.b444b91.css +dolphinscheduler-ui/dist/css/login/index.5866c64.css +dolphinscheduler-ui/dist/js/0.ac94e5d.js +dolphinscheduler-ui/dist/js/0.ac94e5d.js.map +dolphinscheduler-ui/dist/js/1.0b043a3.js +dolphinscheduler-ui/dist/js/1.0b043a3.js.map +dolphinscheduler-ui/dist/js/10.1bce3dc.js +dolphinscheduler-ui/dist/js/10.1bce3dc.js.map +dolphinscheduler-ui/dist/js/11.79f04d8.js +dolphinscheduler-ui/dist/js/11.79f04d8.js.map +dolphinscheduler-ui/dist/js/12.420daa5.js +dolphinscheduler-ui/dist/js/12.420daa5.js.map +dolphinscheduler-ui/dist/js/13.e5bae1c.js +dolphinscheduler-ui/dist/js/13.e5bae1c.js.map +dolphinscheduler-ui/dist/js/14.f2a0dca.js +dolphinscheduler-ui/dist/js/14.f2a0dca.js.map +dolphinscheduler-ui/dist/js/15.45373e8.js +dolphinscheduler-ui/dist/js/15.45373e8.js.map +dolphinscheduler-ui/dist/js/16.fecb0fc.js +dolphinscheduler-ui/dist/js/16.fecb0fc.js.map +dolphinscheduler-ui/dist/js/17.84be279.js +dolphinscheduler-ui/dist/js/17.84be279.js.map +dolphinscheduler-ui/dist/js/18.307ea70.js +dolphinscheduler-ui/dist/js/18.307ea70.js.map +dolphinscheduler-ui/dist/js/19.144db9c.js +dolphinscheduler-ui/dist/js/19.144db9c.js.map +dolphinscheduler-ui/dist/js/2.8b4ef29.js +dolphinscheduler-ui/dist/js/2.8b4ef29.js.map +dolphinscheduler-ui/dist/js/20.4c527e9.js +dolphinscheduler-ui/dist/js/20.4c527e9.js.map +dolphinscheduler-ui/dist/js/21.831b2a2.js +dolphinscheduler-ui/dist/js/21.831b2a2.js.map +dolphinscheduler-ui/dist/js/22.2b4bb2a.js +dolphinscheduler-ui/dist/js/22.2b4bb2a.js.map +dolphinscheduler-ui/dist/js/23.81467ef.js +dolphinscheduler-ui/dist/js/23.81467ef.js.map +dolphinscheduler-ui/dist/js/24.54a00e4.js +dolphinscheduler-ui/dist/js/24.54a00e4.js.map +dolphinscheduler-ui/dist/js/25.8d7bd36.js +dolphinscheduler-ui/dist/js/25.8d7bd36.js.map +dolphinscheduler-ui/dist/js/26.2ec5e78.js +dolphinscheduler-ui/dist/js/26.2ec5e78.js.map +dolphinscheduler-ui/dist/js/27.3ab48c2.js +dolphinscheduler-ui/dist/js/27.3ab48c2.js.map +dolphinscheduler-ui/dist/js/28.363088a.js +dolphinscheduler-ui/dist/js/28.363088a.js.map +dolphinscheduler-ui/dist/js/29.6c5853a.js +dolphinscheduler-ui/dist/js/29.6c5853a.js.map +dolphinscheduler-ui/dist/js/3.a0edb5b.js +dolphinscheduler-ui/dist/js/3.a0edb5b.js.map +dolphinscheduler-ui/dist/js/30.940fdd3.js +dolphinscheduler-ui/dist/js/30.940fdd3.js.map +dolphinscheduler-ui/dist/js/31.168a460.js +dolphinscheduler-ui/dist/js/31.168a460.js.map +dolphinscheduler-ui/dist/js/32.8df6594.js +dolphinscheduler-ui/dist/js/32.8df6594.js.map +dolphinscheduler-ui/dist/js/33.4480bbe.js +dolphinscheduler-ui/dist/js/33.4480bbe.js.map +dolphinscheduler-ui/dist/js/34.b407fe1.js +dolphinscheduler-ui/dist/js/34.b407fe1.js.map +dolphinscheduler-ui/dist/js/35.f340b0a.js +dolphinscheduler-ui/dist/js/35.f340b0a.js.map +dolphinscheduler-ui/dist/js/36.8880c2d.js +dolphinscheduler-ui/dist/js/36.8880c2d.js.map +dolphinscheduler-ui/dist/js/37.ea2a25d.js +dolphinscheduler-ui/dist/js/37.ea2a25d.js.map +dolphinscheduler-ui/dist/js/38.98a59ee.js +dolphinscheduler-ui/dist/js/38.98a59ee.js.map +dolphinscheduler-ui/dist/js/39.a5e958a.js +dolphinscheduler-ui/dist/js/39.a5e958a.js.map +dolphinscheduler-ui/dist/js/4.4ca44db.js +dolphinscheduler-ui/dist/js/4.4ca44db.js.map +dolphinscheduler-ui/dist/js/40.e187b1e.js +dolphinscheduler-ui/dist/js/40.e187b1e.js.map +dolphinscheduler-ui/dist/js/41.0e89182.js +dolphinscheduler-ui/dist/js/41.0e89182.js.map +dolphinscheduler-ui/dist/js/42.341047c.js +dolphinscheduler-ui/dist/js/42.341047c.js.map +dolphinscheduler-ui/dist/js/43.27b8228.js +dolphinscheduler-ui/dist/js/43.27b8228.js.map +dolphinscheduler-ui/dist/js/44.e8869bc.js +dolphinscheduler-ui/dist/js/44.e8869bc.js.map +dolphinscheduler-ui/dist/js/45.8d54901.js +dolphinscheduler-ui/dist/js/45.8d54901.js.map +dolphinscheduler-ui/dist/js/5.e1ed7f3.js +dolphinscheduler-ui/dist/js/5.e1ed7f3.js.map +dolphinscheduler-ui/dist/js/6.241ba07.js +dolphinscheduler-ui/dist/js/6.241ba07.js.map +dolphinscheduler-ui/dist/js/7.ab2e297.js +dolphinscheduler-ui/dist/js/7.ab2e297.js.map +dolphinscheduler-ui/dist/js/8.83ff814.js +dolphinscheduler-ui/dist/js/8.83ff814.js.map +dolphinscheduler-ui/dist/js/9.39cb29f.js +dolphinscheduler-ui/dist/js/9.39cb29f.js.map +dolphinscheduler-ui/dist/js/common.733e342.js +dolphinscheduler-ui/dist/js/common.733e342.js.map +dolphinscheduler-ui/dist/js/home/index.78a5d12.js +dolphinscheduler-ui/dist/js/home/index.78a5d12.js.map +dolphinscheduler-ui/dist/js/login/index.291b8e3.js +dolphinscheduler-ui/dist/js/login/index.291b8e3.js.map +dolphinscheduler-ui/dist/lib/external/ +dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/taskInstance/index.vue +/dolphinscheduler-dao/src/main/resources/dao/data_source.properties -.mvn/wrapper/*.jar - -!/zookeeper_data/ diff --git a/.mvn/wrapper/maven-wrapper.jar b/.mvn/wrapper/maven-wrapper.jar new file mode 100644 index 0000000000..2cc7d4a55c Binary files /dev/null and b/.mvn/wrapper/maven-wrapper.jar differ diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/configuration/dolphin-application.xml b/ambari_plugin/common-services/DOLPHIN/1.2.1/configuration/dolphin-application.xml deleted file mode 100644 index 6e50a1b649..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/1.2.1/configuration/dolphin-application.xml +++ /dev/null @@ -1,467 +0,0 @@ - - - - spring.datasource.initialSize - 5 - - Init connection number - - - int - - - - - spring.datasource.minIdle - 5 - - Min connection number - - - int - - - - - spring.datasource.maxActive - 50 - - Max connection number - - - int - - - - - spring.datasource.maxWait - 60000 - - Max wait time for get a connection in milliseconds. - If configuring maxWait, fair locks are enabled by default and concurrency efficiency decreases. - If necessary, unfair locks can be used by configuring the useUnfairLock attribute to true. - - - int - - - - - spring.datasource.timeBetweenEvictionRunsMillis - 60000 - - Milliseconds for check to close free connections - - - int - - - - - spring.datasource.timeBetweenConnectErrorMillis - 60000 - - The Destroy thread detects the connection interval and closes the physical connection in milliseconds - if the connection idle time is greater than or equal to minEvictableIdleTimeMillis. - - - int - - - - - spring.datasource.minEvictableIdleTimeMillis - 300000 - - The longest time a connection remains idle without being evicted, in milliseconds - - - int - - - - - spring.datasource.validationQuery - SELECT 1 - - The SQL used to check whether the connection is valid requires a query statement. - If validation Query is null, testOnBorrow, testOnReturn, and testWhileIdle will not work. - - - - - spring.datasource.validationQueryTimeout - 3 - - int - - - Check whether the connection is valid for timeout, in seconds - - - - - spring.datasource.testWhileIdle - true - - boolean - - - When applying for a connection, - if it is detected that the connection is idle longer than time Between Eviction Runs Millis, - validation Query is performed to check whether the connection is valid - - - - - spring.datasource.testOnBorrow - true - - boolean - - - Execute validation to check if the connection is valid when applying for a connection - - - - - spring.datasource.testOnReturn - false - - boolean - - - Execute validation to check if the connection is valid when the connection is returned - - - - - spring.datasource.defaultAutoCommit - true - - boolean - - - - - - - spring.datasource.keepAlive - false - - boolean - - - - - - - - spring.datasource.poolPreparedStatements - true - - boolean - - - Open PSCache, specify count PSCache for every connection - - - - - spring.datasource.maxPoolPreparedStatementPerConnectionSize - 20 - - int - - - - - - spring.datasource.spring.datasource.filters - stat,wall,log4j - - - - - spring.datasource.connectionProperties - druid.stat.mergeSql=true;druid.stat.slowSqlMillis=5000 - - - - - - mybatis-plus.mapper-locations - classpath*:/org.apache.dolphinscheduler.dao.mapper/*.xml - - - - - mybatis-plus.typeEnumsPackage - org.apache.dolphinscheduler.*.enums - - - - - mybatis-plus.typeAliasesPackage - org.apache.dolphinscheduler.dao.entity - - Entity scan, where multiple packages are separated by a comma or semicolon - - - - - mybatis-plus.global-config.db-config.id-type - AUTO - - value-list - - - AUTO - - - - INPUT - - - - ID_WORKER - - - - UUID - - - - 1 - - - Primary key type AUTO:" database ID AUTO ", - INPUT:" user INPUT ID", - ID_WORKER:" global unique ID (numeric type unique ID)", - UUID:" global unique ID UUID"; - - - - - mybatis-plus.global-config.db-config.field-strategy - NOT_NULL - - value-list - - - IGNORED - - - - NOT_NULL - - - - NOT_EMPTY - - - - 1 - - - Field policy IGNORED:" ignore judgment ", - NOT_NULL:" not NULL judgment "), - NOT_EMPTY:" not NULL judgment" - - - - - mybatis-plus.global-config.db-config.column-underline - true - - boolean - - - - - - mybatis-plus.global-config.db-config.logic-delete-value - 1 - - int - - - - - - mybatis-plus.global-config.db-config.logic-not-delete-value - 0 - - int - - - - - - mybatis-plus.global-config.db-config.banner - true - - boolean - - - - - - - mybatis-plus.configuration.map-underscore-to-camel-case - true - - boolean - - - - - - mybatis-plus.configuration.cache-enabled - false - - boolean - - - - - - mybatis-plus.configuration.call-setters-on-nulls - true - - boolean - - - - - - mybatis-plus.configuration.jdbc-type-for-null - null - - - - - master.exec.threads - 100 - - int - - - - - - master.exec.task.num - 20 - - int - - - - - - master.heartbeat.interval - 10 - - int - - - - - - master.task.commit.retryTimes - 5 - - int - - - - - - master.task.commit.interval - 1000 - - int - - - - - - master.max.cpuload.avg - 100 - - int - - - - - - master.reserved.memory - 0.1 - - float - - - - - - worker.exec.threads - 100 - - int - - - - - - worker.heartbeat.interval - 10 - - int - - - - - - worker.fetch.task.num - 3 - - int - - - - - - worker.max.cpuload.avg - 100 - - int - - - - - - worker.reserved.memory - 0.1 - - float - - - - - - \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/alerts.json b/ambari_plugin/common-services/DOLPHIN/1.3.3/alerts.json similarity index 95% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/alerts.json rename to ambari_plugin/common-services/DOLPHIN/1.3.3/alerts.json index 769245b366..184f021ac3 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.2.1/alerts.json +++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/alerts.json @@ -65,7 +65,7 @@ "enabled": true, "source": { "type": "SCRIPT", - "path": "DOLPHIN/1.2.1/package/alerts/alert_dolphin_scheduler_status.py", + "path": "DOLPHIN/1.3.3/package/alerts/alert_dolphin_scheduler_status.py", "parameters": [ { @@ -98,7 +98,7 @@ "enabled": true, "source": { "type": "SCRIPT", - "path": "DOLPHIN/1.2.1/package/alerts/alert_dolphin_scheduler_status.py", + "path": "DOLPHIN/1.3.3/package/alerts/alert_dolphin_scheduler_status.py", "parameters": [ { @@ -131,7 +131,7 @@ "enabled": true, "source": { "type": "SCRIPT", - "path": "DOLPHIN/1.2.1/package/alerts/alert_dolphin_scheduler_status.py", + "path": "DOLPHIN/1.3.3/package/alerts/alert_dolphin_scheduler_status.py", "parameters": [ { diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/configuration/dolphin-alert.xml b/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-alert.xml similarity index 95% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/configuration/dolphin-alert.xml rename to ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-alert.xml index 5b82230148..32abcc791d 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.2.1/configuration/dolphin-alert.xml +++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-alert.xml @@ -90,13 +90,6 @@ - - xls.file.path - /tmp/xls - - - - enterprise.wechat.enable false diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/configuration/dolphin-application-api.xml b/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-application-api.xml similarity index 82% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/configuration/dolphin-application-api.xml rename to ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-application-api.xml index ea4cb82afd..766c0f477d 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.2.1/configuration/dolphin-application-api.xml +++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-application-api.xml @@ -34,6 +34,12 @@ + + server.servlet.context-path + /dolphinscheduler/ + + + spring.servlet.multipart.max-file-size 1024 @@ -68,4 +74,14 @@ UTF-8 + + spring.messages.basename + i18n/messages + + + + security.authentication.type + PASSWORD + + \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/configuration/dolphin-common.xml b/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-common.xml similarity index 100% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/configuration/dolphin-common.xml rename to ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-common.xml diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-datasource.xml b/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-datasource.xml new file mode 100644 index 0000000000..02d8de0482 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-datasource.xml @@ -0,0 +1,206 @@ + + + + spring.datasource.initialSize + 5 + + Init connection number + + + int + + + + + spring.datasource.minIdle + 5 + + Min connection number + + + int + + + + + spring.datasource.maxActive + 50 + + Max connection number + + + int + + + + + spring.datasource.maxWait + 60000 + + Max wait time for get a connection in milliseconds. + If configuring maxWait, fair locks are enabled by default and concurrency efficiency decreases. + If necessary, unfair locks can be used by configuring the useUnfairLock attribute to true. + + + int + + + + + spring.datasource.timeBetweenEvictionRunsMillis + 60000 + + Milliseconds for check to close free connections + + + int + + + + + spring.datasource.timeBetweenConnectErrorMillis + 60000 + + The Destroy thread detects the connection interval and closes the physical connection in milliseconds + if the connection idle time is greater than or equal to minEvictableIdleTimeMillis. + + + int + + + + + spring.datasource.minEvictableIdleTimeMillis + 300000 + + The longest time a connection remains idle without being evicted, in milliseconds + + + int + + + + + spring.datasource.validationQuery + SELECT 1 + + The SQL used to check whether the connection is valid requires a query statement. + If validation Query is null, testOnBorrow, testOnReturn, and testWhileIdle will not work. + + + + + spring.datasource.validationQueryTimeout + 3 + + int + + + Check whether the connection is valid for timeout, in seconds + + + + + spring.datasource.testWhileIdle + true + + boolean + + + When applying for a connection, + if it is detected that the connection is idle longer than time Between Eviction Runs Millis, + validation Query is performed to check whether the connection is valid + + + + + spring.datasource.testOnBorrow + true + + boolean + + + Execute validation to check if the connection is valid when applying for a connection + + + + + spring.datasource.testOnReturn + false + + boolean + + + Execute validation to check if the connection is valid when the connection is returned + + + + + spring.datasource.defaultAutoCommit + true + + boolean + + + + + + + spring.datasource.keepAlive + false + + boolean + + + + + + + + spring.datasource.poolPreparedStatements + true + + boolean + + + Open PSCache, specify count PSCache for every connection + + + + + spring.datasource.maxPoolPreparedStatementPerConnectionSize + 20 + + int + + + + + + spring.datasource.spring.datasource.filters + stat,wall,log4j + + + + + spring.datasource.connectionProperties + druid.stat.mergeSql=true;druid.stat.slowSqlMillis=5000 + + + + \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/configuration/dolphin-env.xml b/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-env.xml similarity index 100% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/configuration/dolphin-env.xml rename to ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-env.xml diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-master.xml b/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-master.xml new file mode 100644 index 0000000000..c8eec047fc --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-master.xml @@ -0,0 +1,88 @@ + + + + master.exec.threads + 100 + + int + + master execute thread num + + + + master.exec.task.num + 20 + + int + + master execute task number in parallel + + + + master.heartbeat.interval + 10 + + int + + master heartbeat interval + + + + master.task.commit.retryTimes + 5 + + int + + master commit task retry times + + + + master.task.commit.interval + 1000 + + int + + master commit task interval + + + + master.max.cpuload.avg + 100 + + int + + only less than cpu avg load, master server can work. default value : the number of cpu cores * 2 + + + + master.reserved.memory + 0.3 + only larger than reserved memory, master server can work. default value : physical memory * 1/10, unit is G. + + + + + master.listen.port + 5678 + + int + + master listen port + + + \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/configuration/dolphin-quartz.xml b/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-quartz.xml similarity index 91% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/configuration/dolphin-quartz.xml rename to ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-quartz.xml index 82b59d8827..7a0c68b051 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.2.1/configuration/dolphin-quartz.xml +++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-quartz.xml @@ -106,26 +106,21 @@ - org.quartz.jobStore.dataSource - myDs - - - - org.quartz.dataSource.myDs.connectionProvider.class - org.apache.dolphinscheduler.server.quartz.DruidConnectionProvider + org.quartz.jobStore.acquireTriggersWithinLock + true + + boolean + - org.quartz.dataSource.myDs.maxConnections - 10 - - int - + org.quartz.jobStore.dataSource + myDs - org.quartz.dataSource.myDs.validationQuery - select 1 + org.quartz.dataSource.myDs.connectionProvider.class + org.apache.dolphinscheduler.service.quartz.DruidConnectionProvider \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-worker.xml b/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-worker.xml new file mode 100644 index 0000000000..1ae7a1a765 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-worker.xml @@ -0,0 +1,67 @@ + + + + worker.exec.threads + 100 + + int + + worker execute thread num + + + + worker.heartbeat.interval + 10 + + int + + worker heartbeat interval + + + + worker.max.cpuload.avg + 100 + + int + + only less than cpu avg load, worker server can work. default value : the number of cpu cores * 2 + + + + worker.reserved.memory + 0.3 + only larger than reserved memory, worker server can work. default value : physical memory * 1/10, unit is G. + + + + + worker.listen.port + 1234 + + int + + worker listen port + + + + worker.groups + default + default worker group + + + \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-zookeeper.xml b/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-zookeeper.xml new file mode 100644 index 0000000000..e89962d900 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-zookeeper.xml @@ -0,0 +1,76 @@ + + + + zookeeper.dolphinscheduler.root + /dolphinscheduler + + dolphinscheduler root directory + + + + + zookeeper.session.timeout + 300 + + int + + + + + + + zookeeper.connection.timeout + 300 + + int + + + + + + + zookeeper.retry.base.sleep + 100 + + int + + + + + + + zookeeper.retry.max.sleep + 30000 + + int + + + + + + + zookeeper.retry.maxtime + 5 + + int + + + + + + \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/metainfo.xml b/ambari_plugin/common-services/DOLPHIN/1.3.3/metainfo.xml similarity index 98% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/metainfo.xml rename to ambari_plugin/common-services/DOLPHIN/1.3.3/metainfo.xml index 0d2bbe3163..074306d5cb 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.2.1/metainfo.xml +++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/metainfo.xml @@ -22,7 +22,7 @@ DOLPHIN Dolphin Scheduler 分布式易扩展的可视化DAG工作流任务调度系统 - 1.2.1 + 1.3.3 DOLPHIN_MASTER @@ -103,7 +103,7 @@ any - apache-dolphinscheduler-incubating-1.2.1* + apache-dolphinscheduler-incubating* @@ -134,4 +134,4 @@ - + \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/alerts/alert_dolphin_scheduler_status.py b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/alerts/alert_dolphin_scheduler_status.py similarity index 100% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/package/alerts/alert_dolphin_scheduler_status.py rename to ambari_plugin/common-services/DOLPHIN/1.3.3/package/alerts/alert_dolphin_scheduler_status.py diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/dolphin_alert_service.py b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_alert_service.py similarity index 92% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/dolphin_alert_service.py rename to ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_alert_service.py index 62255a3432..e78c38d272 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/dolphin_alert_service.py +++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_alert_service.py @@ -26,7 +26,8 @@ class DolphinAlertService(Script): import params env.set_params(params) self.install_packages(env) - Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True) + Execute(('chmod', '-R', '777', params.dolphin_home)) + Execute(('chown', '-R', params.dolphin_user + ":" + params.dolphin_group, params.dolphin_home)) def configure(self, env): import params diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/dolphin_api_service.py b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_api_service.py similarity index 93% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/dolphin_api_service.py rename to ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_api_service.py index bdc18fb602..5a28924a9a 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/dolphin_api_service.py +++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_api_service.py @@ -26,7 +26,8 @@ class DolphinApiService(Script): import params env.set_params(params) self.install_packages(env) - Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True) + Execute(('chmod', '-R', '777', params.dolphin_home)) + Execute(('chown', '-R', params.dolphin_user + ":" + params.dolphin_group, params.dolphin_home)) def configure(self, env): import params diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/dolphin_env.py b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_env.py similarity index 78% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/dolphin_env.py rename to ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_env.py index 235605894f..1661d76c75 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/dolphin_env.py +++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_env.py @@ -42,31 +42,12 @@ def dolphin_env(): create_parents=True ) - - Directory(params.dolphin_alert_map['xls.file.path'], - mode=0777, - owner=params.dolphin_user, - group=params.dolphin_group, - create_parents=True - ) Directory(params.dolphin_common_map['data.basedir.path'], mode=0777, owner=params.dolphin_user, group=params.dolphin_group, create_parents=True ) - Directory(params.dolphin_common_map['data.download.basedir.path'], - mode=0777, - owner=params.dolphin_user, - group=params.dolphin_group, - create_parents=True - ) - Directory(params.dolphin_common_map['process.exec.basepath'], - mode=0777, - owner=params.dolphin_user, - group=params.dolphin_group, - create_parents=True - ) File(format(params.dolphin_env_path), @@ -79,11 +60,25 @@ def dolphin_env(): File(format(params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh"), mode=0755, - content=Template("dolphin-daemon.j2"), + content=Template("dolphin-daemon.sh.j2"), owner=params.dolphin_user, group=params.dolphin_group ) + File(format(params.dolphin_conf_dir + "/master.properties"), + mode=0755, + content=Template("master.properties.j2"), + owner=params.dolphin_user, + group=params.dolphin_group + ) + + File(format(params.dolphin_conf_dir + "/worker.properties"), + mode=0755, + content=Template("worker.properties.j2"), + owner=params.dolphin_user, + group=params.dolphin_group + ) + File(format(params.dolphin_conf_dir + "/alert.properties"), mode=0755, @@ -92,9 +87,9 @@ def dolphin_env(): group=params.dolphin_group ) - File(format(params.dolphin_conf_dir + "/application.properties"), + File(format(params.dolphin_conf_dir + "/datasource.properties"), mode=0755, - content=Template("application.properties.j2"), + content=Template("datasource.properties.j2"), owner=params.dolphin_user, group=params.dolphin_group ) @@ -119,3 +114,10 @@ def dolphin_env(): owner=params.dolphin_user, group=params.dolphin_group ) + + File(format(params.dolphin_conf_dir + "/zookeeper.properties"), + mode=0755, + content=Template("zookeeper.properties.j2"), + owner=params.dolphin_user, + group=params.dolphin_group + ) diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/dolphin_logger_service.py b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_logger_service.py similarity index 92% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/dolphin_logger_service.py rename to ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_logger_service.py index f1c19bd66f..fb47e132e1 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/dolphin_logger_service.py +++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_logger_service.py @@ -26,8 +26,8 @@ class DolphinLoggerService(Script): import params env.set_params(params) self.install_packages(env) - Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True) - + Execute(('chmod', '-R', '777', params.dolphin_home)) + Execute(('chown', '-R', params.dolphin_user + ":" + params.dolphin_group, params.dolphin_home)) def configure(self, env): import params params.pika_slave = True diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/dolphin_master_service.py b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_master_service.py similarity index 92% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/dolphin_master_service.py rename to ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_master_service.py index 6ee7ecfcf3..8d64935d26 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/dolphin_master_service.py +++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_master_service.py @@ -27,7 +27,8 @@ class DolphinMasterService(Script): import params env.set_params(params) self.install_packages(env) - Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True) + Execute(('chmod', '-R', '777', params.dolphin_home)) + Execute(('chown', '-R', params.dolphin_user + ":" + params.dolphin_group, params.dolphin_home)) def configure(self, env): import params diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/dolphin_worker_service.py b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_worker_service.py similarity index 92% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/dolphin_worker_service.py rename to ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_worker_service.py index 2d145ee730..1f542c06c2 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/dolphin_worker_service.py +++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_worker_service.py @@ -26,7 +26,8 @@ class DolphinWorkerService(Script): import params env.set_params(params) self.install_packages(env) - Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True) + Execute(('chmod', '-R', '777', params.dolphin_home)) + Execute(('chown', '-R', params.dolphin_user + ":" + params.dolphin_group, params.dolphin_home)) def configure(self, env): import params diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/params.py b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/params.py similarity index 68% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/params.py rename to ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/params.py index 049b2cf3ae..5a9994f559 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/params.py +++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/params.py @@ -54,11 +54,8 @@ dolphin_env_content = dolphin_env_map['dolphinscheduler-env-content'] # database config dolphin_database_config = {} dolphin_database_config['dolphin_database_type'] = dolphin_env_map['dolphin.database.type'] -dolphin_database_config['dolphin_database_host'] = dolphin_env_map['dolphin.database.host'] -dolphin_database_config['dolphin_database_port'] = dolphin_env_map['dolphin.database.port'] dolphin_database_config['dolphin_database_username'] = dolphin_env_map['dolphin.database.username'] dolphin_database_config['dolphin_database_password'] = dolphin_env_map['dolphin.database.password'] - if 'mysql' == dolphin_database_config['dolphin_database_type']: dolphin_database_config['dolphin_database_driver'] = 'com.mysql.jdbc.Driver' dolphin_database_config['driverDelegateClass'] = 'org.quartz.impl.jdbcjobstore.StdJDBCDelegate' @@ -72,6 +69,10 @@ else: + ':' + dolphin_env_map['dolphin.database.port'] \ + '/dolphinscheduler' + + + + # application-alert.properties dolphin_alert_map = {} wechat_push_url = 'https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=$token' @@ -79,27 +80,22 @@ wechat_token_url = 'https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid=$corpId& wechat_team_send_msg = '{\"toparty\":\"{toParty}\",\"agentid\":\"{agentId}\",\"msgtype\":\"text\",\"text\":{\"content\":\"{msg}\"},\"safe\":\"0\"}' wechat_user_send_msg = '{\"touser\":\"{toUser}\",\"agentid\":\"{agentId}\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"{msg}\"}}' -dolphin_alert_map['enterprise.wechat.push.ur'] = wechat_push_url -dolphin_alert_map['enterprise.wechat.token.url'] = wechat_token_url -dolphin_alert_map['enterprise.wechat.team.send.msg'] = wechat_team_send_msg -dolphin_alert_map['enterprise.wechat.user.send.msg'] = wechat_user_send_msg -dolphin_alert_map.update(config['configurations']['dolphin-alert']) +dolphin_alert_config_map = config['configurations']['dolphin-alert'] + +if dolphin_alert_config_map['enterprise.wechat.enable']: + dolphin_alert_map['enterprise.wechat.push.ur'] = wechat_push_url + dolphin_alert_map['enterprise.wechat.token.url'] = wechat_token_url + dolphin_alert_map['enterprise.wechat.team.send.msg'] = wechat_team_send_msg + dolphin_alert_map['enterprise.wechat.user.send.msg'] = wechat_user_send_msg + +dolphin_alert_map.update(dolphin_alert_config_map) + + # application-api.properties dolphin_app_api_map = {} -dolphin_app_api_map['logging.config'] = 'classpath:apiserver_logback.xml' -dolphin_app_api_map['spring.messages.basename'] = 'i18n/messages' -dolphin_app_api_map['server.servlet.context-path'] = '/dolphinscheduler/' dolphin_app_api_map.update(config['configurations']['dolphin-application-api']) -# application-dao.properties -dolphin_application_map = {} -dolphin_application_map['spring.datasource.type'] = 'com.alibaba.druid.pool.DruidDataSource' -dolphin_application_map['spring.datasource.driver-class-name'] = dolphin_database_config['dolphin_database_driver'] -dolphin_application_map['spring.datasource.url'] = dolphin_database_config['dolphin_database_url'] -dolphin_application_map['spring.datasource.username'] = dolphin_database_config['dolphin_database_username'] -dolphin_application_map['spring.datasource.password'] = dolphin_database_config['dolphin_database_password'] -dolphin_application_map.update(config['configurations']['dolphin-application']) # common.properties dolphin_common_map = {} @@ -118,33 +114,42 @@ else: dolphin_common_map_tmp = config['configurations']['dolphin-common'] data_basedir_path = dolphin_common_map_tmp['data.basedir.path'] -process_exec_basepath = data_basedir_path + '/exec' -data_download_basedir_path = data_basedir_path + '/download' -dolphin_common_map['process.exec.basepath'] = process_exec_basepath -dolphin_common_map['data.download.basedir.path'] = data_download_basedir_path dolphin_common_map['dolphinscheduler.env.path'] = dolphin_env_path +dolphin_common_map.update(config['configurations']['dolphin-common']) -zookeeperHosts = default("/clusterHostInfo/zookeeper_hosts", []) -if len(zookeeperHosts) > 0 and "clientPort" in config['configurations']['zoo.cfg']: - clientPort = config['configurations']['zoo.cfg']['clientPort'] - zookeeperPort = ":" + clientPort + "," - dolphin_common_map['zookeeper.quorum'] = zookeeperPort.join(zookeeperHosts) + ":" + clientPort +# datasource.properties +dolphin_datasource_map = {} +dolphin_datasource_map['spring.datasource.type'] = 'com.alibaba.druid.pool.DruidDataSource' +dolphin_datasource_map['spring.datasource.driver-class-name'] = dolphin_database_config['dolphin_database_driver'] +dolphin_datasource_map['spring.datasource.url'] = dolphin_database_config['dolphin_database_url'] +dolphin_datasource_map['spring.datasource.username'] = dolphin_database_config['dolphin_database_username'] +dolphin_datasource_map['spring.datasource.password'] = dolphin_database_config['dolphin_database_password'] +dolphin_datasource_map.update(config['configurations']['dolphin-datasource']) -dolphin_common_map.update(config['configurations']['dolphin-common']) +# master.properties +dolphin_master_map = config['configurations']['dolphin-master'] # quartz.properties dolphin_quartz_map = {} dolphin_quartz_map['org.quartz.jobStore.driverDelegateClass'] = dolphin_database_config['driverDelegateClass'] -dolphin_quartz_map['org.quartz.dataSource.myDs.driver'] = dolphin_database_config['dolphin_database_driver'] -dolphin_quartz_map['org.quartz.dataSource.myDs.URL'] = dolphin_database_config['dolphin_database_url'] -dolphin_quartz_map['org.quartz.dataSource.myDs.user'] = dolphin_database_config['dolphin_database_username'] -dolphin_quartz_map['org.quartz.dataSource.myDs.password'] = dolphin_database_config['dolphin_database_password'] dolphin_quartz_map.update(config['configurations']['dolphin-quartz']) -# if 'ganglia_server_host' in config['clusterHostInfo'] and \ -# len(config['clusterHostInfo']['ganglia_server_host'])>0: -# ganglia_installed = True -# ganglia_server = config['clusterHostInfo']['ganglia_server_host'][0] -# ganglia_report_interval = 60 -# else: -# ganglia_installed = False +# worker.properties +dolphin_worker_map = config['configurations']['dolphin-worker'] + +# zookeeper.properties +dolphin_zookeeper_map={} +zookeeperHosts = default("/clusterHostInfo/zookeeper_hosts", []) +if len(zookeeperHosts) > 0 and "clientPort" in config['configurations']['zoo.cfg']: + clientPort = config['configurations']['zoo.cfg']['clientPort'] + zookeeperPort = ":" + clientPort + "," + dolphin_zookeeper_map['zookeeper.quorum'] = zookeeperPort.join(zookeeperHosts) + ":" + clientPort +dolphin_zookeeper_map.update(config['configurations']['dolphin-zookeeper']) +if 'spring.servlet.multipart.max-file-size' in dolphin_app_api_map: + file_size = dolphin_app_api_map['spring.servlet.multipart.max-file-size'] + dolphin_app_api_map['spring.servlet.multipart.max-file-size'] = file_size + "MB" +if 'spring.servlet.multipart.max-request-size' in dolphin_app_api_map: + request_size = dolphin_app_api_map['spring.servlet.multipart.max-request-size'] + dolphin_app_api_map['spring.servlet.multipart.max-request-size'] = request_size + "MB" + + diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/service_check.py b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/service_check.py similarity index 100% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/service_check.py rename to ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/service_check.py diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/status_params.py b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/status_params.py similarity index 100% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/status_params.py rename to ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/status_params.py diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/templates/alert.properties.j2 b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/alert.properties.j2 similarity index 100% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/package/templates/alert.properties.j2 rename to ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/alert.properties.j2 diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/templates/application-api.properties.j2 b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/application-api.properties.j2 similarity index 100% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/package/templates/application-api.properties.j2 rename to ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/application-api.properties.j2 diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/templates/common.properties.j2 b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/common.properties.j2 similarity index 100% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/package/templates/common.properties.j2 rename to ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/common.properties.j2 diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/datasource.properties.j2 b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/datasource.properties.j2 new file mode 100644 index 0000000000..40aed83543 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/datasource.properties.j2 @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{% for key, value in dolphin_datasource_map.iteritems() -%} + {{key}}={{value}} +{% endfor %} \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/templates/dolphin-daemon.j2 b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/dolphin-daemon.sh.j2 similarity index 83% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/package/templates/dolphin-daemon.j2 rename to ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/dolphin-daemon.sh.j2 index 1dc4bac0ab..c5cc11fb62 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/templates/dolphin-daemon.j2 +++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/dolphin-daemon.sh.j2 @@ -48,22 +48,19 @@ pid={{dolphin_pidfile_dir}}/$command.pid cd $DOLPHINSCHEDULER_HOME if [ "$command" = "api-server" ]; then - LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/apiserver_logback.xml -Dspring.profiles.active=api" + LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/logback-api.xml -Dspring.profiles.active=api" CLASS=org.apache.dolphinscheduler.api.ApiApplicationServer elif [ "$command" = "master-server" ]; then - LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/master_logback.xml -Ddruid.mysql.usePingMethod=false" + LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/logback-master.xml -Ddruid.mysql.usePingMethod=false" CLASS=org.apache.dolphinscheduler.server.master.MasterServer elif [ "$command" = "worker-server" ]; then - LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/worker_logback.xml -Ddruid.mysql.usePingMethod=false" + LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/logback-worker.xml -Ddruid.mysql.usePingMethod=false" CLASS=org.apache.dolphinscheduler.server.worker.WorkerServer elif [ "$command" = "alert-server" ]; then - LOG_FILE="-Dlogback.configurationFile={{dolphin_conf_dir}}/alert_logback.xml" + LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/logback-alert.xml" CLASS=org.apache.dolphinscheduler.alert.AlertServer elif [ "$command" = "logger-server" ]; then - CLASS=org.apache.dolphinscheduler.server.rpc.LoggerServer -elif [ "$command" = "combined-server" ]; then - LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/combined_logback.xml -Dspring.profiles.active=api -Dserver.is-combined-server=true" - CLASS=org.apache.dolphinscheduler.api.CombinedApplicationServer + CLASS=org.apache.dolphinscheduler.server.log.LoggerServer else echo "Error: No command named \`$command' was found." exit 1 diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/master.properties.j2 b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/master.properties.j2 new file mode 100644 index 0000000000..d9b85e14cf --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/master.properties.j2 @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{% for key, value in dolphin_master_map.iteritems() -%} + {{key}}={{value}} +{% endfor %} \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/templates/quartz.properties.j2 b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/quartz.properties.j2 similarity index 100% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/package/templates/quartz.properties.j2 rename to ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/quartz.properties.j2 diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/worker.properties.j2 b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/worker.properties.j2 new file mode 100644 index 0000000000..a008b74084 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/worker.properties.j2 @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{% for key, value in dolphin_worker_map.iteritems() -%} + {{key}}={{value}} +{% endfor %} \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/zookeeper.properties.j2 b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/zookeeper.properties.j2 new file mode 100644 index 0000000000..9eb14eaef3 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/zookeeper.properties.j2 @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{% for key, value in dolphin_zookeeper_map.iteritems() -%} + {{key}}={{value}} +{% endfor %} \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/quicklinks/quicklinks.json b/ambari_plugin/common-services/DOLPHIN/1.3.3/quicklinks/quicklinks.json old mode 100755 new mode 100644 similarity index 100% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/quicklinks/quicklinks.json rename to ambari_plugin/common-services/DOLPHIN/1.3.3/quicklinks/quicklinks.json diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/themes/theme.json b/ambari_plugin/common-services/DOLPHIN/1.3.3/themes/theme.json similarity index 86% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/themes/theme.json rename to ambari_plugin/common-services/DOLPHIN/1.3.3/themes/theme.json index 23e46076aa..953e2323f8 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.2.1/themes/theme.json +++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/themes/theme.json @@ -151,18 +151,40 @@ "subsection-name": "env-row1-col2" }, { - "config": "dolphin-common/res.upload.startup.type", + "config": "dolphin-common/resource.storage.type", "subsection-name": "dynamic-row1-col1" }, + { + "config": "dolphin-common/resource.upload.path", + "subsection-name": "dynamic-row1-col1", + "depends-on": [ + { + "configs":[ + "dolphin-common/resource.storage.type" + ], + "if": "${dolphin-common/resource.storage.type} === HDFS || ${dolphin-common/resource.storage.type} === S3", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, { "config": "dolphin-common/hdfs.root.user", "subsection-name": "dynamic-row1-col1", "depends-on": [ { "configs":[ - "dolphin-common/res.upload.startup.type" + "dolphin-common/resource.storage.type" ], - "if": "${dolphin-common/res.upload.startup.type} === HDFS", + "if": "${dolphin-common/resource.storage.type} === HDFS", "then": { "property_value_attributes": { "visible": true @@ -182,9 +204,9 @@ "depends-on": [ { "configs":[ - "dolphin-common/res.upload.startup.type" + "dolphin-common/resource.storage.type" ], - "if": "${dolphin-common/res.upload.startup.type} === HDFS", + "if": "${dolphin-common/resource.storage.type} === HDFS", "then": { "property_value_attributes": { "visible": true @@ -204,9 +226,9 @@ "depends-on": [ { "configs":[ - "dolphin-common/res.upload.startup.type" + "dolphin-common/resource.storage.type" ], - "if": "${dolphin-common/res.upload.startup.type} === HDFS", + "if": "${dolphin-common/resource.storage.type} === HDFS", "then": { "property_value_attributes": { "visible": true @@ -226,9 +248,9 @@ "depends-on": [ { "configs":[ - "dolphin-common/res.upload.startup.type" + "dolphin-common/resource.storage.type" ], - "if": "${dolphin-common/res.upload.startup.type} === S3", + "if": "${dolphin-common/resource.storage.type} === S3", "then": { "property_value_attributes": { "visible": true @@ -248,9 +270,9 @@ "depends-on": [ { "configs":[ - "dolphin-common/res.upload.startup.type" + "dolphin-common/resource.storage.type" ], - "if": "${dolphin-common/res.upload.startup.type} === S3", + "if": "${dolphin-common/resource.storage.type} === S3", "then": { "property_value_attributes": { "visible": true @@ -270,9 +292,9 @@ "depends-on": [ { "configs":[ - "dolphin-common/res.upload.startup.type" + "dolphin-common/resource.storage.type" ], - "if": "${dolphin-common/res.upload.startup.type} === S3", + "if": "${dolphin-common/resource.storage.type} === S3", "then": { "property_value_attributes": { "visible": true @@ -356,6 +378,28 @@ } ] }, + { + "config": "dolphin-common/kerberos.expire.time", + "subsection-name": "dynamic-row1-col2", + "depends-on": [ + { + "configs":[ + "dolphin-common/hadoop.security.authentication.startup.state" + ], + "if": "${dolphin-common/hadoop.security.authentication.startup.state}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, { "config": "dolphin-alert/enterprise.wechat.enable", "subsection-name": "dynamic-row1-col3" @@ -505,11 +549,17 @@ } }, { - "config": "dolphin-common/res.upload.startup.type", + "config": "dolphin-common/resource.storage.type", "widget": { "type": "combo" } }, + { + "config": "dolphin-common/resource.upload.path", + "widget": { + "type": "text-field" + } + }, { "config": "dolphin-common/hdfs.root.user", "widget": { @@ -570,6 +620,12 @@ "type": "text-field" } }, + { + "config": "dolphin-common/kerberos.expire.time", + "widget": { + "type": "text-field" + } + }, { "config": "dolphin-alert/enterprise.wechat.enable", "widget": { diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/templates/application.properties.j2 b/docker/kubernetes/dolphinscheduler/requirements.yaml similarity index 76% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/package/templates/application.properties.j2 rename to docker/kubernetes/dolphinscheduler/requirements.yaml index 7bb9f8aff3..e219975995 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/templates/application.properties.j2 +++ b/docker/kubernetes/dolphinscheduler/requirements.yaml @@ -14,7 +14,12 @@ # See the License for the specific language governing permissions and # limitations under the License. # - -{% for key, value in dolphin_application_map.iteritems() -%} - {{key}}={{value}} -{% endfor %} \ No newline at end of file +dependencies: +- name: postgresql + version: 8.x.x + repository: https://charts.bitnami.com/bitnami + condition: postgresql.enabled +- name: zookeeper + version: 5.x.x + repository: https://charts.bitnami.com/bitnami + condition: redis.enabled \ No newline at end of file diff --git a/dolphinscheduler-alert/pom.xml b/dolphinscheduler-alert/pom.xml index 215916ddf7..a44d101ffe 100644 --- a/dolphinscheduler-alert/pom.xml +++ b/dolphinscheduler-alert/pom.xml @@ -21,7 +21,7 @@ org.apache.dolphinscheduler dolphinscheduler - 1.3.2-SNAPSHOT + 1.3.4-SNAPSHOT dolphinscheduler-alert ${project.artifactId} diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPluginTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPluginTest.java index c2f4006752..cdbcf276d3 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPluginTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPluginTest.java @@ -74,7 +74,7 @@ public class EmailAlertPluginTest { alertInfo.setAlertData(alertData); List list = new ArrayList(){{ add("xx@xx.com"); }}; alertInfo.addProp("receivers", list); - Map ret = plugin.process(alertInfo); - assertFalse(Boolean.parseBoolean(String.valueOf(ret.get(Constants.STATUS)))); +// Map ret = plugin.process(alertInfo); +// assertFalse(Boolean.parseBoolean(String.valueOf(ret.get(Constants.STATUS)))); } } \ No newline at end of file diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index cd6e04fc26..2af0ed2ffa 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -21,7 +21,7 @@ org.apache.dolphinscheduler dolphinscheduler - 1.3.2-SNAPSHOT + 1.3.4-SNAPSHOT dolphinscheduler-api ${project.artifactId} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AccessTokenController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AccessTokenController.java index 2457177cdf..17faad04bc 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AccessTokenController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AccessTokenController.java @@ -86,7 +86,7 @@ public class AccessTokenController extends BaseController { logger.info("login user {}, create token , userId : {} , token expire time : {} , token : {}", loginUser.getUserName(), userId, expireTime, token); - Map result = accessTokenService.createToken(userId, expireTime, token); + Map result = accessTokenService.createToken(loginUser, userId, expireTime, token); return returnDataList(result); } @@ -106,7 +106,7 @@ public class AccessTokenController extends BaseController { @RequestParam(value = "userId") int userId, @RequestParam(value = "expireTime") String expireTime) { logger.info("login user {}, generate token , userId : {} , token expire time : {}", loginUser, userId, expireTime); - Map result = accessTokenService.generateToken(userId, expireTime); + Map result = accessTokenService.generateToken(loginUser, userId, expireTime); return returnDataList(result); } @@ -185,7 +185,7 @@ public class AccessTokenController extends BaseController { logger.info("login user {}, update token , userId : {} , token expire time : {} , token : {}", loginUser.getUserName(), userId, expireTime, token); - Map result = accessTokenService.updateToken(id, userId, expireTime, token); + Map result = accessTokenService.updateToken(loginUser, id, userId, expireTime, token); return returnDataList(result); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java index 195b97e0bf..07859f7006 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java @@ -47,6 +47,7 @@ import org.apache.dolphinscheduler.api.service.ResourcesService; import org.apache.dolphinscheduler.api.service.UdfFuncService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.ProgramType; import org.apache.dolphinscheduler.common.enums.ResourceType; import org.apache.dolphinscheduler.common.enums.UdfType; import org.apache.dolphinscheduler.common.utils.ParameterUtils; @@ -95,6 +96,13 @@ public class ResourcesController extends BaseController { /** * + * @param loginUser login user + * @param type type + * @param alias alias + * @param description description + * @param pid parent id + * @param currentDir current directory + * @return create result code */ @ApiOperation(value = "createDirctory", notes = "CREATE_RESOURCE_NOTES") @ApiImplicitParams({ @@ -159,6 +167,7 @@ public class ResourcesController extends BaseController { * @param resourceId resource id * @param type resource type * @param description description + * @param file resource file * @return update result code */ @ApiOperation(value = "updateResource", notes = "UPDATE_RESOURCE_NOTES") @@ -166,7 +175,8 @@ public class ResourcesController extends BaseController { @ApiImplicitParam(name = "id", value = "RESOURCE_ID", required = true, dataType = "Int", example = "100"), @ApiImplicitParam(name = "type", value = "RESOURCE_TYPE", required = true, dataType = "ResourceType"), @ApiImplicitParam(name = "name", value = "RESOURCE_NAME", required = true, dataType = "String"), - @ApiImplicitParam(name = "description", value = "RESOURCE_DESC", dataType = "String") + @ApiImplicitParam(name = "description", value = "RESOURCE_DESC", dataType = "String"), + @ApiImplicitParam(name = "file", value = "RESOURCE_FILE", required = true, dataType = "MultipartFile") }) @PostMapping(value = "/update") @ApiException(UPDATE_RESOURCE_ERROR) @@ -174,10 +184,11 @@ public class ResourcesController extends BaseController { @RequestParam(value = "id") int resourceId, @RequestParam(value = "type") ResourceType type, @RequestParam(value = "name") String alias, - @RequestParam(value = "description", required = false) String description) { - logger.info("login user {}, update resource, type: {}, resource alias: {}, desc: {}", - loginUser.getUserName(), type, alias, description); - return resourceService.updateResource(loginUser, resourceId, alias, description, type); + @RequestParam(value = "description", required = false) String description, + @RequestParam(value = "file" ,required = false) MultipartFile file) { + logger.info("login user {}, update resource, type: {}, resource alias: {}, desc: {}, file: {}", + loginUser.getUserName(), type, alias, description, file); + return resourceService.updateResource(loginUser, resourceId, alias, description, type, file); } /** @@ -299,7 +310,7 @@ public class ResourcesController extends BaseController { * @param type resource type * @return resource list */ - @ApiOperation(value = "queryResourceJarList", notes = "QUERY_RESOURCE_LIST_NOTES") + @ApiOperation(value = "queryResourceByProgramType", notes = "QUERY_RESOURCE_LIST_NOTES") @ApiImplicitParams({ @ApiImplicitParam(name = "type", value = "RESOURCE_TYPE", required = true, dataType = "ResourceType") }) @@ -307,10 +318,14 @@ public class ResourcesController extends BaseController { @ResponseStatus(HttpStatus.OK) @ApiException(QUERY_RESOURCES_LIST_ERROR) public Result queryResourceJarList(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @RequestParam(value = "type") ResourceType type + @RequestParam(value = "type") ResourceType type, + @RequestParam(value = "programType",required = false) ProgramType programType ) { - logger.info("query resource list, login user:{}, resource type:{}", loginUser.getUserName(), type.toString()); - Map result = resourceService.queryResourceJarList(loginUser, type); + String programTypeName = programType == null ? "" : programType.name(); + String userName = loginUser.getUserName(); + userName = userName.replaceAll("[\n|\r|\t]", "_"); + logger.info("query resource list, login user:{}, resource type:{}, program type:{}", userName,programTypeName); + Map result = resourceService.queryResourceByProgramType(loginUser, type,programType); return returnDataList(result); } @@ -594,7 +609,7 @@ public class ResourcesController extends BaseController { @GetMapping(value = "/udf-func/list-paging") @ResponseStatus(HttpStatus.OK) @ApiException(QUERY_UDF_FUNCTION_LIST_PAGING_ERROR) - public Result queryUdfFuncList(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + public Result queryUdfFuncListPaging(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam("pageNo") Integer pageNo, @RequestParam(value = "searchVal", required = false) String searchVal, @RequestParam("pageSize") Integer pageSize @@ -611,23 +626,25 @@ public class ResourcesController extends BaseController { } /** - * query resource list by type + * query udf func list by type * * @param loginUser login user * @param type resource type * @return resource list */ - @ApiOperation(value = "queryResourceList", notes = "QUERY_RESOURCE_LIST_NOTES") + @ApiOperation(value = "queryUdfFuncList", notes = "QUERY_UDF_FUNC_LIST_NOTES") @ApiImplicitParams({ @ApiImplicitParam(name = "type", value = "UDF_TYPE", required = true, dataType = "UdfType") }) @GetMapping(value = "/udf-func/list") @ResponseStatus(HttpStatus.OK) @ApiException(QUERY_DATASOURCE_BY_TYPE_ERROR) - public Result queryResourceList(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + public Result queryUdfFuncList(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam("type") UdfType type) { - logger.info("query datasource list, user:{}, type:{}", loginUser.getUserName(), type); - Map result = udfFuncService.queryResourceList(loginUser, type.ordinal()); + String userName = loginUser.getUserName(); + userName = userName.replaceAll("[\n|\r|\t]", "_"); + logger.info("query udf func list, user:{}, type:{}", userName, type); + Map result = udfFuncService.queryUdfFuncList(loginUser, type.ordinal()); return returnDataList(result); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/UsersController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/UsersController.java index 8d6f9fc820..9b16265e32 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/UsersController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/UsersController.java @@ -166,7 +166,7 @@ public class UsersController extends BaseController { @RequestParam(value = "state", required = false) int state) throws Exception { logger.info("login user {}, updateProcessInstance user, userName: {}, email: {}, tenantId: {}, userPassword: {}, phone: {}, user queue: {}, state: {}", loginUser.getUserName(), userName, email, tenantId, Constants.PASSWORD_DEFAULT, phone, queue, state); - Map result = usersService.updateUser(id, userName, userPassword, email, tenantId, phone, queue, state); + Map result = usersService.updateUser(loginUser, id, userName, userPassword, email, tenantId, phone, queue, state); return returnDataList(result); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AccessTokenService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AccessTokenService.java index 98eef47090..b1c320566f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AccessTokenService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AccessTokenService.java @@ -44,7 +44,8 @@ public interface AccessTokenService { * @param token token string * @return create result code */ - Map createToken(int userId, String expireTime, String token); + Map createToken(User loginUser, int userId, String expireTime, String token); + /** * generate token @@ -53,7 +54,7 @@ public interface AccessTokenService { * @param expireTime token expire time * @return token string */ - Map generateToken(int userId, String expireTime); + Map generateToken(User loginUser, int userId, String expireTime); /** * delete access token @@ -73,5 +74,5 @@ public interface AccessTokenService { * @param token token string * @return update result code */ - Map updateToken(int id, int userId, String expireTime, String token); + Map updateToken(User loginUser, int id, int userId, String expireTime, String token); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseService.java index 4b094ea494..e9b25250a8 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseService.java @@ -95,6 +95,25 @@ public class BaseService { } + /** + * check + * + * @param result result + * @param bool bool + * @param userNoOperationPerm status + * @return check result + */ + protected boolean check(Map result, boolean bool, Status userNoOperationPerm) { + //only admin can operate + if (bool) { + result.put(Constants.STATUS, userNoOperationPerm); + result.put(Constants.MSG, userNoOperationPerm.getMsg()); + return true; + } + return false; + } + + /** * get cookie info by name * diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java index bd7598979d..dffc55edfe 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java @@ -29,6 +29,7 @@ import org.apache.dolphinscheduler.api.exceptions.ServiceException; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.ProgramType; import org.apache.dolphinscheduler.common.enums.ResourceType; import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.dao.entity.*; @@ -87,7 +88,7 @@ public class ResourcesService extends BaseService { * @param currentDir current directory * @return create directory result */ - @Transactional(rollbackFor = RuntimeException.class) + @Transactional(rollbackFor = Exception.class) public Result createDirectory(User loginUser, String name, String description, @@ -101,8 +102,11 @@ public class ResourcesService extends BaseService { putMsg(result, Status.HDFS_NOT_STARTUP); return result; } - String fullName = "/".equals(currentDir) ? String.format("%s%s",currentDir,name):String.format("%s/%s",currentDir,name); - + String fullName = currentDir.equals("/") ? String.format("%s%s",currentDir,name):String.format("%s/%s",currentDir,name); + result = verifyResourceName(fullName,type,loginUser); + if (!result.getCode().equals(Status.SUCCESS.getCode())) { + return result; + } if (pid != -1) { Resource parentResource = resourcesMapper.selectById(pid); @@ -165,7 +169,7 @@ public class ResourcesService extends BaseService { * @param currentDir current directory * @return create result code */ - @Transactional(rollbackFor = RuntimeException.class) + @Transactional(rollbackFor = Exception.class) public Result createResource(User loginUser, String name, String desc, @@ -230,7 +234,7 @@ public class ResourcesService extends BaseService { } // check resoure name exists - String fullName = "/".equals(currentDir) ? String.format("%s%s",currentDir,name):String.format("%s/%s",currentDir,name); + String fullName = currentDir.equals("/") ? String.format("%s%s",currentDir,name):String.format("%s/%s",currentDir,name); if (checkResourceExists(fullName, 0, type.ordinal())) { logger.error("resource {} has exist, can't recreate", name); putMsg(result, Status.RESOURCE_EXIST); @@ -288,14 +292,16 @@ public class ResourcesService extends BaseService { * @param name name * @param desc description * @param type resource type + * @param file resource file * @return update result code */ - @Transactional(rollbackFor = RuntimeException.class) + @Transactional(rollbackFor = Exception.class) public Result updateResource(User loginUser, int resourceId, String name, String desc, - ResourceType type) { + ResourceType type, + MultipartFile file) { Result result = new Result(); // if resource upload startup @@ -315,7 +321,7 @@ public class ResourcesService extends BaseService { return result; } - if (name.equals(resource.getAlias()) && desc.equals(resource.getDescription())) { + if (file == null && name.equals(resource.getAlias()) && desc.equals(resource.getDescription())) { putMsg(result, Status.SUCCESS); return result; } @@ -331,6 +337,42 @@ public class ResourcesService extends BaseService { return result; } + if (file != null) { + + // file is empty + if (file.isEmpty()) { + logger.error("file is empty: {}", file.getOriginalFilename()); + putMsg(result, Status.RESOURCE_FILE_IS_EMPTY); + return result; + } + + // file suffix + String fileSuffix = FileUtils.suffix(file.getOriginalFilename()); + String nameSuffix = FileUtils.suffix(name); + + // determine file suffix + if (!(StringUtils.isNotEmpty(fileSuffix) && fileSuffix.equalsIgnoreCase(nameSuffix))) { + /** + * rename file suffix and original suffix must be consistent + */ + logger.error("rename file suffix and original suffix must be consistent: {}", file.getOriginalFilename()); + putMsg(result, Status.RESOURCE_SUFFIX_FORBID_CHANGE); + return result; + } + + //If resource type is UDF, only jar packages are allowed to be uploaded, and the suffix must be .jar + if (Constants.UDF.equals(type.name()) && !JAR.equalsIgnoreCase(FileUtils.suffix(originFullName))) { + logger.error(Status.UDF_RESOURCE_SUFFIX_NOT_JAR.getMsg()); + putMsg(result, Status.UDF_RESOURCE_SUFFIX_NOT_JAR); + return result; + } + if (file.getSize() > Constants.MAX_FILE_SIZE) { + logger.error("file size is too large: {}", file.getOriginalFilename()); + putMsg(result, Status.RESOURCE_SIZE_EXCEED_LIMIT); + return result; + } + } + // query tenant by user id String tenantCode = getTenantCode(resource.getUserId(),result); if (StringUtils.isEmpty(tenantCode)){ @@ -380,31 +422,61 @@ public class ResourcesService extends BaseService { } // updateResource data - List childrenResource = listAllChildren(resource,false); Date now = new Date(); resource.setAlias(name); resource.setFullName(fullName); resource.setDescription(desc); resource.setUpdateTime(now); + if (file != null) { + resource.setFileName(file.getOriginalFilename()); + resource.setSize(file.getSize()); + } try { resourcesMapper.updateById(resource); - if (resource.isDirectory() && CollectionUtils.isNotEmpty(childrenResource)) { - String matcherFullName = Matcher.quoteReplacement(fullName); - List childResourceList = new ArrayList<>(); - List resourceList = resourcesMapper.listResourceByIds(childrenResource.toArray(new Integer[childrenResource.size()])); - childResourceList = resourceList.stream().map(t -> { - t.setFullName(t.getFullName().replaceFirst(originFullName, matcherFullName)); - t.setUpdateTime(now); - return t; - }).collect(Collectors.toList()); - resourcesMapper.batchUpdateResource(childResourceList); + if (resource.isDirectory()) { + List childrenResource = listAllChildren(resource,false); + if (CollectionUtils.isNotEmpty(childrenResource)) { + String matcherFullName = Matcher.quoteReplacement(fullName); + List childResourceList = new ArrayList<>(); + Integer[] childResIdArray = childrenResource.toArray(new Integer[childrenResource.size()]); + List resourceList = resourcesMapper.listResourceByIds(childResIdArray); + childResourceList = resourceList.stream().map(t -> { + t.setFullName(t.getFullName().replaceFirst(originFullName, matcherFullName)); + t.setUpdateTime(now); + return t; + }).collect(Collectors.toList()); + resourcesMapper.batchUpdateResource(childResourceList); + + if (ResourceType.UDF.equals(resource.getType())) { + List udfFuncs = udfFunctionMapper.listUdfByResourceId(childResIdArray); + if (CollectionUtils.isNotEmpty(udfFuncs)) { + udfFuncs = udfFuncs.stream().map(t -> { + t.setResourceName(t.getResourceName().replaceFirst(originFullName, matcherFullName)); + t.setUpdateTime(now); + return t; + }).collect(Collectors.toList()); + udfFunctionMapper.batchUpdateUdfFunc(udfFuncs); + } + } + } + } else if (ResourceType.UDF.equals(resource.getType())) { + List udfFuncs = udfFunctionMapper.listUdfByResourceId(new Integer[]{resourceId}); + if (CollectionUtils.isNotEmpty(udfFuncs)) { + udfFuncs = udfFuncs.stream().map(t -> { + t.setResourceName(fullName); + t.setUpdateTime(now); + return t; + }).collect(Collectors.toList()); + udfFunctionMapper.batchUpdateUdfFunc(udfFuncs); + } + } putMsg(result, Status.SUCCESS); Map dataMap = new BeanMap(resource); - Map resultMap = new HashMap<>(); + Map resultMap = new HashMap<>(5); for (Map.Entry entry: dataMap.entrySet()) { if (!Constants.CLASS.equalsIgnoreCase(entry.getKey().toString())) { resultMap.put(entry.getKey().toString(), entry.getValue()); @@ -415,11 +487,31 @@ public class ResourcesService extends BaseService { logger.error(Status.UPDATE_RESOURCE_ERROR.getMsg(), e); throw new ServiceException(Status.UPDATE_RESOURCE_ERROR); } + // if name unchanged, return directly without moving on HDFS - if (originResourceName.equals(name)) { + if (originResourceName.equals(name) && file == null) { + return result; + } + + if (file != null) { + // fail upload + if (!upload(loginUser, fullName, file, type)) { + logger.error("upload resource: {} file: {} failed.", name, file.getOriginalFilename()); + putMsg(result, Status.HDFS_OPERATION_ERROR); + throw new RuntimeException(String.format("upload resource: %s file: %s failed.", name, file.getOriginalFilename())); + } + if (!fullName.equals(originFullName)) { + try { + HadoopUtils.getInstance().delete(originHdfsFileName,false); + } catch (IOException e) { + logger.error(e.getMessage(),e); + throw new RuntimeException(String.format("delete resource: %s failed.", originFullName)); + } + } return result; } + // get the path of dest file in hdfs String destHdfsFileName = HadoopUtils.getHdfsFileName(resource.getType(),tenantCode,fullName); @@ -449,7 +541,7 @@ public class ResourcesService extends BaseService { */ public Map queryResourceListPaging(User loginUser, int direcotryId, ResourceType type, String searchVal, Integer pageNo, Integer pageSize) { - HashMap result = new HashMap<>(); + HashMap result = new HashMap<>(5); Page page = new Page(pageNo, pageSize); int userId = loginUser.getId(); if (isAdmin(loginUser)) { @@ -550,7 +642,7 @@ public class ResourcesService extends BaseService { */ public Map queryResourceList(User loginUser, ResourceType type) { - Map result = new HashMap<>(); + Map result = new HashMap<>(5); int userId = loginUser.getId(); if(isAdmin(loginUser)){ @@ -565,21 +657,33 @@ public class ResourcesService extends BaseService { } /** - * query resource list + * query resource list by program type * * @param loginUser login user * @param type resource type * @return resource list */ - public Map queryResourceJarList(User loginUser, ResourceType type) { + public Map queryResourceByProgramType(User loginUser, ResourceType type, ProgramType programType) { - Map result = new HashMap<>(); + Map result = new HashMap<>(5); + String suffix = ".jar"; int userId = loginUser.getId(); if(isAdmin(loginUser)){ userId = 0; } + if (programType != null) { + switch (programType) { + case JAVA: + break; + case SCALA: + break; + case PYTHON: + suffix = ".py"; + break; + } + } List allResourceList = resourcesMapper.queryResourceListAuthored(userId, type.ordinal(),0); - List resources = new ResourceFilter(".jar",new ArrayList<>(allResourceList)).filter(); + List resources = new ResourceFilter(suffix,new ArrayList<>(allResourceList)).filter(); Visitor resourceTreeVisitor = new ResourceTreeVisitor(resources); result.put(Constants.DATA_LIST, resourceTreeVisitor.visit().getChildren()); putMsg(result,Status.SUCCESS); @@ -829,7 +933,7 @@ public class ResourcesService extends BaseService { * @param content content * @return create result code */ - @Transactional(rollbackFor = RuntimeException.class) + @Transactional(rollbackFor = Exception.class) public Result onlineCreateResource(User loginUser, ResourceType type, String fileName, String fileSuffix, String desc, String content,int pid,String currentDirectory) { Result result = new Result(); // if resource upload startup @@ -852,12 +956,25 @@ public class ResourcesService extends BaseService { } String name = fileName.trim() + "." + nameSuffix; - String fullName = "/".equals(currentDirectory) ? String.format("%s%s",currentDirectory,name):String.format("%s/%s",currentDirectory,name); + String fullName = currentDirectory.equals("/") ? String.format("%s%s",currentDirectory,name):String.format("%s/%s",currentDirectory,name); result = verifyResourceName(fullName,type,loginUser); if (!result.getCode().equals(Status.SUCCESS.getCode())) { return result; } + if (pid != -1) { + Resource parentResource = resourcesMapper.selectById(pid); + + if (parentResource == null) { + putMsg(result, Status.PARENT_RESOURCE_NOT_EXIST); + return result; + } + + if (!hasPerm(loginUser, parentResource.getUserId())) { + putMsg(result, Status.USER_NO_OPERATION_PERM); + return result; + } + } // save data Date now = new Date(); @@ -891,7 +1008,7 @@ public class ResourcesService extends BaseService { * @param content content * @return update result cod */ - @Transactional(rollbackFor = RuntimeException.class) + @Transactional(rollbackFor = Exception.class) public Result updateResourceContent(int resourceId, String content) { Result result = new Result(); @@ -1096,7 +1213,7 @@ public class ResourcesService extends BaseService { * @return unauthorized result code */ public Map unauthorizedUDFFunction(User loginUser, Integer userId) { - Map result = new HashMap<>(); + Map result = new HashMap<>(5); //only admin can operate if (checkAdmin(loginUser, result)) { return result; @@ -1148,7 +1265,7 @@ public class ResourcesService extends BaseService { * @return authorized result */ public Map authorizedFile(User loginUser, Integer userId) { - Map result = new HashMap<>(); + Map result = new HashMap<>(5); if (checkAdmin(loginUser, result)){ return result; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java index 04f641f279..cd962fdc70 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java @@ -148,7 +148,7 @@ public class UdfFuncService extends BaseService{ */ public Map queryUdfFuncDetail(int id) { - Map result = new HashMap<>(); + Map result = new HashMap<>(5); UdfFunc udfFunc = udfFuncMapper.selectById(id); if (udfFunc == null) { putMsg(result, Status.RESOURCE_NOT_EXIST); @@ -244,7 +244,7 @@ public class UdfFuncService extends BaseService{ * @return udf function list page */ public Map queryUdfFuncListPaging(User loginUser, String searchVal, Integer pageNo, Integer pageSize) { - Map result = new HashMap<>(); + Map result = new HashMap<>(5); PageInfo pageInfo = new PageInfo(pageNo, pageSize); @@ -276,15 +276,19 @@ public class UdfFuncService extends BaseService{ } /** - * query data resource by type + * query udf list * * @param loginUser login user - * @param type resource type - * @return resource list + * @param type udf type + * @return udf func list */ - public Map queryResourceList(User loginUser, Integer type) { - Map result = new HashMap<>(); - List udfFuncList = udfFuncMapper.getUdfFuncByType(loginUser.getId(), type); + public Map queryUdfFuncList(User loginUser, Integer type) { + Map result = new HashMap<>(5); + int userId = loginUser.getId(); + if (isAdmin(loginUser)) { + userId = 0; + } + List udfFuncList = udfFuncMapper.getUdfFuncByType(userId, type); result.put(Constants.DATA_LIST, udfFuncList); putMsg(result, Status.SUCCESS); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java index 89038ad09f..e47eb42cb5 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java @@ -103,7 +103,7 @@ public class UsersService extends BaseService { String queue, int state) throws Exception { - Map result = new HashMap<>(); + Map result = new HashMap<>(5); //check all user params String msg = this.checkUserParams(userName, userPassword, email, phone); @@ -231,7 +231,7 @@ public class UsersService extends BaseService { * @return user list page */ public Map queryUserList(User loginUser, String searchVal, Integer pageNo, Integer pageSize) { - Map result = new HashMap<>(); + Map result = new HashMap<>(5); if (check(result, !isAdmin(loginUser), Status.USER_NO_OPERATION_PERM)) { return result; @@ -253,6 +253,8 @@ public class UsersService extends BaseService { /** * updateProcessInstance user * + * + * @param loginUser * @param userId user id * @param userName user name * @param userPassword user password @@ -263,7 +265,7 @@ public class UsersService extends BaseService { * @return update result code * @throws Exception exception */ - public Map updateUser(int userId, + public Map updateUser(User loginUser, int userId, String userName, String userPassword, String email, @@ -271,16 +273,17 @@ public class UsersService extends BaseService { String phone, String queue, int state) throws Exception { - Map result = new HashMap<>(); + Map result = new HashMap<>(5); result.put(Constants.STATUS, false); + if (check(result, !hasPerm(loginUser, userId), Status.USER_NO_OPERATION_PERM)) { + return result; + } User user = userMapper.selectById(userId); - if (user == null) { putMsg(result, Status.USER_NOT_EXIST, userId); return result; } - if (StringUtils.isNotEmpty(userName)) { if (!CheckUtils.checkUserName(userName)){ @@ -394,7 +397,7 @@ public class UsersService extends BaseService { * @throws Exception exception when operate hdfs */ public Map deleteUserById(User loginUser, int id) throws Exception { - Map result = new HashMap<>(); + Map result = new HashMap<>(5); //only admin can operate if (!isAdmin(loginUser)) { putMsg(result, Status.USER_NO_OPERATION_PERM, id); @@ -434,7 +437,7 @@ public class UsersService extends BaseService { */ @Transactional(rollbackFor = RuntimeException.class) public Map grantProject(User loginUser, int userId, String projectIds) { - Map result = new HashMap<>(); + Map result = new HashMap<>(5); result.put(Constants.STATUS, false); //only admin can operate @@ -484,7 +487,7 @@ public class UsersService extends BaseService { */ @Transactional(rollbackFor = RuntimeException.class) public Map grantResources(User loginUser, int userId, String resourceIds) { - Map result = new HashMap<>(); + Map result = new HashMap<>(5); //only admin can operate if (check(result, !isAdmin(loginUser), Status.USER_NO_OPERATION_PERM)) { return result; @@ -581,7 +584,7 @@ public class UsersService extends BaseService { */ @Transactional(rollbackFor = RuntimeException.class) public Map grantUDFFunction(User loginUser, int userId, String udfIds) { - Map result = new HashMap<>(); + Map result = new HashMap<>(5); //only admin can operate if (check(result, !isAdmin(loginUser), Status.USER_NO_OPERATION_PERM)) { @@ -628,7 +631,7 @@ public class UsersService extends BaseService { */ @Transactional(rollbackFor = RuntimeException.class) public Map grantDataSource(User loginUser, int userId, String datasourceIds) { - Map result = new HashMap<>(); + Map result = new HashMap<>(5); result.put(Constants.STATUS, false); //only admin can operate @@ -708,7 +711,7 @@ public class UsersService extends BaseService { * @return user list */ public Map queryAllGeneralUsers(User loginUser) { - Map result = new HashMap<>(); + Map result = new HashMap<>(5); //only admin can operate if (check(result, !isAdmin(loginUser), Status.USER_NO_OPERATION_PERM)) { return result; @@ -729,7 +732,7 @@ public class UsersService extends BaseService { * @return user list */ public Map queryUserList(User loginUser) { - Map result = new HashMap<>(); + Map result = new HashMap<>(5); //only admin can operate if (check(result, !isAdmin(loginUser), Status.USER_NO_OPERATION_PERM)) { return result; @@ -773,7 +776,7 @@ public class UsersService extends BaseService { */ public Map unauthorizedUser(User loginUser, Integer alertgroupId) { - Map result = new HashMap<>(); + Map result = new HashMap<>(5); //only admin can operate if (check(result, !isAdmin(loginUser), Status.USER_NO_OPERATION_PERM)) { return result; @@ -809,7 +812,7 @@ public class UsersService extends BaseService { * @return authorized result code */ public Map authorizedUser(User loginUser, Integer alertgroupId) { - Map result = new HashMap<>(); + Map result = new HashMap<>(5); //only admin can operate if (check(result, !isAdmin(loginUser), Status.USER_NO_OPERATION_PERM)) { return result; @@ -821,24 +824,6 @@ public class UsersService extends BaseService { return result; } - /** - * check - * - * @param result result - * @param bool bool - * @param userNoOperationPerm status - * @return check result - */ - private boolean check(Map result, boolean bool, Status userNoOperationPerm) { - //only admin can operate - if (bool) { - result.put(Constants.STATUS, userNoOperationPerm); - result.put(Constants.MSG, userNoOperationPerm.getMsg()); - return true; - } - return false; - } - /** * @param tenantId tenant id * @return true if tenant exists, otherwise return false diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AccessTokenServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AccessTokenServiceImpl.java index 54151d902f..da85621041 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AccessTokenServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AccessTokenServiceImpl.java @@ -86,9 +86,14 @@ public class AccessTokenServiceImpl extends BaseService implements AccessTokenSe * @param token token string * @return create result code */ - public Map createToken(int userId, String expireTime, String token) { + public Map createToken(User loginUser, int userId, String expireTime, String token) { Map result = new HashMap<>(5); + if (!hasPerm(loginUser,userId)){ + putMsg(result, Status.USER_NO_OPERATION_PERM); + return result; + } + if (userId <= 0) { throw new IllegalArgumentException("User id should not less than or equals to 0."); } @@ -118,8 +123,12 @@ public class AccessTokenServiceImpl extends BaseService implements AccessTokenSe * @param expireTime token expire time * @return token string */ - public Map generateToken(int userId, String expireTime) { + public Map generateToken(User loginUser, int userId, String expireTime) { Map result = new HashMap<>(5); + if (!hasPerm(loginUser,userId)){ + putMsg(result, Status.USER_NO_OPERATION_PERM); + return result; + } String token = EncryptionUtils.getMd5(userId + expireTime + System.currentTimeMillis()); result.put(Constants.DATA_LIST, token); putMsg(result, Status.SUCCESS); @@ -144,8 +153,8 @@ public class AccessTokenServiceImpl extends BaseService implements AccessTokenSe return result; } - if (loginUser.getId() != accessToken.getUserId() && - loginUser.getUserType() != UserType.ADMIN_USER) { + + if (!hasPerm(loginUser,accessToken.getUserId())){ putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } @@ -164,9 +173,12 @@ public class AccessTokenServiceImpl extends BaseService implements AccessTokenSe * @param token token string * @return update result code */ - public Map updateToken(int id, int userId, String expireTime, String token) { + public Map updateToken(User loginUser, int id, int userId, String expireTime, String token) { Map result = new HashMap<>(5); - + if (!hasPerm(loginUser,userId)){ + putMsg(result, Status.USER_NO_OPERATION_PERM); + return result; + } AccessToken accessToken = accessTokenMapper.selectById(id); if (accessToken == null) { logger.error("access token not exist, access token id {}", id); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java index c71f2980f5..1574e7f0e7 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java @@ -130,7 +130,6 @@ public class LoggerServiceImpl implements LoggerService { logClient.getLogBytes(host, Constants.RPC_PORT, taskInstance.getLogPath())); } - /** * get host * diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java index b7ab24bb13..e86ea8301e 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java @@ -49,6 +49,7 @@ import org.apache.dolphinscheduler.common.model.TaskNodeRelation; import org.apache.dolphinscheduler.common.process.ProcessDag; import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.process.ResourceInfo; +import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.DateUtils; @@ -166,7 +167,7 @@ public class ProcessDefinitionServiceImpl extends BaseService implements String processDefinitionJson, String desc, String locations, - String connects) throws JsonProcessingException { + String connects) { Map result = new HashMap<>(); Project project = projectMapper.queryByName(projectName); @@ -229,23 +230,40 @@ public class ProcessDefinitionServiceImpl extends BaseService implements /** * get resource ids - * * @param processData process data * @return resource ids */ private String getResourceIds(ProcessData processData) { - return Optional.ofNullable(processData.getTasks()) - .orElse(Collections.emptyList()) - .stream() - .map(taskNode -> TaskParametersUtils.getParameters(taskNode.getType(), taskNode.getParams())) - .filter(Objects::nonNull) - .flatMap(parameters -> parameters.getResourceFilesList().stream()) - .map(ResourceInfo::getId) - .distinct() - .map(Objects::toString) - .collect(Collectors.joining(",")); - } + List tasks = processData.getTasks(); + Set resourceIds = new HashSet<>(); + StringBuilder sb = new StringBuilder(); + if (CollectionUtils.isEmpty(tasks)) { + return sb.toString(); + } + for (TaskNode taskNode : tasks) { + String taskParameter = taskNode.getParams(); + AbstractParameters params = TaskParametersUtils.getParameters(taskNode.getType(), taskParameter); + if (params == null) { + continue; + } + if (CollectionUtils.isNotEmpty(params.getResourceFilesList())) { + Set tempSet = params.getResourceFilesList(). + stream() + .filter(t -> t.getId() != 0) + .map(ResourceInfo::getId) + .collect(Collectors.toSet()); + resourceIds.addAll(tempSet); + } + } + for (int i : resourceIds) { + if (sb.length() > 0) { + sb.append(","); + } + sb.append(i); + } + return sb.toString(); + } /** * query process definition list * @@ -255,7 +273,7 @@ public class ProcessDefinitionServiceImpl extends BaseService implements */ public Map queryProcessDefinitionList(User loginUser, String projectName) { - HashMap result = new HashMap<>(); + HashMap result = new HashMap<>(5); Project project = projectMapper.queryByName(projectName); Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); @@ -348,10 +366,15 @@ public class ProcessDefinitionServiceImpl extends BaseService implements * @param connects connects for nodes * @return update result code */ - public Map updateProcessDefinition(User loginUser, String projectName, int id, String name, - String processDefinitionJson, String desc, - String locations, String connects) { - Map result = new HashMap<>(); + public Map updateProcessDefinition(User loginUser, + String projectName, + int id, + String name, + String processDefinitionJson, + String desc, + String locations, + String connects) { + Map result = new HashMap<>(5); Project project = projectMapper.queryByName(projectName); Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); @@ -462,7 +485,7 @@ public class ProcessDefinitionServiceImpl extends BaseService implements @Transactional(rollbackFor = RuntimeException.class) public Map deleteProcessDefinitionById(User loginUser, String projectName, Integer processDefinitionId) { - Map result = new HashMap<>(); + Map result = new HashMap<>(5); Project project = projectMapper.queryByName(projectName); Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); @@ -674,6 +697,17 @@ public class ProcessDefinitionServiceImpl extends BaseService implements } } + /** + * get export process metadata string + * @param processDefinitionId process definition id + * @param processDefinition process definition + * @return export process metadata string + */ + public String exportProcessMetaDataStr(Integer processDefinitionId, ProcessDefinition processDefinition) { + //create workflow json file + return JSONUtils.toJsonString(exportProcessMetaData(processDefinitionId, processDefinition)); + } + /** * get export process metadata string * @@ -758,7 +792,7 @@ public class ProcessDefinitionServiceImpl extends BaseService implements */ @Transactional(rollbackFor = RuntimeException.class) public Map importProcessDefinition(User loginUser, MultipartFile file, String currentProjectName) { - Map result = new HashMap<>(); + Map result = new HashMap<>(5); String processMetaJson = FileUtils.file2String(file); List processMetaList = JSONUtils.toList(processMetaJson, ProcessMeta.class); @@ -853,7 +887,7 @@ public class ProcessDefinitionServiceImpl extends BaseService implements processMeta.getProcessDefinitionLocations(), processMeta.getProcessDefinitionConnects()); putMsg(result, Status.SUCCESS); - } catch (JsonProcessingException e) { + } catch (Exception e) { logger.error("import process meta json data: {}", e.getMessage(), e); putMsg(result, Status.IMPORT_PROCESS_DEFINE_ERROR); } @@ -927,7 +961,7 @@ public class ProcessDefinitionServiceImpl extends BaseService implements } //recursive sub-process parameter correction map key for old process id value for new process id - Map subProcessIdMap = new HashMap<>(); + Map subProcessIdMap = new HashMap<>(20); List subProcessList = StreamUtils.asStream(jsonArray.elements()) .filter(elem -> checkTaskHasSubProcess(JSONUtils.parseObject(elem.toString()).path("type").asText())) @@ -1215,7 +1249,7 @@ public class ProcessDefinitionServiceImpl extends BaseService implements */ public Map queryProcessDefinitionAllByProjectId(Integer projectId) { - HashMap result = new HashMap<>(); + HashMap result = new HashMap<>(5); List resourceList = processDefineMapper.queryAllDefinitionList(projectId); result.put(Constants.DATA_LIST, resourceList); @@ -1425,7 +1459,7 @@ public class ProcessDefinitionServiceImpl extends BaseService implements Integer processId, Project targetProject) throws JsonProcessingException { - Map result = new HashMap<>(); + Map result = new HashMap<>(5); ProcessDefinition processDefinition = processDefineMapper.selectById(processId); if (processDefinition == null) { @@ -1444,6 +1478,41 @@ public class ProcessDefinitionServiceImpl extends BaseService implements } } + /** + * copy process definition + * + * @param loginUser login user + * @param projectName project name + * @param processId process definition id + * @return copy result code + */ + public Map copyProcessDefinition(User loginUser, String projectName, Integer processId) { + + Map result = new HashMap<>(5); + Project project = projectMapper.queryByName(projectName); + + Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); + Status resultStatus = (Status) checkResult.get(Constants.STATUS); + if (resultStatus != Status.SUCCESS) { + return checkResult; + } + + ProcessDefinition processDefinition = processDefineMapper.selectById(processId); + if (processDefinition == null) { + putMsg(result, Status.PROCESS_DEFINE_NOT_EXIST, processId); + return result; + } else { + return createProcessDefinition( + loginUser, + projectName, + processDefinition.getName() + "_copy_" + System.currentTimeMillis(), + processDefinition.getProcessDefinitionJson(), + processDefinition.getDescription(), + processDefinition.getLocations(), + processDefinition.getConnects()); + } + } + /** * batch copy process definition * diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java index b24298da41..ecb9bae98c 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java @@ -283,6 +283,26 @@ public class TenantServiceImpl extends BaseService implements TenantService { return processInstanceMapper.queryByTenantIdAndStatus(tenant.getId(), Constants.NOT_TERMINATED_STATES); } + /** + * query tenant list + * + * @param tenantCode tenant code + * @return tenant list + */ + public Map queryTenantList(String tenantCode) { + + Map result = new HashMap<>(5); + + List resourceList = tenantMapper.queryByTenantCode(tenantCode); + if (CollectionUtils.isNotEmpty(resourceList)) { + result.put(Constants.DATA_LIST, resourceList); + putMsg(result, Status.SUCCESS); + } else { + putMsg(result, Status.TENANT_NOT_EXIST); + } + return result; + } + /** * query tenant list * diff --git a/dolphinscheduler-api/src/main/resources/logback-api.xml b/dolphinscheduler-api/src/main/resources/logback-api.xml index 2df90d8392..e5cb37afac 100644 --- a/dolphinscheduler-api/src/main/resources/logback-api.xml +++ b/dolphinscheduler-api/src/main/resources/logback-api.xml @@ -55,7 +55,6 @@ - diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AccessTokenServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AccessTokenServiceTest.java index f5543487ea..e10d7185f9 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AccessTokenServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AccessTokenServiceTest.java @@ -81,7 +81,7 @@ public class AccessTokenServiceTest { public void testCreateToken() { when(accessTokenMapper.insert(any(AccessToken.class))).thenReturn(2); - Map result = accessTokenService.createToken(1, getDate(), "AccessTokenServiceTest"); + Map result = accessTokenService.createToken(getLoginUser(), 1, getDate(), "AccessTokenServiceTest"); logger.info(result.toString()); Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); } @@ -89,7 +89,7 @@ public class AccessTokenServiceTest { @Test public void testGenerateToken() { - Map result = accessTokenService.generateToken(Integer.MAX_VALUE, getDate()); + Map result = accessTokenService.generateToken(getLoginUser(), Integer.MAX_VALUE,getDate()); logger.info(result.toString()); Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); String token = (String) result.get(Constants.DATA_LIST); @@ -121,16 +121,24 @@ public class AccessTokenServiceTest { public void testUpdateToken() { when(accessTokenMapper.selectById(1)).thenReturn(getEntity()); - Map result = accessTokenService.updateToken(1, Integer.MAX_VALUE, getDate(), "token"); + Map result = accessTokenService.updateToken(getLoginUser(), 1,Integer.MAX_VALUE,getDate(),"token"); logger.info(result.toString()); Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); // not exist - result = accessTokenService.updateToken(2, Integer.MAX_VALUE, getDate(), "token"); + result = accessTokenService.updateToken(getLoginUser(), 2,Integer.MAX_VALUE,getDate(),"token"); logger.info(result.toString()); Assert.assertEquals(Status.ACCESS_TOKEN_NOT_EXIST, result.get(Constants.STATUS)); } + + private User getLoginUser(){ + User loginUser = new User(); + loginUser.setId(1); + loginUser.setUserType(UserType.ADMIN_USER); + return loginUser; + } + /** * create entity */ diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java index eb71cc87e7..55d7b09173 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java @@ -1035,15 +1035,15 @@ public class ProcessDefinitionServiceTest { taskNode5.setType("SHELL"); ShellParameters shellParameters5 = new ShellParameters(); ResourceInfo resourceInfo5A = new ResourceInfo(); - resourceInfo5A.setId(0); + resourceInfo5A.setId(1); ResourceInfo resourceInfo5B = new ResourceInfo(); - resourceInfo5B.setId(1); + resourceInfo5B.setId(2); shellParameters5.setResourceList(Arrays.asList(resourceInfo5A, resourceInfo5B)); taskNode5.setParams(JSONUtils.toJsonString(shellParameters5)); input5.setTasks(Collections.singletonList(taskNode5)); String output5 = (String) testMethod.invoke(processDefinitionService, input5); assertThat(output5.split(",")).hasSize(2) - .containsExactlyInAnyOrder("0", "1"); + .containsExactlyInAnyOrder("1", "2"); // when resource id list is 0 1 1 2, then return 0,1,2 ProcessData input6 = new ProcessData(); @@ -1051,7 +1051,7 @@ public class ProcessDefinitionServiceTest { taskNode6.setType("SHELL"); ShellParameters shellParameters6 = new ShellParameters(); ResourceInfo resourceInfo6A = new ResourceInfo(); - resourceInfo6A.setId(0); + resourceInfo6A.setId(3); ResourceInfo resourceInfo6B = new ResourceInfo(); resourceInfo6B.setId(1); ResourceInfo resourceInfo6C = new ResourceInfo(); @@ -1065,7 +1065,7 @@ public class ProcessDefinitionServiceTest { String output6 = (String) testMethod.invoke(processDefinitionService, input6); assertThat(output6.split(",")).hasSize(3) - .containsExactlyInAnyOrder("0", "1", "2"); + .containsExactlyInAnyOrder("3", "1", "2"); } /** diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java index 407f6b587f..14cd394611 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java @@ -139,6 +139,10 @@ public class ResourcesServiceTest { Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(),result.getMsg()); //PARENT_RESOURCE_NOT_EXIST + user.setId(1); + user.setTenantId(1); + Mockito.when(userMapper.selectById(1)).thenReturn(getUser()); + Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); Mockito.when(resourcesMapper.selectById(Mockito.anyInt())).thenReturn(null); result = resourcesService.createDirectory(user,"directoryTest","directory test",ResourceType.FILE,1,"/"); @@ -159,19 +163,19 @@ public class ResourcesServiceTest { PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false); User user = new User(); //HDFS_NOT_STARTUP - Result result = resourcesService.updateResource(user,1,"ResourcesServiceTest","ResourcesServiceTest",ResourceType.FILE); + Result result = resourcesService.updateResource(user,1,"ResourcesServiceTest","ResourcesServiceTest",ResourceType.FILE,null); logger.info(result.toString()); Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(),result.getMsg()); //RESOURCE_NOT_EXIST Mockito.when(resourcesMapper.selectById(1)).thenReturn(getResource()); PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); - result = resourcesService.updateResource(user,0,"ResourcesServiceTest","ResourcesServiceTest",ResourceType.FILE); + result = resourcesService.updateResource(user,0,"ResourcesServiceTest","ResourcesServiceTest",ResourceType.FILE,null); logger.info(result.toString()); Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(),result.getMsg()); //USER_NO_OPERATION_PERM - result = resourcesService.updateResource(user,1,"ResourcesServiceTest","ResourcesServiceTest",ResourceType.FILE); + result = resourcesService.updateResource(user,1,"ResourcesServiceTest","ResourcesServiceTest",ResourceType.FILE,null); logger.info(result.toString()); Assert.assertEquals(Status.USER_NO_OPERATION_PERM.getMsg(),result.getMsg()); @@ -186,7 +190,7 @@ public class ResourcesServiceTest { } catch (IOException e) { logger.error(e.getMessage(),e); } - result = resourcesService.updateResource(user, 1, "ResourcesServiceTest1.jar", "ResourcesServiceTest", ResourceType.UDF); + result = resourcesService.updateResource(user, 1, "ResourcesServiceTest1.jar", "ResourcesServiceTest", ResourceType.UDF,null); Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(),result.getMsg()); //SUCCESS @@ -199,25 +203,25 @@ public class ResourcesServiceTest { logger.error(e.getMessage(),e); } - result = resourcesService.updateResource(user,1,"ResourcesServiceTest.jar","ResourcesServiceTest",ResourceType.FILE); + result = resourcesService.updateResource(user,1,"ResourcesServiceTest.jar","ResourcesServiceTest",ResourceType.FILE,null); logger.info(result.toString()); Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg()); //RESOURCE_EXIST Mockito.when(resourcesMapper.queryResourceList("/ResourcesServiceTest1.jar", 0, 0)).thenReturn(getResourceList()); - result = resourcesService.updateResource(user,1,"ResourcesServiceTest1.jar","ResourcesServiceTest",ResourceType.FILE); + result = resourcesService.updateResource(user,1,"ResourcesServiceTest1.jar","ResourcesServiceTest",ResourceType.FILE,null); logger.info(result.toString()); Assert.assertEquals(Status.RESOURCE_EXIST.getMsg(),result.getMsg()); //USER_NOT_EXIST Mockito.when(userMapper.selectById(Mockito.anyInt())).thenReturn(null); - result = resourcesService.updateResource(user,1,"ResourcesServiceTest1.jar","ResourcesServiceTest",ResourceType.UDF); + result = resourcesService.updateResource(user,1,"ResourcesServiceTest1.jar","ResourcesServiceTest",ResourceType.UDF,null); logger.info(result.toString()); Assert.assertTrue(Status.USER_NOT_EXIST.getCode() == result.getCode()); //TENANT_NOT_EXIST Mockito.when(userMapper.selectById(1)).thenReturn(getUser()); Mockito.when(tenantMapper.queryById(Mockito.anyInt())).thenReturn(null); - result = resourcesService.updateResource(user,1,"ResourcesServiceTest1.jar","ResourcesServiceTest",ResourceType.UDF); + result = resourcesService.updateResource(user,1,"ResourcesServiceTest1.jar","ResourcesServiceTest",ResourceType.UDF,null); logger.info(result.toString()); Assert.assertEquals(Status.TENANT_NOT_EXIST.getMsg(),result.getMsg()); @@ -231,7 +235,7 @@ public class ResourcesServiceTest { logger.error(e.getMessage(),e); } - result = resourcesService.updateResource(user,1,"ResourcesServiceTest1.jar","ResourcesServiceTest1.jar",ResourceType.UDF); + result = resourcesService.updateResource(user,1,"ResourcesServiceTest1.jar","ResourcesServiceTest1.jar",ResourceType.UDF,null); logger.info(result.toString()); Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg()); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java index 9ec24bbb50..884e9b6b36 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java @@ -149,9 +149,11 @@ public class UdfFuncServiceTest { } @Test - public void testQueryResourceList(){ - Mockito.when(udfFuncMapper.getUdfFuncByType(1, 1)).thenReturn(getList()); - Map result = udfFuncService.queryResourceList(getLoginUser(),1); + public void testQueryUdfFuncList(){ + User user = getLoginUser(); + user.setUserType(UserType.GENERAL_USER); + Mockito.when(udfFuncMapper.getUdfFuncByType(user.getId(), UdfType.HIVE.ordinal())).thenReturn(getList()); + Map result = udfFuncService.queryUdfFuncList(user,UdfType.HIVE.ordinal()); logger.info(result.toString()); Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); List udfFuncList = (List) result.get(Constants.DATA_LIST); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java index c4d3d6e126..d86c2cc93e 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java @@ -225,13 +225,13 @@ public class UsersServiceTest { String userPassword = "userTest0001"; try { //user not exist - Map result = usersService.updateUser(0,userName,userPassword,"3443@qq.com",1,"13457864543","queue", 1); + Map result = usersService.updateUser(getLoginUser(), 0,userName,userPassword,"3443@qq.com",1,"13457864543","queue", 1); Assert.assertEquals(Status.USER_NOT_EXIST, result.get(Constants.STATUS)); logger.info(result.toString()); //success when(userMapper.selectById(1)).thenReturn(getUser()); - result = usersService.updateUser(1,userName,userPassword,"32222s@qq.com",1,"13457864543","queue", 1); + result = usersService.updateUser(getLoginUser(), 1,userName,userPassword,"32222s@qq.com",1,"13457864543","queue", 1); logger.info(result.toString()); Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); } catch (Exception e) { @@ -357,6 +357,12 @@ public class UsersServiceTest { } + private User getLoginUser(){ + User loginUser = new User(); + loginUser.setId(1); + loginUser.setUserType(UserType.ADMIN_USER); + return loginUser; + } @Test public void getUserInfo(){ diff --git a/dolphinscheduler-common/pom.xml b/dolphinscheduler-common/pom.xml index 2ade59550f..52f2361581 100644 --- a/dolphinscheduler-common/pom.xml +++ b/dolphinscheduler-common/pom.xml @@ -21,7 +21,7 @@ org.apache.dolphinscheduler dolphinscheduler - 1.3.2-SNAPSHOT + 1.3.4-SNAPSHOT dolphinscheduler-common dolphinscheduler-common diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Event.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Event.java new file mode 100644 index 0000000000..9cec2766f1 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Event.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.enums; + +public enum Event { + ACK, + RESULT; +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/process/ResourceInfo.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/process/ResourceInfo.java index a7fc0839eb..287f7267bc 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/process/ResourceInfo.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/process/ResourceInfo.java @@ -42,6 +42,4 @@ public class ResourceInfo { public void setRes(String res) { this.res = res; } - - } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/flink/FlinkParameters.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/flink/FlinkParameters.java index 231dd33146..1b1f0a6c5d 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/flink/FlinkParameters.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/flink/FlinkParameters.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.common.task.flink; import org.apache.dolphinscheduler.common.enums.ProgramType; import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.AbstractParameters; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; import java.util.ArrayList; import java.util.List; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/spark/SparkParameters.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/spark/SparkParameters.java index 4e58201bf3..32a2a6b05d 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/spark/SparkParameters.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/spark/SparkParameters.java @@ -216,7 +216,7 @@ public class SparkParameters extends AbstractParameters { @Override public boolean checkParameters() { - return mainJar != null && programType != null && sparkVersion != null; + return mainJar != null && programType != null; } @Override diff --git a/dolphinscheduler-dao/pom.xml b/dolphinscheduler-dao/pom.xml index c474f6d992..6bdaf416d9 100644 --- a/dolphinscheduler-dao/pom.xml +++ b/dolphinscheduler-dao/pom.xml @@ -16,12 +16,13 @@ ~ limitations under the License. --> - + 4.0.0 org.apache.dolphinscheduler dolphinscheduler - 1.3.2-SNAPSHOT + 1.3.4-SNAPSHOT dolphinscheduler-dao ${project.artifactId} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Command.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Command.java index 7d52dc93f3..cba0151828 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Command.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Command.java @@ -131,6 +131,7 @@ public class Command { WarningType warningType, int warningGroupId, Date scheduleTime, + String workerGroup, Priority processInstancePriority) { this.commandType = commandType; this.executorId = executorId; @@ -143,6 +144,7 @@ public class Command { this.failureStrategy = failureStrategy; this.startTime = new Date(); this.updateTime = new Date(); + this.workerGroup = workerGroup; this.processInstancePriority = processInstancePriority; } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.java index 1c521ce91e..a9ebbf000c 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.java @@ -82,6 +82,20 @@ public interface ProcessInstanceMapper extends BaseMapper { * @param endTime endTime * @return process instance IPage */ + + /** + * process instance page + * @param page page + * @param projectId projectId + * @param processDefinitionId processDefinitionId + * @param searchVal searchVal + * @param executorId executorId + * @param statusArray statusArray + * @param host host + * @param startTime startTime + * @param endTime endTime + * @return process instance page + */ IPage queryProcessInstanceListPaging(Page page, @Param("projectId") int projectId, @Param("processDefinitionId") Integer processDefinitionId, diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java index a2ce6b29b8..b7351f4b49 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java @@ -100,5 +100,12 @@ public interface UdfFuncMapper extends BaseMapper { */ List listAuthorizedUdfByResourceId(@Param("userId") int userId,@Param("resourceIds") int[] resourceIds); + /** + * batch update udf func + * @param udfFuncList udf list + * @return update num + */ + int batchUpdateUdfFunc(@Param("udfFuncList") List udfFuncList); + } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/DolphinSchedulerManager.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/DolphinSchedulerManager.java index 8d1d862640..b2daae28cb 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/DolphinSchedulerManager.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/DolphinSchedulerManager.java @@ -117,6 +117,8 @@ public class DolphinSchedulerManager { upgradeDao.upgradeDolphinScheduler(schemaDir); if ("1.3.0".equals(schemaVersion)) { upgradeDao.upgradeDolphinSchedulerWorkerGroup(); + } else if ("1.3.2".equals(schemaVersion)) { + upgradeDao.upgradeDolphinSchedulerResourceList(); } version = schemaVersion; } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ResourceDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ResourceDao.java new file mode 100644 index 0000000000..b47971e1f0 --- /dev/null +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ResourceDao.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.upgrade; + +import org.apache.dolphinscheduler.common.utils.ConnectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.util.HashMap; +import java.util.Map; + +/** + * resource dao + */ +public class ResourceDao { + public static final Logger logger = LoggerFactory.getLogger(ProcessDefinitionDao.class); + + /** + * list all resources + * @param conn connection + * @return map that key is full_name and value is id + */ + Map listAllResources(Connection conn){ + Map resourceMap = new HashMap<>(); + + String sql = String.format("SELECT id,full_name FROM t_ds_resources"); + ResultSet rs = null; + PreparedStatement pstmt = null; + try { + pstmt = conn.prepareStatement(sql); + rs = pstmt.executeQuery(); + + while (rs.next()){ + Integer id = rs.getInt(1); + String fullName = rs.getString(2); + resourceMap.put(fullName,id); + } + + } catch (Exception e) { + logger.error(e.getMessage(),e); + throw new RuntimeException("sql: " + sql, e); + } finally { + ConnectionUtils.releaseResource(rs, pstmt, conn); + } + + return resourceMap; + } + +} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java index 29c625337d..f0ffc52f68 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java @@ -16,10 +16,10 @@ */ package org.apache.dolphinscheduler.dao.upgrade; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.dao.AbstractBaseDao; import org.apache.dolphinscheduler.dao.datasource.ConnectionFactory; @@ -34,7 +34,9 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.text.MessageFormat; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.stream.Collectors; public abstract class UpgradeDao extends AbstractBaseDao { @@ -88,7 +90,7 @@ public abstract class UpgradeDao extends AbstractBaseDao { /** * init schema */ - public void initSchema(){ + public void initSchema() { DbType dbType = getDbType(); String initSqlPath = ""; if (dbType != null) { @@ -111,6 +113,7 @@ public abstract class UpgradeDao extends AbstractBaseDao { /** * init scheam + * * @param initSqlPath initSqlPath */ public void initSchema(String initSqlPath) { @@ -126,6 +129,7 @@ public abstract class UpgradeDao extends AbstractBaseDao { /** * run DML + * * @param initSqlPath initSqlPath */ private void runInitDML(String initSqlPath) { @@ -148,20 +152,20 @@ public abstract class UpgradeDao extends AbstractBaseDao { try { conn.rollback(); } catch (SQLException e1) { - logger.error(e1.getMessage(),e1); + logger.error(e1.getMessage(), e1); } - logger.error(e.getMessage(),e); - throw new RuntimeException(e.getMessage(),e); + logger.error(e.getMessage(), e); + throw new RuntimeException(e.getMessage(), e); } catch (Exception e) { try { if (null != conn) { conn.rollback(); } } catch (SQLException e1) { - logger.error(e1.getMessage(),e1); + logger.error(e1.getMessage(), e1); } - logger.error(e.getMessage(),e); - throw new RuntimeException(e.getMessage(),e); + logger.error(e.getMessage(), e); + throw new RuntimeException(e.getMessage(), e); } finally { ConnectionUtils.releaseResource(conn); @@ -171,6 +175,7 @@ public abstract class UpgradeDao extends AbstractBaseDao { /** * run DDL + * * @param initSqlPath initSqlPath */ private void runInitDDL(String initSqlPath) { @@ -189,12 +194,12 @@ public abstract class UpgradeDao extends AbstractBaseDao { } catch (IOException e) { - logger.error(e.getMessage(),e); - throw new RuntimeException(e.getMessage(),e); + logger.error(e.getMessage(), e); + throw new RuntimeException(e.getMessage(), e); } catch (Exception e) { - logger.error(e.getMessage(),e); - throw new RuntimeException(e.getMessage(),e); + logger.error(e.getMessage(), e); + throw new RuntimeException(e.getMessage(), e); } finally { ConnectionUtils.releaseResource(conn); @@ -204,6 +209,7 @@ public abstract class UpgradeDao extends AbstractBaseDao { /** * determines whether a table exists + * * @param tableName tableName * @return if table exist return true，else return false */ @@ -211,20 +217,22 @@ public abstract class UpgradeDao extends AbstractBaseDao { /** * determines whether a field exists in the specified table - * @param tableName tableName + * + * @param tableName tableName * @param columnName columnName - * @return if column name exist return true，else return false + * @return if column name exist return true，else return false */ - public abstract boolean isExistsColumn(String tableName,String columnName); + public abstract boolean isExistsColumn(String tableName, String columnName); /** * get current version + * * @param versionName versionName * @return version */ public String getCurrentVersion(String versionName) { - String sql = String.format("select version from %s",versionName); + String sql = String.format("select version from %s", versionName); Connection conn = null; ResultSet rs = null; PreparedStatement pstmt = null; @@ -241,7 +249,7 @@ public abstract class UpgradeDao extends AbstractBaseDao { return version; } catch (SQLException e) { - logger.error(e.getMessage(),e); + logger.error(e.getMessage(), e); throw new RuntimeException("sql: " + sql, e); } finally { ConnectionUtils.releaseResource(rs, pstmt, conn); @@ -251,6 +259,7 @@ public abstract class UpgradeDao extends AbstractBaseDao { /** * upgrade DolphinScheduler + * * @param schemaDir schema dir */ public void upgradeDolphinScheduler(String schemaDir) { @@ -268,52 +277,121 @@ public abstract class UpgradeDao extends AbstractBaseDao { public void upgradeDolphinSchedulerWorkerGroup() { updateProcessDefinitionJsonWorkerGroup(); } + + /** + * upgrade DolphinScheduler resource list + * ds-1.3.2 modify the resource list for process definition json + */ + public void upgradeDolphinSchedulerResourceList() { + updateProcessDefinitionJsonResourceList(); + } + /** * updateProcessDefinitionJsonWorkerGroup */ - protected void updateProcessDefinitionJsonWorkerGroup(){ + protected void updateProcessDefinitionJsonWorkerGroup() { WorkerGroupDao workerGroupDao = new WorkerGroupDao(); ProcessDefinitionDao processDefinitionDao = new ProcessDefinitionDao(); - Map replaceProcessDefinitionMap = new HashMap<>(); + Map replaceProcessDefinitionMap = new HashMap<>(); try { Map oldWorkerGroupMap = workerGroupDao.queryAllOldWorkerGroup(dataSource.getConnection()); - Map processDefinitionJsonMap = processDefinitionDao.queryAllProcessDefinition(dataSource.getConnection()); + Map processDefinitionJsonMap = processDefinitionDao.queryAllProcessDefinition(dataSource.getConnection()); - for (Map.Entry entry : processDefinitionJsonMap.entrySet()){ + for (Map.Entry entry : processDefinitionJsonMap.entrySet()) { ObjectNode jsonObject = JSONUtils.parseObject(entry.getValue()); ArrayNode tasks = JSONUtils.parseArray(jsonObject.get("tasks").toString()); - for (int i = 0 ;i < tasks.size() ; i++){ + for (int i = 0; i < tasks.size(); i++) { ObjectNode task = (ObjectNode) tasks.path(i); ObjectNode workerGroupNode = (ObjectNode) task.path("workerGroupId"); Integer workerGroupId = -1; - if(workerGroupNode != null && workerGroupNode.canConvertToInt()){ + if (workerGroupNode != null && workerGroupNode.canConvertToInt()) { workerGroupId = workerGroupNode.asInt(-1); } if (workerGroupId == -1) { task.put("workerGroup", "default"); - }else { + } else { task.put("workerGroup", oldWorkerGroupMap.get(workerGroupId)); } } jsonObject.remove("task"); - jsonObject.put("tasks",tasks); + jsonObject.put("tasks", tasks); - replaceProcessDefinitionMap.put(entry.getKey(),jsonObject.toString()); + replaceProcessDefinitionMap.put(entry.getKey(), jsonObject.toString()); } - if (replaceProcessDefinitionMap.size() > 0){ - processDefinitionDao.updateProcessDefinitionJson(dataSource.getConnection(),replaceProcessDefinitionMap); + if (replaceProcessDefinitionMap.size() > 0) { + processDefinitionDao.updateProcessDefinitionJson(dataSource.getConnection(), replaceProcessDefinitionMap); } - }catch (Exception e){ - logger.error("update process definition json workergroup error",e); + } catch (Exception e) { + logger.error("update process definition json workergroup error", e); + } + } + + /** + * updateProcessDefinitionJsonResourceList + */ + protected void updateProcessDefinitionJsonResourceList() { + ResourceDao resourceDao = new ResourceDao(); + ProcessDefinitionDao processDefinitionDao = new ProcessDefinitionDao(); + Map replaceProcessDefinitionMap = new HashMap<>(); + try { + Map resourcesMap = resourceDao.listAllResources(dataSource.getConnection()); + Map processDefinitionJsonMap = processDefinitionDao.queryAllProcessDefinition(dataSource.getConnection()); + + for (Map.Entry entry : processDefinitionJsonMap.entrySet()) { + ObjectNode jsonObject = JSONUtils.parseObject(entry.getValue()); + ArrayNode tasks = JSONUtils.parseArray(jsonObject.get("tasks").toString()); + + for (int i = 0; i < tasks.size(); i++) { + ObjectNode task = (ObjectNode) tasks.get(i); + ObjectNode param = (ObjectNode) task.get("params"); + if (param != null) { + + List resourceList = JSONUtils.toList(param.get("resourceList").toString(), ResourceInfo.class); + ResourceInfo mainJar = JSONUtils.parseObject(param.get("mainJar").toString(), ResourceInfo.class); + if (mainJar != null && mainJar.getId() == 0) { + String fullName = mainJar.getRes().startsWith("/") ? mainJar.getRes() : String.format("/%s", mainJar.getRes()); + if (resourcesMap.containsKey(fullName)) { + mainJar.setId(resourcesMap.get(fullName)); + param.put("mainJar", JSONUtils.parseObject(JSONUtils.toJsonString(mainJar))); + } + } + + if (CollectionUtils.isNotEmpty(resourceList)) { + List newResourceList = resourceList.stream().map(resInfo -> { + String fullName = resInfo.getRes().startsWith("/") ? resInfo.getRes() : String.format("/%s", resInfo.getRes()); + if (resInfo.getId() == 0 && resourcesMap.containsKey(fullName)) { + resInfo.setId(resourcesMap.get(fullName)); + } + return resInfo; + }).collect(Collectors.toList()); + param.put("resourceList", JSONUtils.parseObject(JSONUtils.toJsonString(newResourceList))); + } + } + task.put("params", param); + + } + + jsonObject.remove("tasks"); + + jsonObject.put("tasks", tasks); + + replaceProcessDefinitionMap.put(entry.getKey(), jsonObject.toString()); + } + if (replaceProcessDefinitionMap.size() > 0) { + processDefinitionDao.updateProcessDefinitionJson(dataSource.getConnection(), replaceProcessDefinitionMap); + } + } catch (Exception e) { + logger.error("update process definition json resource list error", e); } } /** * upgradeDolphinScheduler DML + * * @param schemaDir schemaDir */ private void upgradeDolphinSchedulerDML(String schemaDir) { @@ -321,8 +399,8 @@ public abstract class UpgradeDao extends AbstractBaseDao { if (StringUtils.isEmpty(rootDir)) { throw new RuntimeException("Environment variable user.dir not found"); } - String sqlFilePath = MessageFormat.format("{0}/sql/upgrade/{1}/{2}/dolphinscheduler_dml.sql",rootDir,schemaDir,getDbType().name().toLowerCase()); - logger.info("sqlSQLFilePath"+sqlFilePath); + String sqlFilePath = MessageFormat.format("{0}/sql/upgrade/{1}/{2}/dolphinscheduler_dml.sql", rootDir, schemaDir, getDbType().name().toLowerCase()); + logger.info("sqlSQLFilePath" + sqlFilePath); Connection conn = null; PreparedStatement pstmt = null; try { @@ -334,13 +412,13 @@ public abstract class UpgradeDao extends AbstractBaseDao { scriptRunner.runScript(sqlReader); if (isExistsTable(T_VERSION_NAME)) { // Change version in the version table to the new version - String upgradeSQL = String.format("update %s set version = ?",T_VERSION_NAME); + String upgradeSQL = String.format("update %s set version = ?", T_VERSION_NAME); pstmt = conn.prepareStatement(upgradeSQL); pstmt.setString(1, schemaVersion); pstmt.executeUpdate(); - }else if (isExistsTable(T_NEW_VERSION_NAME)) { + } else if (isExistsTable(T_NEW_VERSION_NAME)) { // Change version in the version table to the new version - String upgradeSQL = String.format("update %s set version = ?",T_NEW_VERSION_NAME); + String upgradeSQL = String.format("update %s set version = ?", T_NEW_VERSION_NAME); pstmt = conn.prepareStatement(upgradeSQL); pstmt.setString(1, schemaVersion); pstmt.executeUpdate(); @@ -350,38 +428,38 @@ public abstract class UpgradeDao extends AbstractBaseDao { try { conn.rollback(); } catch (SQLException e1) { - logger.error(e1.getMessage(),e1); + logger.error(e1.getMessage(), e1); } - logger.error(e.getMessage(),e); + logger.error(e.getMessage(), e); throw new RuntimeException("sql file not found ", e); } catch (IOException e) { try { conn.rollback(); } catch (SQLException e1) { - logger.error(e1.getMessage(),e1); + logger.error(e1.getMessage(), e1); } - logger.error(e.getMessage(),e); - throw new RuntimeException(e.getMessage(),e); + logger.error(e.getMessage(), e); + throw new RuntimeException(e.getMessage(), e); } catch (SQLException e) { try { if (null != conn) { conn.rollback(); } } catch (SQLException e1) { - logger.error(e1.getMessage(),e1); + logger.error(e1.getMessage(), e1); } - logger.error(e.getMessage(),e); - throw new RuntimeException(e.getMessage(),e); + logger.error(e.getMessage(), e); + throw new RuntimeException(e.getMessage(), e); } catch (Exception e) { try { if (null != conn) { conn.rollback(); } } catch (SQLException e1) { - logger.error(e1.getMessage(),e1); + logger.error(e1.getMessage(), e1); } - logger.error(e.getMessage(),e); - throw new RuntimeException(e.getMessage(),e); + logger.error(e.getMessage(), e); + throw new RuntimeException(e.getMessage(), e); } finally { ConnectionUtils.releaseResource(pstmt, conn); } @@ -390,13 +468,14 @@ public abstract class UpgradeDao extends AbstractBaseDao { /** * upgradeDolphinScheduler DDL + * * @param schemaDir schemaDir */ private void upgradeDolphinSchedulerDDL(String schemaDir) { if (StringUtils.isEmpty(rootDir)) { throw new RuntimeException("Environment variable user.dir not found"); } - String sqlFilePath = MessageFormat.format("{0}/sql/upgrade/{1}/{2}/dolphinscheduler_ddl.sql",rootDir,schemaDir,getDbType().name().toLowerCase()); + String sqlFilePath = MessageFormat.format("{0}/sql/upgrade/{1}/{2}/dolphinscheduler_ddl.sql", rootDir, schemaDir, getDbType().name().toLowerCase()); Connection conn = null; PreparedStatement pstmt = null; try { @@ -411,20 +490,20 @@ public abstract class UpgradeDao extends AbstractBaseDao { } catch (FileNotFoundException e) { - logger.error(e.getMessage(),e); + logger.error(e.getMessage(), e); throw new RuntimeException("sql file not found ", e); } catch (IOException e) { - logger.error(e.getMessage(),e); - throw new RuntimeException(e.getMessage(),e); + logger.error(e.getMessage(), e); + throw new RuntimeException(e.getMessage(), e); } catch (SQLException e) { - logger.error(e.getMessage(),e); - throw new RuntimeException(e.getMessage(),e); + logger.error(e.getMessage(), e); + throw new RuntimeException(e.getMessage(), e); } catch (Exception e) { - logger.error(e.getMessage(),e); - throw new RuntimeException(e.getMessage(),e); + logger.error(e.getMessage(), e); + throw new RuntimeException(e.getMessage(), e); } finally { ConnectionUtils.releaseResource(pstmt, conn); } @@ -434,15 +513,16 @@ public abstract class UpgradeDao extends AbstractBaseDao { /** * update version + * * @param version version */ public void updateVersion(String version) { // Change version in the version table to the new version String versionName = T_VERSION_NAME; - if(!SchemaUtils.isAGreatVersion("1.2.0" , version)){ + if (!SchemaUtils.isAGreatVersion("1.2.0", version)) { versionName = "t_ds_version"; } - String upgradeSQL = String.format("update %s set version = ?",versionName); + String upgradeSQL = String.format("update %s set version = ?", versionName); PreparedStatement pstmt = null; Connection conn = null; try { @@ -452,7 +532,7 @@ public abstract class UpgradeDao extends AbstractBaseDao { pstmt.executeUpdate(); } catch (SQLException e) { - logger.error(e.getMessage(),e); + logger.error(e.getMessage(), e); throw new RuntimeException("sql: " + upgradeSQL, e); } finally { ConnectionUtils.releaseResource(pstmt, conn); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/DagHelper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/DagHelper.java index d3b829cb4f..68d547f0a0 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/DagHelper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/DagHelper.java @@ -22,6 +22,7 @@ import org.apache.dolphinscheduler.common.graph.DAG; import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.model.TaskNodeRelation; import org.apache.dolphinscheduler.common.process.ProcessDag; +import org.apache.dolphinscheduler.common.task.conditions.ConditionsParameters; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.StringUtils; @@ -165,9 +166,6 @@ public class DagHelper { resultList.add(startNode); } if (CollectionUtils.isEmpty(depList)) { - if (null != startNode) { - visitedNodeNameList.add(startNode.getName()); - } return resultList; } for (String depNodeName : depList) { @@ -252,71 +250,86 @@ public class DagHelper { return null; } - /** - * get start vertex in one dag - * it would find the post node if the start vertex is forbidden running - * @param parentNodeName previous node + * the task can be submit when all the depends nodes are forbidden or complete + * @param taskNode taskNode * @param dag dag * @param completeTaskList completeTaskList - * @return start Vertex list + * @return can submit */ - public static Collection getStartVertex(String parentNodeName, DAG dag, - Map completeTaskList){ - - if(completeTaskList == null){ - completeTaskList = new HashMap<>(); + public static boolean allDependsForbiddenOrEnd(TaskNode taskNode, + DAG dag, + Map skipTaskNodeList, + Map completeTaskList) { + List dependList = taskNode.getDepList(); + if (dependList == null) { + return true; } - Collection startVertexs = null; - if(StringUtils.isNotEmpty(parentNodeName)){ - startVertexs = dag.getSubsequentNodes(parentNodeName); - }else{ - startVertexs = dag.getBeginNode(); + for (String dependNodeName : dependList) { + TaskNode dependNode = dag.getNode(dependNodeName); + if (completeTaskList.containsKey(dependNodeName) + || dependNode.isForbidden() + || skipTaskNodeList.containsKey(dependNodeName)) { + continue; + } else { + return false; + } } + return true; + } - List tmpStartVertexs = new ArrayList<>(); - if(startVertexs!= null){ - tmpStartVertexs.addAll(startVertexs); + /** + * parse the successor nodes of previous node. + * this function parse the condition node to find the right branch. + * also check all the depends nodes forbidden or complete + * @param preNodeName + * @return successor nodes + */ + public static Set parsePostNodes(String preNodeName, + Map skipTaskNodeList, + DAG dag, + Map completeTaskList) { + Set postNodeList = new HashSet<>(); + Collection startVertexes = new ArrayList<>(); + if (preNodeName == null) { + startVertexes = dag.getBeginNode(); + } else if (dag.getNode(preNodeName).isConditionsTask()) { + List conditionTaskList = parseConditionTask(preNodeName, skipTaskNodeList, dag, completeTaskList); + startVertexes.addAll(conditionTaskList); + } else { + startVertexes = dag.getSubsequentNodes(preNodeName); } - - for(String start : startVertexs){ - TaskNode startNode = dag.getNode(start); - if(!startNode.isForbidden() && !completeTaskList.containsKey(start)){ - // the start can be submit if not forbidden and not in complete tasks + for (String subsequent : startVertexes) { + TaskNode taskNode = dag.getNode(subsequent); + if (isTaskNodeNeedSkip(taskNode, skipTaskNodeList)) { + setTaskNodeSkip(subsequent, dag, completeTaskList, skipTaskNodeList ); continue; } - // then submit the post nodes - Collection postNodes = getStartVertex(start, dag, completeTaskList); - for(String post : postNodes){ - TaskNode postNode = dag.getNode(post); - if(taskNodeCanSubmit(postNode, dag, completeTaskList)){ - tmpStartVertexs.add(post); - } + if (!DagHelper.allDependsForbiddenOrEnd(taskNode, dag, skipTaskNodeList, completeTaskList)) { + continue; } - tmpStartVertexs.remove(start); + if (taskNode.isForbidden() || completeTaskList.containsKey(subsequent)) { + postNodeList.addAll(parsePostNodes(subsequent, skipTaskNodeList, dag, completeTaskList)); + continue; + } + postNodeList.add(subsequent); } - return tmpStartVertexs; + return postNodeList; } /** - * the task can be submit when all the depends nodes are forbidden or complete - * @param taskNode taskNode - * @param dag dag - * @param completeTaskList completeTaskList - * @return can submit + * if all of the task dependence are skipped, skip it too. + * @param taskNode + * @return */ - public static boolean taskNodeCanSubmit(TaskNode taskNode, - DAG dag, - Map completeTaskList) { - - List dependList = taskNode.getDepList(); - if(dependList == null){ - return true; + private static boolean isTaskNodeNeedSkip(TaskNode taskNode, + Map skipTaskNodeList + ){ + if(CollectionUtils.isEmpty(taskNode.getDepList())){ + return false; } - - for(String dependNodeName : dependList){ - TaskNode dependNode = dag.getNode(dependNodeName); - if(!dependNode.isForbidden() && !completeTaskList.containsKey(dependNodeName)){ + for(String depNode : taskNode.getDepList()){ + if(!skipTaskNodeList.containsKey(depNode)){ return false; } } @@ -324,6 +337,66 @@ public class DagHelper { } + /** + * parse condition task find the branch process + * set skip flag for another one. + * @param nodeName + * @return + */ + public static List parseConditionTask(String nodeName, + Map skipTaskNodeList, + DAG dag, + Map completeTaskList){ + List conditionTaskList = new ArrayList<>(); + TaskNode taskNode = dag.getNode(nodeName); + if (!taskNode.isConditionsTask()){ + return conditionTaskList; + } + if (!completeTaskList.containsKey(nodeName)){ + return conditionTaskList; + } + TaskInstance taskInstance = completeTaskList.get(nodeName); + ConditionsParameters conditionsParameters = + JSONUtils.parseObject(taskNode.getConditionResult(), ConditionsParameters.class); + List skipNodeList = new ArrayList<>(); + if(taskInstance.getState().typeIsSuccess()){ + conditionTaskList = conditionsParameters.getSuccessNode(); + skipNodeList = conditionsParameters.getFailedNode(); + }else if(taskInstance.getState().typeIsFailure()){ + conditionTaskList = conditionsParameters.getFailedNode(); + skipNodeList = conditionsParameters.getSuccessNode(); + }else{ + conditionTaskList.add(nodeName); + } + for(String failedNode : skipNodeList){ + setTaskNodeSkip(failedNode, dag, completeTaskList, skipTaskNodeList); + } + return conditionTaskList; + } + + /** + * set task node and the post nodes skip flag + * @param skipNodeName + * @param dag + * @param completeTaskList + * @param skipTaskNodeList + */ + private static void setTaskNodeSkip(String skipNodeName, + DAG dag, + Map completeTaskList, + Map skipTaskNodeList){ + skipTaskNodeList.putIfAbsent(skipNodeName, dag.getNode(skipNodeName)); + Collection postNodeList = dag.getSubsequentNodes(skipNodeName); + for(String post : postNodeList){ + TaskNode postNode = dag.getNode(post); + if(isTaskNodeNeedSkip(postNode, skipTaskNodeList)){ + setTaskNodeSkip(post, dag, completeTaskList, skipTaskNodeList); + } + } + } + + + /*** * build dag graph * @param processDag processDag @@ -383,7 +456,7 @@ public class DagHelper { */ public static boolean haveConditionsAfterNode(String parentNodeName, DAG dag - ){ + ){ boolean result = false; Set subsequentNodes = dag.getSubsequentNodes(parentNodeName); if(CollectionUtils.isEmpty(subsequentNodes)){ diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.xml index e38d1637d6..18de6db620 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.xml @@ -111,4 +111,17 @@ + + + + update t_ds_udfs + + resource_name=#{udf.resourceName}, + update_time=#{udf.updateTime} + + + id=#{udf.id} + + + \ No newline at end of file diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapperTest.java index 47d8d89b40..ca672e0281 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapperTest.java @@ -19,11 +19,13 @@ package org.apache.dolphinscheduler.dao.mapper; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; + import org.apache.dolphinscheduler.common.enums.UdfType; import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.dao.entity.UDFUser; import org.apache.dolphinscheduler.dao.entity.UdfFunc; import org.apache.dolphinscheduler.dao.entity.User; + import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -33,6 +35,7 @@ import org.springframework.test.annotation.Rollback; import org.springframework.test.context.junit4.SpringRunner; import org.springframework.transaction.annotation.Transactional; +import java.util.ArrayList; import java.util.Arrays; import java.util.Date; import java.util.List; @@ -56,9 +59,10 @@ public class UdfFuncMapperTest { /** * insert one udf + * * @return UdfFunc */ - private UdfFunc insertOne(){ + private UdfFunc insertOne() { UdfFunc udfFunc = new UdfFunc(); udfFunc.setUserId(1); udfFunc.setFuncName("dolphin_udf_func"); @@ -74,9 +78,10 @@ public class UdfFuncMapperTest { /** * insert one udf + * * @return */ - private UdfFunc insertOne(User user){ + private UdfFunc insertOne(User user) { UdfFunc udfFunc = new UdfFunc(); udfFunc.setUserId(user.getId()); udfFunc.setFuncName("dolphin_udf_func"); @@ -92,9 +97,10 @@ public class UdfFuncMapperTest { /** * insert one user + * * @return User */ - private User insertOneUser(){ + private User insertOneUser() { User user = new User(); user.setUserName("user1"); user.setUserPassword("1"); @@ -109,9 +115,10 @@ public class UdfFuncMapperTest { /** * insert one user + * * @return User */ - private User insertOneUser(String userName){ + private User insertOneUser(String userName) { User user = new User(); user.setUserName(userName); user.setUserPassword("1"); @@ -126,11 +133,12 @@ public class UdfFuncMapperTest { /** * insert UDFUser - * @param user user + * + * @param user user * @param udfFunc udf func * @return UDFUser */ - private UDFUser insertOneUDFUser(User user,UdfFunc udfFunc){ + private UDFUser insertOneUDFUser(User user, UdfFunc udfFunc) { UDFUser udfUser = new UDFUser(); udfUser.setUdfId(udfFunc.getId()); udfUser.setUserId(user.getId()); @@ -142,9 +150,10 @@ public class UdfFuncMapperTest { /** * create general user + * * @return User */ - private User createGeneralUser(String userName){ + private User createGeneralUser(String userName) { User user = new User(); user.setUserName(userName); user.setUserPassword("1"); @@ -161,7 +170,7 @@ public class UdfFuncMapperTest { * test update */ @Test - public void testUpdate(){ + public void testUpdate() { //insertOne UdfFunc udfFunc = insertOne(); udfFunc.setResourceName("dolphin_resource_update"); @@ -178,7 +187,7 @@ public class UdfFuncMapperTest { * test delete */ @Test - public void testDelete(){ + public void testDelete() { //insertOne UdfFunc udfFunc = insertOne(); //delete @@ -190,7 +199,7 @@ public class UdfFuncMapperTest { * test query */ @Test - public void testQuery(){ + public void testQuery() { //insertOne UdfFunc udfFunc = insertOne(); //query @@ -207,9 +216,9 @@ public class UdfFuncMapperTest { UdfFunc udfFunc = insertOne(); //insertOne UdfFunc udfFunc1 = insertOne(); - int[] idArray = new int[]{udfFunc.getId(),udfFunc1.getId()}; + int[] idArray = new int[]{udfFunc.getId(), udfFunc1.getId()}; //queryUdfByIdStr - List udfFuncList = udfFuncMapper.queryUdfByIdStr(idArray,""); + List udfFuncList = udfFuncMapper.queryUdfByIdStr(idArray, ""); Assert.assertNotEquals(udfFuncList.size(), 0); } @@ -223,8 +232,8 @@ public class UdfFuncMapperTest { //insertOne UdfFunc udfFunc = insertOne(user); //queryUdfFuncPaging - Page page = new Page(1,3); - IPage udfFuncIPage = udfFuncMapper.queryUdfFuncPaging(page,user.getId(),""); + Page page = new Page(1, 3); + IPage udfFuncIPage = udfFuncMapper.queryUdfFuncPaging(page, user.getId(), ""); Assert.assertNotEquals(udfFuncIPage.getTotal(), 0); } @@ -279,7 +288,7 @@ public class UdfFuncMapperTest { } @Test - public void testListAuthorizedUdfFunc(){ + public void testListAuthorizedUdfFunc() { //create general user User generalUser1 = createGeneralUser("user1"); User generalUser2 = createGeneralUser("user2"); @@ -289,18 +298,30 @@ public class UdfFuncMapperTest { UdfFunc unauthorizdUdfFunc = insertOne(generalUser2); //udf function ids - Integer[] udfFuncIds = new Integer[]{udfFunc.getId(),unauthorizdUdfFunc.getId()}; + Integer[] udfFuncIds = new Integer[]{udfFunc.getId(), unauthorizdUdfFunc.getId()}; List authorizedUdfFunc = udfFuncMapper.listAuthorizedUdfFunc(generalUser1.getId(), udfFuncIds); - Assert.assertEquals(generalUser1.getId(),udfFunc.getUserId()); - Assert.assertNotEquals(generalUser1.getId(),unauthorizdUdfFunc.getUserId()); + Assert.assertEquals(generalUser1.getId(), udfFunc.getUserId()); + Assert.assertNotEquals(generalUser1.getId(), unauthorizdUdfFunc.getUserId()); Assert.assertFalse(authorizedUdfFunc.stream().map(t -> t.getId()).collect(toList()).containsAll(Arrays.asList(udfFuncIds))); //authorize object unauthorizdUdfFunc to generalUser1 - insertOneUDFUser(generalUser1,unauthorizdUdfFunc); + insertOneUDFUser(generalUser1, unauthorizdUdfFunc); authorizedUdfFunc = udfFuncMapper.listAuthorizedUdfFunc(generalUser1.getId(), udfFuncIds); Assert.assertTrue(authorizedUdfFunc.stream().map(t -> t.getId()).collect(toList()).containsAll(Arrays.asList(udfFuncIds))); } + + @Test + public void batchUpdateUdfFuncTest() { + //create general user + User generalUser1 = createGeneralUser("user1"); + UdfFunc udfFunc = insertOne(generalUser1); + udfFunc.setResourceName("/updateTest"); + List udfFuncList = new ArrayList<>(); + udfFuncList.add(udfFunc); + Assert.assertTrue(udfFuncMapper.batchUpdateUdfFunc(udfFuncList) > 0); + + } } \ No newline at end of file diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/DagHelperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/DagHelperTest.java index 14dfe0b750..88e71c4891 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/DagHelperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/DagHelperTest.java @@ -18,7 +18,9 @@ package org.apache.dolphinscheduler.dao.utils; import com.fasterxml.jackson.core.JsonProcessingException; + import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.TaskDependType; import org.apache.dolphinscheduler.common.graph.DAG; import org.apache.dolphinscheduler.common.model.TaskNode; @@ -27,6 +29,7 @@ import org.apache.dolphinscheduler.common.process.ProcessDag; import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.dao.entity.ProcessData; import org.apache.dolphinscheduler.dao.entity.TaskInstance; + import org.junit.Assert; import org.junit.Test; @@ -34,6 +37,8 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; + /** * dag helper test @@ -41,15 +46,17 @@ import java.util.Map; public class DagHelperTest { /** * test task node can submit + * * @throws JsonProcessingException if error throws JsonProcessingException */ @Test public void testTaskNodeCanSubmit() throws JsonProcessingException { - //1->2->3->5 - //4->3 + //1->2->3->5->7 + //4->3->6 DAG dag = generateDag(); TaskNode taskNode3 = dag.getNode("3"); - Map completeTaskList = new HashMap<>(); + Map completeTaskList = new HashMap<>(); + Map skipNodeList = new HashMap<>(); completeTaskList.putIfAbsent("1", new TaskInstance()); Boolean canSubmit = false; @@ -58,27 +65,206 @@ public class DagHelperTest { node2.setRunFlag(Constants.FLOWNODE_RUN_FLAG_FORBIDDEN); TaskNode nodex = dag.getNode("4"); nodex.setRunFlag(Constants.FLOWNODE_RUN_FLAG_FORBIDDEN); - canSubmit = DagHelper.taskNodeCanSubmit(taskNode3, dag, completeTaskList); + canSubmit = DagHelper.allDependsForbiddenOrEnd(taskNode3, dag, skipNodeList, completeTaskList); Assert.assertEquals(canSubmit, true); // 2forbidden, 3 cannot be submit completeTaskList.putIfAbsent("2", new TaskInstance()); TaskNode nodey = dag.getNode("4"); nodey.setRunFlag(""); - canSubmit = DagHelper.taskNodeCanSubmit(taskNode3, dag, completeTaskList); + canSubmit = DagHelper.allDependsForbiddenOrEnd(taskNode3, dag, skipNodeList, completeTaskList); Assert.assertEquals(canSubmit, false); // 2/3 forbidden submit 5 TaskNode node3 = dag.getNode("3"); node3.setRunFlag(Constants.FLOWNODE_RUN_FLAG_FORBIDDEN); + TaskNode node8 = dag.getNode("8"); + node8.setRunFlag(Constants.FLOWNODE_RUN_FLAG_FORBIDDEN); TaskNode node5 = dag.getNode("5"); - canSubmit = DagHelper.taskNodeCanSubmit(node5, dag, completeTaskList); + canSubmit = DagHelper.allDependsForbiddenOrEnd(node5, dag, skipNodeList, completeTaskList); Assert.assertEquals(canSubmit, true); - } + } /** - * 1->2->3->5 - * 4->3 + * test parse post node list + */ + @Test + public void testParsePostNodeList() throws JsonProcessingException { + DAG dag = generateDag(); + Map completeTaskList = new HashMap<>(); + Map skipNodeList = new HashMap<>(); + + Set postNodes = null; + //complete : null + // expect post: 1/4 + postNodes = DagHelper.parsePostNodes(null, skipNodeList, dag, completeTaskList); + Assert.assertEquals(2, postNodes.size()); + Assert.assertTrue(postNodes.contains("1")); + Assert.assertTrue(postNodes.contains("4")); + + //complete : 1 + // expect post: 2/4 + completeTaskList.put("1", new TaskInstance()); + postNodes = DagHelper.parsePostNodes(null, skipNodeList, dag, completeTaskList); + Assert.assertEquals(2, postNodes.size()); + Assert.assertTrue(postNodes.contains("2")); + Assert.assertTrue(postNodes.contains("4")); + + // complete : 1/2 + // expect post: 4 + completeTaskList.put("2", new TaskInstance()); + postNodes = DagHelper.parsePostNodes(null, skipNodeList, dag, completeTaskList); + Assert.assertEquals(2, postNodes.size()); + Assert.assertTrue(postNodes.contains("4")); + Assert.assertTrue(postNodes.contains("8")); + + // complete : 1/2/4 + // expect post: 3 + completeTaskList.put("4", new TaskInstance()); + postNodes = DagHelper.parsePostNodes(null, skipNodeList, dag, completeTaskList); + Assert.assertEquals(2, postNodes.size()); + Assert.assertTrue(postNodes.contains("3")); + Assert.assertTrue(postNodes.contains("8")); + + // complete : 1/2/4/3 + // expect post: 8/6 + completeTaskList.put("3", new TaskInstance()); + postNodes = DagHelper.parsePostNodes(null, skipNodeList, dag, completeTaskList); + Assert.assertEquals(2, postNodes.size()); + Assert.assertTrue(postNodes.contains("8")); + Assert.assertTrue(postNodes.contains("6")); + + // complete : 1/2/4/3/8 + // expect post: 6/5 + completeTaskList.put("8", new TaskInstance()); + postNodes = DagHelper.parsePostNodes(null, skipNodeList, dag, completeTaskList); + Assert.assertEquals(2, postNodes.size()); + Assert.assertTrue(postNodes.contains("5")); + Assert.assertTrue(postNodes.contains("6")); + // complete : 1/2/4/3/5/6/8 + // expect post: 7 + completeTaskList.put("6", new TaskInstance()); + completeTaskList.put("5", new TaskInstance()); + postNodes = DagHelper.parsePostNodes(null, skipNodeList, dag, completeTaskList); + Assert.assertEquals(1, postNodes.size()); + Assert.assertTrue(postNodes.contains("7")); + } + + /** + * test forbidden post node + * + * @throws JsonProcessingException + */ + @Test + public void testForbiddenPostNode() throws JsonProcessingException { + DAG dag = generateDag(); + Map completeTaskList = new HashMap<>(); + Map skipNodeList = new HashMap<>(); + Set postNodes = null; + // dag: 1-2-3-5-7 4-3-6 2-8-5-7 + // forbid:2 complete:1 post:4/8 + completeTaskList.put("1", new TaskInstance()); + TaskNode node2 = dag.getNode("2"); + node2.setRunFlag(Constants.FLOWNODE_RUN_FLAG_FORBIDDEN); + postNodes = DagHelper.parsePostNodes(null, skipNodeList, dag, completeTaskList); + Assert.assertEquals(2, postNodes.size()); + Assert.assertTrue(postNodes.contains("4")); + Assert.assertTrue(postNodes.contains("8")); + + //forbid:2/4 complete:1 post:3/8 + TaskNode node4 = dag.getNode("4"); + node4.setRunFlag(Constants.FLOWNODE_RUN_FLAG_FORBIDDEN); + postNodes = DagHelper.parsePostNodes(null, skipNodeList, dag, completeTaskList); + Assert.assertEquals(2, postNodes.size()); + Assert.assertTrue(postNodes.contains("3")); + Assert.assertTrue(postNodes.contains("8")); + + //forbid:2/4/5 complete:1/8 post:3 + completeTaskList.put("8", new TaskInstance()); + TaskNode node5 = dag.getNode("5"); + node5.setRunFlag(Constants.FLOWNODE_RUN_FLAG_FORBIDDEN); + postNodes = DagHelper.parsePostNodes(null, skipNodeList, dag, completeTaskList); + Assert.assertEquals(1, postNodes.size()); + Assert.assertTrue(postNodes.contains("3")); + } + + /** + * test condition post node + * + * @throws JsonProcessingException + */ + @Test + public void testConditionPostNode() throws JsonProcessingException { + DAG dag = generateDag(); + Map completeTaskList = new HashMap<>(); + Map skipNodeList = new HashMap<>(); + Set postNodes = null; + // dag: 1-2-3-5-7 4-3-6 2-8-5-7 + // 3-if + completeTaskList.put("1", new TaskInstance()); + completeTaskList.put("2", new TaskInstance()); + completeTaskList.put("4", new TaskInstance()); + TaskNode node3 = dag.getNode("3"); + node3.setType("CONDITIONS"); + node3.setConditionResult("{\n" + + + " \"successNode\": [5\n" + + + " ],\n" + + + " \"failedNode\": [6\n" + + + " ]\n" + + + " }"); + completeTaskList.remove("3"); + TaskInstance taskInstance = new TaskInstance(); + taskInstance.setState(ExecutionStatus.SUCCESS); + //complete 1/2/3/4 expect:8 + completeTaskList.put("3", taskInstance); + postNodes = DagHelper.parsePostNodes(null, skipNodeList, dag, completeTaskList); + Assert.assertEquals(1, postNodes.size()); + Assert.assertTrue(postNodes.contains("8")); + + //2.complete 1/2/3/4/8 expect:5 skip:6 + completeTaskList.put("8", new TaskInstance()); + postNodes = DagHelper.parsePostNodes(null, skipNodeList, dag, completeTaskList); + Assert.assertTrue(postNodes.contains("5")); + Assert.assertEquals(1, skipNodeList.size()); + Assert.assertTrue(skipNodeList.containsKey("6")); + + // 3.complete 1/2/3/4/5/8 expect post:7 skip:6 + skipNodeList.clear(); + TaskInstance taskInstance1 = new TaskInstance(); + taskInstance.setState(ExecutionStatus.SUCCESS); + completeTaskList.put("5", taskInstance1); + postNodes = DagHelper.parsePostNodes(null, skipNodeList, dag, completeTaskList); + Assert.assertEquals(1, postNodes.size()); + Assert.assertTrue(postNodes.contains("7")); + Assert.assertEquals(1, skipNodeList.size()); + Assert.assertTrue(skipNodeList.containsKey("6")); + + // dag: 1-2-3-5-7 4-3-6 + // 3-if , complete:1/2/3/4 + // 1.failure:3 expect post:6 skip:5/7 + skipNodeList.clear(); + completeTaskList.remove("3"); + taskInstance = new TaskInstance(); + taskInstance.setState(ExecutionStatus.FAILURE); + completeTaskList.put("3", taskInstance); + postNodes = DagHelper.parsePostNodes(null, skipNodeList, dag, completeTaskList); + Assert.assertEquals(1, postNodes.size()); + Assert.assertTrue(postNodes.contains("6")); + Assert.assertEquals(2, skipNodeList.size()); + Assert.assertTrue(skipNodeList.containsKey("5")); + Assert.assertTrue(skipNodeList.containsKey("7")); + } + + /** + * 1->2->3->5->7 + * 4->3->6 + * 2->8->5->7 + * * @return dag * @throws JsonProcessingException if error throws JsonProcessingException */ @@ -87,11 +273,13 @@ public class DagHelperTest { TaskNode node1 = new TaskNode(); node1.setId("1"); node1.setName("1"); + node1.setType("SHELL"); taskNodeList.add(node1); TaskNode node2 = new TaskNode(); node2.setId("2"); node2.setName("2"); + node2.setType("SHELL"); List dep2 = new ArrayList<>(); dep2.add("1"); node2.setDepList(dep2); @@ -101,11 +289,13 @@ public class DagHelperTest { TaskNode node4 = new TaskNode(); node4.setId("4"); node4.setName("4"); + node4.setType("SHELL"); taskNodeList.add(node4); TaskNode node3 = new TaskNode(); node3.setId("3"); node3.setName("3"); + node3.setType("SHELL"); List dep3 = new ArrayList<>(); dep3.add("2"); dep3.add("4"); @@ -115,29 +305,61 @@ public class DagHelperTest { TaskNode node5 = new TaskNode(); node5.setId("5"); node5.setName("5"); + node5.setType("SHELL"); List dep5 = new ArrayList<>(); dep5.add("3"); + dep5.add("8"); node5.setDepList(dep5); taskNodeList.add(node5); + TaskNode node6 = new TaskNode(); + node6.setId("6"); + node6.setName("6"); + node6.setType("SHELL"); + List dep6 = new ArrayList<>(); + dep6.add("3"); + node6.setDepList(dep6); + taskNodeList.add(node6); + + TaskNode node7 = new TaskNode(); + node7.setId("7"); + node7.setName("7"); + node7.setType("SHELL"); + List dep7 = new ArrayList<>(); + dep7.add("5"); + node7.setDepList(dep7); + taskNodeList.add(node7); + + TaskNode node8 = new TaskNode(); + node8.setId("8"); + node8.setName("8"); + node8.setType("SHELL"); + List dep8 = new ArrayList<>(); + dep8.add("2"); + node8.setDepList(dep8); + taskNodeList.add(node8); + List startNodes = new ArrayList<>(); - List recoveryNodes = new ArrayList<>(); + List recoveryNodes = new ArrayList<>(); List destTaskNodeList = DagHelper.generateFlowNodeListByStartNode(taskNodeList, startNodes, recoveryNodes, TaskDependType.TASK_POST); - List taskNodeRelations =DagHelper.generateRelationListByFlowNodes(destTaskNodeList); + List taskNodeRelations = DagHelper.generateRelationListByFlowNodes(destTaskNodeList); ProcessDag processDag = new ProcessDag(); processDag.setEdges(taskNodeRelations); processDag.setNodes(destTaskNodeList); - return DagHelper.buildDagGraph(processDag); } @Test public void testBuildDagGraph() { - String shellJson = "{\"globalParams\":[],\"tasks\":[{\"type\":\"SHELL\",\"id\":\"tasks-9527\",\"name\":\"shell-1\"," + - "\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"#!/bin/bash\\necho \\\"shell-1\\\"\"}," + - "\"description\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\",\"retryInterval\":\"1\"," + - "\"timeout\":{\"strategy\":\"\",\"interval\":1,\"enable\":false},\"taskInstancePriority\":\"MEDIUM\"," + + String shellJson = "{\"globalParams\":[],\"tasks\":[{\"type\":\"SHELL\",\"id\":\"tasks-9527\",\"name\":\"shell-1\"," + + + "\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"#!/bin/bash\\necho \\\"shell-1\\\"\"}," + + + "\"description\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\",\"retryInterval\":\"1\"," + + + "\"timeout\":{\"strategy\":\"\",\"interval\":1,\"enable\":false},\"taskInstancePriority\":\"MEDIUM\"," + + "\"workerGroupId\":-1,\"preTasks\":[]}],\"tenantId\":1,\"timeout\":0}"; ProcessData processData = JSONUtils.parseObject(shellJson, ProcessData.class); diff --git a/dolphinscheduler-dist/pom.xml b/dolphinscheduler-dist/pom.xml index b193dd85c8..c461564dbf 100644 --- a/dolphinscheduler-dist/pom.xml +++ b/dolphinscheduler-dist/pom.xml @@ -20,7 +20,7 @@ dolphinscheduler org.apache.dolphinscheduler - 1.3.2-SNAPSHOT + 1.3.4-SNAPSHOT 4.0.0 diff --git a/dolphinscheduler-microbench/pom.xml b/dolphinscheduler-microbench/pom.xml index 6b11b2e2d6..606ecd3c38 100644 --- a/dolphinscheduler-microbench/pom.xml +++ b/dolphinscheduler-microbench/pom.xml @@ -21,7 +21,7 @@ dolphinscheduler org.apache.dolphinscheduler - 1.3.2-SNAPSHOT + 1.3.4-SNAPSHOT 4.0.0 diff --git a/dolphinscheduler-plugin-api/pom.xml b/dolphinscheduler-plugin-api/pom.xml index 7db15e73c3..0c2547d137 100644 --- a/dolphinscheduler-plugin-api/pom.xml +++ b/dolphinscheduler-plugin-api/pom.xml @@ -23,7 +23,7 @@ org.apache.dolphinscheduler dolphinscheduler - 1.3.2-SNAPSHOT + 1.3.4-SNAPSHOT dolphinscheduler-plugin-api ${project.artifactId} diff --git a/dolphinscheduler-remote/pom.xml b/dolphinscheduler-remote/pom.xml index 4af03a362f..3ac7b914a5 100644 --- a/dolphinscheduler-remote/pom.xml +++ b/dolphinscheduler-remote/pom.xml @@ -20,7 +20,7 @@ dolphinscheduler org.apache.dolphinscheduler - 1.3.2-SNAPSHOT + 1.3.4-SNAPSHOT 4.0.0 diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/CommandType.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/CommandType.java index 4f477fb467..77c6064e5b 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/CommandType.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/CommandType.java @@ -78,6 +78,16 @@ public enum CommandType { */ TASK_EXECUTE_RESPONSE, + /** + * db task ack + */ + DB_TASK_ACK, + + /** + * db task response + */ + DB_TASK_RESPONSE, + /** * kill task */ diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/DBTaskAckCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/DBTaskAckCommand.java new file mode 100644 index 0000000000..ae302841fa --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/DBTaskAckCommand.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.remote.command; + + +import org.apache.dolphinscheduler.common.utils.JSONUtils; + +import java.io.Serializable; + +/** + * db task ack request command + */ +public class DBTaskAckCommand implements Serializable { + + private int taskInstanceId; + private int status; + + public DBTaskAckCommand(int status,int taskInstanceId) { + this.status = status; + this.taskInstanceId = taskInstanceId; + } + + public int getTaskInstanceId() { + return taskInstanceId; + } + + public void setTaskInstanceId(int taskInstanceId) { + this.taskInstanceId = taskInstanceId; + } + + public int getStatus() { + return status; + } + + public void setStatus(int status) { + this.status = status; + } + + /** + * package response command + * @return command + */ + public Command convert2Command(){ + Command command = new Command(); + command.setType(CommandType.DB_TASK_ACK); + byte[] body = JSONUtils.toJsonByteArray(this); + command.setBody(body); + return command; + } + + + @Override + public String toString() { + return "DBTaskAckCommand{" + + "taskInstanceId=" + taskInstanceId + + ", status=" + status + + '}'; + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/DBTaskResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/DBTaskResponseCommand.java new file mode 100644 index 0000000000..e46cffafe2 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/DBTaskResponseCommand.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.remote.command; + +import org.apache.dolphinscheduler.common.utils.JSONUtils; + +import java.io.Serializable; + +/** + * db task final result response command + */ +public class DBTaskResponseCommand implements Serializable { + + private int taskInstanceId; + private int status; + + public DBTaskResponseCommand(int status,int taskInstanceId) { + this.status = status; + this.taskInstanceId = taskInstanceId; + } + + public int getStatus() { + return status; + } + + public void setStatus(int status) { + this.status = status; + } + + public int getTaskInstanceId() { + return taskInstanceId; + } + + public void setTaskInstanceId(int taskInstanceId) { + this.taskInstanceId = taskInstanceId; + } + + /** + * package response command + * @return command + */ + public Command convert2Command(){ + Command command = new Command(); + command.setType(CommandType.DB_TASK_RESPONSE); + byte[] body = JSONUtils.toJsonByteArray(this); + command.setBody(body); + return command; + } + + @Override + public String toString() { + return "DBTaskResponseCommand{" + + "taskInstanceId=" + taskInstanceId + + ", status=" + status + + '}'; + } +} diff --git a/dolphinscheduler-server/pom.xml b/dolphinscheduler-server/pom.xml index 4cbce0ab47..10fa58faa3 100644 --- a/dolphinscheduler-server/pom.xml +++ b/dolphinscheduler-server/pom.xml @@ -21,7 +21,7 @@ org.apache.dolphinscheduler dolphinscheduler - 1.3.2-SNAPSHOT + 1.3.4-SNAPSHOT dolphinscheduler-server dolphinscheduler-server diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/TaskLogAppender.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/TaskLogAppender.java new file mode 100644 index 0000000000..0819b9fce3 --- /dev/null +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/TaskLogAppender.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.server.log; + +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.FileAppender; +import org.slf4j.Marker; + +import static ch.qos.logback.classic.ClassicConstants.FINALIZE_SESSION_MARKER; + +/** + * Task log appender + */ +public class TaskLogAppender extends FileAppender{ + @Override + protected void append(ILoggingEvent event) { + Marker marker = event.getMarker(); + if (marker !=null) { + if (marker.equals(FINALIZE_SESSION_MARKER)) { + stop(); + } + } + super.subAppend(event); + } +} diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManager.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManager.java index ee9b86babb..18a7d4691d 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManager.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManager.java @@ -17,8 +17,9 @@ package org.apache.dolphinscheduler.server.master.dispatch.executor; -import com.github.rholder.retry.RetryException; -import org.apache.dolphinscheduler.common.utils.RetryerUtils; +import org.apache.commons.collections.CollectionUtils; + +import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.remote.NettyRemotingClient; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; @@ -31,14 +32,15 @@ import org.apache.dolphinscheduler.server.master.processor.TaskAckProcessor; import org.apache.dolphinscheduler.server.master.processor.TaskKillResponseProcessor; import org.apache.dolphinscheduler.server.master.processor.TaskResponseProcessor; import org.apache.dolphinscheduler.server.registry.ZookeeperNodeManager; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import javax.annotation.PostConstruct; + import java.util.*; -import java.util.concurrent.ExecutionException; /** * netty executor manager @@ -86,11 +88,17 @@ public class NettyExecutorManager extends AbstractExecutorManager{ */ @Override public Boolean execute(ExecutionContext context) throws ExecuteException { - LinkedList allNodes = new LinkedList<>(); - Set nodes = getAllNodes(context); - if (nodes != null) { - allNodes.addAll(nodes); - } + + /** + * all nodes + */ + Set allNodes = getAllNodes(context); + + /** + * fail nodes + */ + Set failNodeSet = new HashSet<>(); + /** * build command accord executeContext */ @@ -99,27 +107,31 @@ public class NettyExecutorManager extends AbstractExecutorManager{ /** * execute task host */ - String startHostAddress = context.getHost().getAddress(); - // remove start host address and add it to head - allNodes.remove(startHostAddress); - allNodes.addFirst(startHostAddress); - + Host host = context.getHost(); boolean success = false; - for (String address : allNodes) { + while (!success) { try { - Host host = Host.of(address); - doExecute(host, command); + doExecute(host,command); success = true; context.setHost(host); - break; } catch (ExecuteException ex) { - logger.error("retry execute command : {} host : {}", command, address); + logger.error(String.format("execute command : %s error", command), ex); + try { + failNodeSet.add(host.getAddress()); + Set tmpAllIps = new HashSet<>(allNodes); + Collection remained = CollectionUtils.subtract(tmpAllIps, failNodeSet); + if (remained != null && remained.size() > 0) { + host = Host.of(remained.iterator().next()); + logger.error("retry execute command : {} host : {}", command, host); + } else { + throw new ExecuteException("fail after try all nodes"); + } + } catch (Throwable t) { + throw new ExecuteException("fail after try all nodes"); + } } } - if (!success) { - throw new ExecuteException("fail after try all nodes"); - } - + return success; } @@ -136,13 +148,24 @@ public class NettyExecutorManager extends AbstractExecutorManager{ * @throws ExecuteException if error throws ExecuteException */ private void doExecute(final Host host, final Command command) throws ExecuteException { - try { - RetryerUtils.retryCall(() -> { + /** + * retry count，default retry 3 + */ + int retryCount = 3; + boolean success = false; + do { + try { nettyRemotingClient.send(host, command); - return Boolean.TRUE; - }); - } catch (ExecutionException | RetryException e) { - throw new ExecuteException(String.format("send command : %s to %s error", command, host), e); + success = true; + } catch (Exception ex) { + logger.error(String.format("send command : %s to %s error", command, host), ex); + retryCount--; + ThreadUtils.sleep(100); + } + } while (retryCount >= 0 && !success); + + if (!success) { + throw new ExecuteException(String.format("send command : %s to %s error", command, host)); } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessor.java index 0b4d26d1ce..51d068ad08 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessor.java @@ -17,14 +17,9 @@ package org.apache.dolphinscheduler.server.master.processor; -import static org.apache.dolphinscheduler.common.Constants.SLEEP_TIME_MILLIS; - import org.apache.dolphinscheduler.common.enums.ExecutionStatus; -import org.apache.dolphinscheduler.common.thread.Stopper; -import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.Preconditions; -import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.TaskExecuteAckCommand; @@ -35,7 +30,6 @@ import org.apache.dolphinscheduler.server.master.cache.impl.TaskInstanceCacheMan import org.apache.dolphinscheduler.server.master.processor.queue.TaskResponseEvent; import org.apache.dolphinscheduler.server.master.processor.queue.TaskResponseService; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; -import org.apache.dolphinscheduler.service.process.ProcessService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -59,16 +53,9 @@ public class TaskAckProcessor implements NettyRequestProcessor { */ private final TaskInstanceCacheManager taskInstanceCacheManager; - - /** - * processService - */ - private ProcessService processService; - - public TaskAckProcessor() { + public TaskAckProcessor(){ this.taskResponseService = SpringApplicationContext.getBean(TaskResponseService.class); this.taskInstanceCacheManager = SpringApplicationContext.getBean(TaskInstanceCacheManagerImpl.class); - this.processService = SpringApplicationContext.getBean(ProcessService.class); } /** @@ -94,19 +81,10 @@ public class TaskAckProcessor implements NettyRequestProcessor { workerAddress, taskAckCommand.getExecutePath(), taskAckCommand.getLogPath(), - taskAckCommand.getTaskInstanceId()); + taskAckCommand.getTaskInstanceId(), + channel); taskResponseService.addResponse(taskResponseEvent); - - while (Stopper.isRunning()) { - TaskInstance taskInstance = processService.findTaskInstanceById(taskAckCommand.getTaskInstanceId()); - - if (taskInstance != null && ackStatus.typeIsRunning()) { - break; - } - ThreadUtils.sleep(SLEEP_TIME_MILLIS); - } - } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessor.java index ffe3d5d9a8..080fdd540d 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessor.java @@ -17,14 +17,9 @@ package org.apache.dolphinscheduler.server.master.processor; -import static org.apache.dolphinscheduler.common.Constants.SLEEP_TIME_MILLIS; - import org.apache.dolphinscheduler.common.enums.ExecutionStatus; -import org.apache.dolphinscheduler.common.thread.Stopper; -import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.Preconditions; -import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.TaskExecuteResponseCommand; @@ -34,7 +29,6 @@ import org.apache.dolphinscheduler.server.master.cache.impl.TaskInstanceCacheMan import org.apache.dolphinscheduler.server.master.processor.queue.TaskResponseEvent; import org.apache.dolphinscheduler.server.master.processor.queue.TaskResponseService; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; -import org.apache.dolphinscheduler.service.process.ProcessService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -58,15 +52,9 @@ public class TaskResponseProcessor implements NettyRequestProcessor { */ private final TaskInstanceCacheManager taskInstanceCacheManager; - /** - * processService - */ - private ProcessService processService; - - public TaskResponseProcessor() { + public TaskResponseProcessor(){ this.taskResponseService = SpringApplicationContext.getBean(TaskResponseService.class); this.taskInstanceCacheManager = SpringApplicationContext.getBean(TaskInstanceCacheManagerImpl.class); - this.processService = SpringApplicationContext.getBean(ProcessService.class); } /** @@ -85,26 +73,16 @@ public class TaskResponseProcessor implements NettyRequestProcessor { taskInstanceCacheManager.cacheTaskInstance(responseCommand); - ExecutionStatus responseStatus = ExecutionStatus.of(responseCommand.getStatus()); - // TaskResponseEvent TaskResponseEvent taskResponseEvent = TaskResponseEvent.newResult(ExecutionStatus.of(responseCommand.getStatus()), responseCommand.getEndTime(), responseCommand.getProcessId(), responseCommand.getAppIds(), responseCommand.getTaskInstanceId(), - responseCommand.getVarPool()); - + responseCommand.getVarPool(), + channel); taskResponseService.addResponse(taskResponseEvent); - - while (Stopper.isRunning()) { - TaskInstance taskInstance = processService.findTaskInstanceById(taskResponseEvent.getTaskInstanceId()); - - if (taskInstance != null && responseStatus.typeIsFinished()) { - break; - } - ThreadUtils.sleep(SLEEP_TIME_MILLIS); - } } + } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseEvent.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseEvent.java index ba07be50f3..0ca558a560 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseEvent.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseEvent.java @@ -18,10 +18,14 @@ package org.apache.dolphinscheduler.server.master.processor.queue; import com.fasterxml.jackson.annotation.JsonFormat; + +import org.apache.dolphinscheduler.common.enums.Event; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import java.util.Date; +import io.netty.channel.Channel; + /** * task event */ @@ -83,8 +87,19 @@ public class TaskResponseEvent { * varPool */ private String varPool; + + /** + * channel + */ + private Channel channel; - public static TaskResponseEvent newAck(ExecutionStatus state, Date startTime, String workerAddress, String executePath, String logPath, int taskInstanceId) { + public static TaskResponseEvent newAck(ExecutionStatus state, + Date startTime, + String workerAddress, + String executePath, + String logPath, + int taskInstanceId, + Channel channel) { TaskResponseEvent event = new TaskResponseEvent(); event.setState(state); event.setStartTime(startTime); @@ -93,10 +108,17 @@ public class TaskResponseEvent { event.setLogPath(logPath); event.setTaskInstanceId(taskInstanceId); event.setEvent(Event.ACK); + event.setChannel(channel); return event; } - public static TaskResponseEvent newResult(ExecutionStatus state, Date endTime, int processId, String appIds, int taskInstanceId, String varPool) { + public static TaskResponseEvent newResult(ExecutionStatus state, + Date endTime, + int processId, + String appIds, + int taskInstanceId, + String varPool, + Channel channel) { TaskResponseEvent event = new TaskResponseEvent(); event.setState(state); event.setEndTime(endTime); @@ -105,6 +127,7 @@ public class TaskResponseEvent { event.setTaskInstanceId(taskInstanceId); event.setEvent(Event.RESULT); event.setVarPool(varPool); + event.setChannel(channel); return event; } @@ -196,8 +219,11 @@ public class TaskResponseEvent { this.event = event; } - public enum Event{ - ACK, - RESULT; + public Channel getChannel() { + return channel; + } + + public void setChannel(Channel channel) { + this.channel = channel; } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseService.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseService.java index 6434db70e5..65315c7bd9 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseService.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseService.java @@ -17,7 +17,13 @@ package org.apache.dolphinscheduler.server.master.processor.queue; +import io.netty.channel.Channel; +import org.apache.dolphinscheduler.common.enums.Event; +import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.thread.Stopper; +import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.remote.command.DBTaskAckCommand; +import org.apache.dolphinscheduler.remote.command.DBTaskResponseCommand; import org.apache.dolphinscheduler.service.process.ProcessService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -123,24 +129,50 @@ public class TaskResponseService { * @param taskResponseEvent taskResponseEvent */ private void persist(TaskResponseEvent taskResponseEvent){ - TaskResponseEvent.Event event = taskResponseEvent.getEvent(); + Event event = taskResponseEvent.getEvent(); + Channel channel = taskResponseEvent.getChannel(); switch (event){ case ACK: - processService.changeTaskState(taskResponseEvent.getState(), - taskResponseEvent.getStartTime(), - taskResponseEvent.getWorkerAddress(), - taskResponseEvent.getExecutePath(), - taskResponseEvent.getLogPath(), - taskResponseEvent.getTaskInstanceId()); + try { + TaskInstance taskInstance = processService.findTaskInstanceById(taskResponseEvent.getTaskInstanceId()); + if (taskInstance != null){ + processService.changeTaskState(taskResponseEvent.getState(), + taskResponseEvent.getStartTime(), + taskResponseEvent.getWorkerAddress(), + taskResponseEvent.getExecutePath(), + taskResponseEvent.getLogPath(), + taskResponseEvent.getTaskInstanceId()); + } + // if taskInstance is null (maybe deleted) . retry will be meaningless . so ack success + DBTaskAckCommand taskAckCommand = new DBTaskAckCommand(ExecutionStatus.SUCCESS.getCode(),taskResponseEvent.getTaskInstanceId()); + channel.writeAndFlush(taskAckCommand.convert2Command()); + }catch (Exception e){ + logger.error("worker ack master error",e); + DBTaskAckCommand taskAckCommand = new DBTaskAckCommand(ExecutionStatus.FAILURE.getCode(),-1); + channel.writeAndFlush(taskAckCommand.convert2Command()); + } break; case RESULT: - processService.changeTaskState(taskResponseEvent.getState(), - taskResponseEvent.getEndTime(), - taskResponseEvent.getProcessId(), - taskResponseEvent.getAppIds(), - taskResponseEvent.getTaskInstanceId(), - taskResponseEvent.getVarPool()); + try { + TaskInstance taskInstance = processService.findTaskInstanceById(taskResponseEvent.getTaskInstanceId()); + if (taskInstance != null){ + processService.changeTaskState(taskResponseEvent.getState(), + taskResponseEvent.getEndTime(), + taskResponseEvent.getProcessId(), + taskResponseEvent.getAppIds(), + taskResponseEvent.getTaskInstanceId(), + taskResponseEvent.getVarPool() + ); + } + // if taskInstance is null (maybe deleted) . retry will be meaningless . so response success + DBTaskResponseCommand taskResponseCommand = new DBTaskResponseCommand(ExecutionStatus.SUCCESS.getCode(),taskResponseEvent.getTaskInstanceId()); + channel.writeAndFlush(taskResponseCommand.convert2Command()); + }catch (Exception e){ + logger.error("worker response master error",e); + DBTaskResponseCommand taskResponseCommand = new DBTaskResponseCommand(ExecutionStatus.FAILURE.getCode(),-1); + channel.writeAndFlush(taskResponseCommand.convert2Command()); + } break; default: throw new IllegalArgumentException("invalid event type : " + event); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/ConditionsTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/ConditionsTaskExecThread.java index 021f10d444..11598d9ace 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/ConditionsTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/ConditionsTaskExecThread.java @@ -39,7 +39,6 @@ import java.util.concurrent.ConcurrentHashMap; public class ConditionsTaskExecThread extends MasterBaseTaskExecThread { - /** * dependent parameters */ @@ -134,7 +133,6 @@ public class ConditionsTaskExecThread extends MasterBaseTaskExecThread { this.dependentParameters = JSONUtils.parseObject(this.taskInstance.getDependency(), DependentParameters.class); } - /** * depend result for depend item * @param item @@ -158,5 +156,4 @@ public class ConditionsTaskExecThread extends MasterBaseTaskExecThread { return dependResult; } - } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/DependentTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/DependentTaskExecThread.java index 319afedd7b..5b56911fd7 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/DependentTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/DependentTaskExecThread.java @@ -149,6 +149,10 @@ public class DependentTaskExecThread extends MasterBaseTaskExecThread { logger.error("process instance not exists , master task exec thread exit"); return true; } + if (checkTaskTimeout()) { + this.checkTimeoutFlag = !alertTimeout(); + handleTimeoutFailed(); + } if(this.cancel || this.processInstance.getState() == ExecutionStatus.READY_STOP){ cancelTaskInstance(); break; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java index ea3ad19950..f5c3708af1 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java @@ -14,14 +14,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.dolphinscheduler.server.master.runner; import static org.apache.dolphinscheduler.common.Constants.UNDERLINE; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; +import org.apache.dolphinscheduler.common.enums.TaskTimeoutStrategy; +import org.apache.dolphinscheduler.common.model.TaskNode; +import org.apache.dolphinscheduler.common.task.TaskTimeoutParameter; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.dao.AlertDao; +import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.server.master.config.MasterConfig; @@ -34,6 +37,11 @@ import java.util.concurrent.Callable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.dolphinscheduler.common.Constants.*; + +import java.util.Date; +import java.util.concurrent.Callable; + /** * master task exec base class @@ -81,10 +89,19 @@ public class MasterBaseTaskExecThread implements Callable { */ private TaskPriorityQueue taskUpdateQueue; + /** + * whether need check task time out. + */ + protected boolean checkTimeoutFlag = false; + + /** + * task timeout parameters + */ + protected TaskTimeoutParameter taskTimeoutParameter; + /** * constructor of MasterBaseTaskExecThread - * - * @param taskInstance task instance + * @param taskInstance task instance */ public MasterBaseTaskExecThread(TaskInstance taskInstance) { this.processService = SpringApplicationContext.getBean(ProcessService.class); @@ -93,6 +110,27 @@ public class MasterBaseTaskExecThread implements Callable { this.taskInstance = taskInstance; this.masterConfig = SpringApplicationContext.getBean(MasterConfig.class); this.taskUpdateQueue = SpringApplicationContext.getBean(TaskPriorityQueueImpl.class); + initTaskParams(); + } + + /** + * init task ordinary parameters + */ + private void initTaskParams() { + initTimeoutParams(); + } + + /** + * init task timeout parameters + */ + private void initTimeoutParams() { + String taskJson = taskInstance.getTaskJson(); + TaskNode taskNode = JSONUtils.parseObject(taskJson, TaskNode.class); + taskTimeoutParameter = taskNode.getTaskTimeoutParameter(); + + if(taskTimeoutParameter.getEnable()){ + checkTimeoutFlag = true; + } } /** @@ -113,7 +151,6 @@ public class MasterBaseTaskExecThread implements Callable { /** * submit master base task exec thread - * * @return TaskInstance */ protected TaskInstance submit() { @@ -154,14 +191,13 @@ public class MasterBaseTaskExecThread implements Callable { return task; } - /** * dispatcht task - * * @param taskInstance taskInstance * @return whether submit task success */ public Boolean dispatchTask(TaskInstance taskInstance) { + try{ if(taskInstance.isConditionsTask() || taskInstance.isDependTask() @@ -198,7 +234,6 @@ public class MasterBaseTaskExecThread implements Callable { } } - /** * buildTaskPriorityInfo * @@ -227,7 +262,6 @@ public class MasterBaseTaskExecThread implements Callable { /** * submit wait complete - * * @return true */ protected Boolean submitWaitComplete() { @@ -236,7 +270,6 @@ public class MasterBaseTaskExecThread implements Callable { /** * call - * * @return boolean * @throws Exception exception */ @@ -246,4 +279,56 @@ public class MasterBaseTaskExecThread implements Callable { return submitWaitComplete(); } + /** + * alert time out + * @return + */ + protected boolean alertTimeout(){ + if( TaskTimeoutStrategy.FAILED == this.taskTimeoutParameter.getStrategy()){ + return true; + } + logger.warn("process id:{} process name:{} task id: {},name:{} execution time out", + processInstance.getId(), processInstance.getName(), taskInstance.getId(), taskInstance.getName()); + // send warn mail + ProcessDefinition processDefine = processService.findProcessDefineById(processInstance.getProcessDefinitionId()); + alertDao.sendTaskTimeoutAlert(processInstance.getWarningGroupId(),processDefine.getReceivers(), + processDefine.getReceiversCc(), processInstance.getId(), processInstance.getName(), + taskInstance.getId(),taskInstance.getName()); + return true; + } + + /** + * handle time out for time out strategy warn&&failed + */ + protected void handleTimeoutFailed(){ + if(TaskTimeoutStrategy.WARN == this.taskTimeoutParameter.getStrategy()){ + return; + } + logger.info("process id:{} name:{} task id:{} name:{} cancel because of timeout.", + processInstance.getId(), processInstance.getName(), taskInstance.getId(), taskInstance.getName()); + this.cancel = true; + } + + /** + * check task remain time valid + * @return + */ + protected boolean checkTaskTimeout(){ + if (!checkTimeoutFlag || taskInstance.getStartTime() == null){ + return false; + } + long remainTime = getRemainTime(taskTimeoutParameter.getInterval() * 60L); + return remainTime <= 0; + } + + /** + * get remain time + * + * @return remain time + */ + protected long getRemainTime(long timeoutSeconds) { + Date startTime = taskInstance.getStartTime(); + long usedTime = (System.currentTimeMillis() - startTime.getTime()) / 1000; + return timeoutSeconds - usedTime; + } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java index 3c28e16651..2bc8031bce 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java @@ -68,6 +68,7 @@ import java.util.Date; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; @@ -348,11 +349,12 @@ public class MasterExecThread implements Runnable { * @throws Exception exception */ private void prepareProcess() throws Exception { - // init task queue - initTaskQueue(); // gen process dag buildFlowDag(); + + // init task queue + initTaskQueue(); logger.info("prepare process :{} end", processInstance.getId()); } @@ -407,6 +409,9 @@ public class MasterExecThread implements Runnable { if(task.isTaskComplete()){ completeTaskList.put(task.getName(), task); } + if(task.isConditionsTask() || DagHelper.haveConditionsAfterNode(task.getName(), dag)){ + continue; + } if(task.getState().typeIsFailure() && !task.taskCanRetry()){ errorTaskList.put(task.getName(), task); } @@ -498,6 +503,9 @@ public class MasterExecThread implements Runnable { // task instance whether alert taskInstance.setAlertFlag(Flag.NO); + // task instance start time + taskInstance.setStartTime(null); + // task instance flag taskInstance.setFlag(Flag.YES); @@ -532,132 +540,13 @@ public class MasterExecThread implements Runnable { return taskInstance; } - - - /** - * if all of the task dependence are skip, skip it too. - * @param taskNode - * @return - */ - private boolean isTaskNodeNeedSkip(TaskNode taskNode){ - if(CollectionUtils.isEmpty(taskNode.getDepList())){ - return false; - } - for(String depNode : taskNode.getDepList()){ - if(!skipTaskNodeList.containsKey(depNode)){ - return false; - } - } - return true; - } - - /** - * set task node skip if dependence all skip - * @param taskNodesSkipList - */ - private void setTaskNodeSkip(List taskNodesSkipList){ - for(String skipNode : taskNodesSkipList){ - skipTaskNodeList.putIfAbsent(skipNode, dag.getNode(skipNode)); - Collection postNodeList = DagHelper.getStartVertex(skipNode, dag, completeTaskList); - List postSkipList = new ArrayList<>(); - for(String post : postNodeList){ - TaskNode postNode = dag.getNode(post); - if(isTaskNodeNeedSkip(postNode)){ - postSkipList.add(post); - } - } - setTaskNodeSkip(postSkipList); - } - } - - - /** - * parse condition task find the branch process - * set skip flag for another one. - * @param nodeName - * @return - */ - private List parseConditionTask(String nodeName){ - List conditionTaskList = new ArrayList<>(); - TaskNode taskNode = dag.getNode(nodeName); - if(!taskNode.isConditionsTask()){ - return conditionTaskList; - } - ConditionsParameters conditionsParameters = - JSONUtils.parseObject(taskNode.getConditionResult(), ConditionsParameters.class); - - TaskInstance taskInstance = completeTaskList.get(nodeName); - if(taskInstance == null){ - logger.error("task instance {} cannot find, please check it!", nodeName); - return conditionTaskList; - } - - if(taskInstance.getState().typeIsSuccess()){ - conditionTaskList = conditionsParameters.getSuccessNode(); - setTaskNodeSkip(conditionsParameters.getFailedNode()); - }else if(taskInstance.getState().typeIsFailure()){ - conditionTaskList = conditionsParameters.getFailedNode(); - setTaskNodeSkip(conditionsParameters.getSuccessNode()); - }else{ - conditionTaskList.add(nodeName); - } - return conditionTaskList; - } - - /** - * parse post node list of previous node - * if condition node: return process according to the settings - * if post node completed, return post nodes of the completed node - * @param previousNodeName - * @return - */ - private List parsePostNodeList(String previousNodeName){ - List postNodeList = new ArrayList<>(); - - TaskNode taskNode = dag.getNode(previousNodeName); - if(taskNode != null && taskNode.isConditionsTask()){ - return parseConditionTask(previousNodeName); - } - Collection postNodeCollection = DagHelper.getStartVertex(previousNodeName, dag, completeTaskList); - List postSkipList = new ArrayList<>(); - // delete success node, parse the past nodes - // if conditions node, - // 1. parse the branch process according the conditions setting - // 2. set skip flag on anther branch process - for(String postNode : postNodeCollection){ - if(completeTaskList.containsKey(postNode)){ - TaskInstance postTaskInstance = completeTaskList.get(postNode); - if(dag.getNode(postNode).isConditionsTask()){ - List conditionTaskNodeList = parseConditionTask(postNode); - for(String conditions : conditionTaskNodeList){ - postNodeList.addAll(parsePostNodeList(conditions)); - } - }else if(postTaskInstance.getState().typeIsSuccess()){ - postNodeList.addAll(parsePostNodeList(postNode)); - }else{ - postNodeList.add(postNode); - } - - }else if(isTaskNodeNeedSkip(dag.getNode(postNode))){ - postSkipList.add(postNode); - setTaskNodeSkip(postSkipList); - postSkipList.clear(); - }else{ - postNodeList.add(postNode); - } - } - return postNodeList; - } - /** * submit post node * @param parentNodeName parent node name */ private Map propToValue = new ConcurrentHashMap(); private void submitPostNode(String parentNodeName){ - - List submitTaskNodeList = parsePostNodeList(parentNodeName); - + Set submitTaskNodeList = DagHelper.parsePostNodes(parentNodeName, skipTaskNodeList, dag, completeTaskList); List taskInstances = new ArrayList<>(); for(String taskNode : submitTaskNodeList){ try { @@ -702,7 +591,6 @@ public class MasterExecThread implements Runnable { if(startNodes.contains(taskName)){ return DependResult.SUCCESS; } - TaskNode taskNode = dag.getNode(taskName); List depNameList = taskNode.getDepList(); for(String depsNode : depNameList ){ @@ -716,23 +604,42 @@ public class MasterExecThread implements Runnable { return DependResult.WAITING; } ExecutionStatus depTaskState = completeTaskList.get(depsNode).getState(); - // conditions task would not return failed. - if(depTaskState.typeIsFailure() - && !DagHelper.haveConditionsAfterNode(depsNode, dag ) - && !dag.getNode(depsNode).isConditionsTask()){ - return DependResult.FAILED; - } - if(depTaskState.typeIsPause() || depTaskState.typeIsCancel()){ return DependResult.WAITING; } + // ignore task state if current task is condition + if(taskNode.isConditionsTask()){ + continue; + } + if(!dependTaskSuccess(depsNode, taskName)){ + return DependResult.FAILED; + } } - logger.info("taskName: {} completeDependTaskList: {}", taskName, Arrays.toString(completeTaskList.keySet().toArray())); - return DependResult.SUCCESS; } + /** + * depend node is completed, but here need check the condition task branch is the next node + * @param dependNodeName + * @param nextNodeName + * @return + */ + private boolean dependTaskSuccess(String dependNodeName, String nextNodeName){ + if(dag.getNode(dependNodeName).isConditionsTask()){ + //condition task need check the branch to run + List nextTaskList = DagHelper.parseConditionTask(dependNodeName, skipTaskNodeList, dag, completeTaskList); + if(!nextTaskList.contains(nextNodeName)){ + return false; + } + }else { + ExecutionStatus depTaskState = completeTaskList.get(dependNodeName).getState(); + if(depTaskState.typeIsFailure()){ + return false; + } + } + return true; + } /** * query task instance by complete state @@ -889,6 +796,24 @@ public class MasterExecThread implements Runnable { return state; } + /** + * whether standby task list have retry tasks + * @return + */ + private boolean retryTaskExists() { + + boolean result = false; + + for(String taskName : readyToSubmitTaskList.keySet()){ + TaskInstance task = readyToSubmitTaskList.get(taskName); + if(task.getState().typeIsFailure()){ + result = true; + break; + } + } + return result; + } + /** * whether complement end * @return Boolean whether is complement end @@ -976,7 +901,7 @@ public class MasterExecThread implements Runnable { // submit start node submitPostNode(null); boolean sendTimeWarning = false; - while(!processInstance.isProcessInstanceStop()){ + while(!processInstance.isProcessInstanceStop() && Stopper.isRunning()){ // send warning email if process time out. if(!sendTimeWarning && checkProcessTimeOut(processInstance) ){ diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java index 72ee0fcb89..b23ad45774 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java @@ -121,15 +121,6 @@ public class MasterTaskExecThread extends MasterBaseTaskExecThread { taskInstance = processService.findTaskInstanceById(taskInstance.getId()); logger.info("wait task: process id: {}, task id:{}, task name:{} complete", this.taskInstance.getProcessInstanceId(), this.taskInstance.getId(), this.taskInstance.getName()); - // task time out - boolean checkTimeout = false; - TaskTimeoutParameter taskTimeoutParameter = getTaskTimeoutParameter(); - if(taskTimeoutParameter.getEnable()){ - TaskTimeoutStrategy strategy = taskTimeoutParameter.getStrategy(); - if(strategy == TaskTimeoutStrategy.WARN || strategy == TaskTimeoutStrategy.WARNFAILED){ - checkTimeout = true; - } - } while (Stopper.isRunning()){ try { @@ -150,18 +141,8 @@ public class MasterTaskExecThread extends MasterBaseTaskExecThread { taskInstanceCacheManager.removeByTaskInstanceId(taskInstance.getId()); break; } - if(checkTimeout){ - long remainTime = DateUtils.getRemainTime(taskInstance.getStartTime(), taskTimeoutParameter.getInterval() * 60L); - if (remainTime < 0) { - logger.warn("task id: {} execution time out",taskInstance.getId()); - // process define - ProcessDefinition processDefine = processService.findProcessDefineById(processInstance.getProcessDefinitionId()); - // send warn mail - alertDao.sendTaskTimeoutAlert(processInstance.getWarningGroupId(),processDefine.getReceivers(), - processDefine.getReceiversCc(), processInstance.getId(), processInstance.getName(), - taskInstance.getId(),taskInstance.getName()); - checkTimeout = false; - } + if (checkTaskTimeout()) { + this.checkTimeoutFlag = !alertTimeout(); } // updateProcessInstance task instance taskInstance = processService.findTaskInstanceById(taskInstance.getId()); @@ -248,13 +229,4 @@ public class MasterTaskExecThread extends MasterBaseTaskExecThread { return true; } - /** - * get task timeout parameter - * @return TaskTimeoutParameter - */ - private TaskTimeoutParameter getTaskTimeoutParameter(){ - String taskJson = taskInstance.getTaskJson(); - TaskNode taskNode = JSONUtils.parseObject(taskJson, TaskNode.class); - return taskNode.getTaskTimeoutParameter(); - } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/SubProcessTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/SubProcessTaskExecThread.java index 2b1508dc44..74b1c2f271 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/SubProcessTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/SubProcessTaskExecThread.java @@ -130,19 +130,20 @@ public class SubProcessTaskExecThread extends MasterBaseTaskExecThread { while (Stopper.isRunning()) { // waiting for subflow process instance establishment if (subProcessInstance == null) { - Thread.sleep(Constants.SLEEP_TIME_MILLIS); - if(!setTaskInstanceState()){ continue; } } subProcessInstance = processService.findProcessInstanceById(subProcessInstance.getId()); + if (checkTaskTimeout()) { + this.checkTimeoutFlag = !alertTimeout(); + handleTimeoutFailed(); + } updateParentProcessState(); if (subProcessInstance.getState().typeIsFinished()){ break; } - if(this.processInstance.getState() == ExecutionStatus.READY_PAUSE){ // parent process "ready to pause" , child process "pause" pauseSubProcess(); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/AlertManager.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/AlertManager.java index 58ade83d68..6b8d926c0f 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/AlertManager.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/AlertManager.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.server.utils; import org.apache.dolphinscheduler.common.enums.AlertType; import org.apache.dolphinscheduler.common.enums.CommandType; +import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.common.enums.ShowType; import org.apache.dolphinscheduler.common.enums.WarningType; import org.apache.dolphinscheduler.common.utils.JSONUtils; @@ -197,6 +198,9 @@ public class AlertManager { public void sendAlertProcessInstance(ProcessInstance processInstance, List taskInstances) { + if(Flag.YES == processInstance.getIsSubProcess()){ + return; + } boolean sendWarnning = false; WarningType warningType = processInstance.getWarningType(); switch (warningType) { diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/DependentExecute.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/DependentExecute.java index 71c7d959e1..7f76baaa52 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/DependentExecute.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/DependentExecute.java @@ -104,9 +104,7 @@ public class DependentExecute { ProcessInstance processInstance = findLastProcessInterval(dependentItem.getDefinitionId(), dateInterval); if(processInstance == null){ - logger.error("cannot find the right process instance: definition id:{}, start:{}, end:{}", - dependentItem.getDefinitionId(), dateInterval.getStartTime(), dateInterval.getEndTime() ); - return DependResult.FAILED; + return DependResult.WAITING; } // need to check workflow for updates, so get all task and check the task state if(dependentItem.getDepTasks().equals(Constants.DEPENDENT_ALL)){ diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/UDFUtils.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/UDFUtils.java index 3a8c8fe7d6..1bc79ab94e 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/UDFUtils.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/UDFUtils.java @@ -70,14 +70,16 @@ public class UDFUtils { */ private static void buildJarSql(List sqls, Map udfFuncTenantCodeMap) { String defaultFS = HadoopUtils.getInstance().getConfiguration().get(Constants.FS_DEFAULTFS); - + String resourceFullName; Set> entries = udfFuncTenantCodeMap.entrySet(); for (Map.Entry entry:entries){ String uploadPath = HadoopUtils.getHdfsUdfDir(entry.getValue()); if (!uploadPath.startsWith("hdfs:")) { uploadPath = defaultFS + uploadPath; } - sqls.add(String.format("add jar %s%s", uploadPath, entry.getKey().getResourceName())); + resourceFullName = entry.getKey().getResourceName(); + resourceFullName = resourceFullName.startsWith("/") ? resourceFullName : String.format("/%s",resourceFullName); + sqls.add(String.format("add jar %s%s", uploadPath, resourceFullName)); } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java index f0833cb7e0..6895de3d4a 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java @@ -22,9 +22,12 @@ import org.apache.dolphinscheduler.remote.NettyRemotingServer; import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.config.NettyServerConfig; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; +import org.apache.dolphinscheduler.server.worker.processor.DBTaskAckProcessor; +import org.apache.dolphinscheduler.server.worker.processor.DBTaskResponseProcessor; import org.apache.dolphinscheduler.server.worker.processor.TaskExecuteProcessor; import org.apache.dolphinscheduler.server.worker.processor.TaskKillProcessor; import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistry; +import org.apache.dolphinscheduler.server.worker.runner.RetryReportTaskStatusThread; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -70,6 +73,9 @@ public class WorkerServer { @Autowired private SpringApplicationContext springApplicationContext; + @Autowired + private RetryReportTaskStatusThread retryReportTaskStatusThread; + /** * worker server startup * @@ -95,11 +101,16 @@ public class WorkerServer { this.nettyRemotingServer = new NettyRemotingServer(serverConfig); this.nettyRemotingServer.registerProcessor(CommandType.TASK_EXECUTE_REQUEST, new TaskExecuteProcessor()); this.nettyRemotingServer.registerProcessor(CommandType.TASK_KILL_REQUEST, new TaskKillProcessor()); + this.nettyRemotingServer.registerProcessor(CommandType.DB_TASK_ACK, new DBTaskAckProcessor()); + this.nettyRemotingServer.registerProcessor(CommandType.DB_TASK_RESPONSE, new DBTaskResponseProcessor()); this.nettyRemotingServer.start(); // worker registry this.workerRegistry.registry(); + // retry report task status + this.retryReportTaskStatusThread.start(); + /** * register hooks, which are called before the process exits */ diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/cache/ResponceCache.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/cache/ResponceCache.java new file mode 100644 index 0000000000..3639b8eba3 --- /dev/null +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/cache/ResponceCache.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.worker.cache; + +import org.apache.dolphinscheduler.common.enums.Event; +import org.apache.dolphinscheduler.remote.command.Command; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Responce Cache : cache worker send master result + */ +public class ResponceCache { + + private static final ResponceCache instance = new ResponceCache(); + + private ResponceCache(){} + + public static ResponceCache get(){ + return instance; + } + + private Map ackCache = new ConcurrentHashMap<>(); + private Map responseCache = new ConcurrentHashMap<>(); + + + /** + * cache response + * @param taskInstanceId taskInstanceId + * @param command command + * @param event event ACK/RESULT + */ + public void cache(Integer taskInstanceId, Command command, Event event){ + switch (event){ + case ACK: + ackCache.put(taskInstanceId,command); + break; + case RESULT: + responseCache.put(taskInstanceId,command); + break; + default: + throw new IllegalArgumentException("invalid event type : " + event); + } + } + + + /** + * remove ack cache + * @param taskInstanceId taskInstanceId + */ + public void removeAckCache(Integer taskInstanceId){ + ackCache.remove(taskInstanceId); + } + + /** + * remove reponse cache + * @param taskInstanceId taskInstanceId + */ + public void removeResponseCache(Integer taskInstanceId){ + responseCache.remove(taskInstanceId); + } + + /** + * getAckCache + * @return getAckCache + */ + public Map getAckCache(){ + return ackCache; + } + + /** + * getResponseCache + * @return getResponseCache + */ + public Map getResponseCache(){ + return responseCache; + } +} diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/DBTaskAckProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/DBTaskAckProcessor.java new file mode 100644 index 0000000000..551661722f --- /dev/null +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/DBTaskAckProcessor.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.worker.processor; + +import io.netty.channel.Channel; +import org.apache.dolphinscheduler.common.enums.ExecutionStatus; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.Preconditions; +import org.apache.dolphinscheduler.remote.command.*; +import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; +import org.apache.dolphinscheduler.server.worker.cache.ResponceCache; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * db task ack processor + */ +public class DBTaskAckProcessor implements NettyRequestProcessor { + + private final Logger logger = LoggerFactory.getLogger(DBTaskAckProcessor.class); + + + @Override + public void process(Channel channel, Command command) { + Preconditions.checkArgument(CommandType.DB_TASK_ACK == command.getType(), + String.format("invalid command type : %s", command.getType())); + + DBTaskAckCommand taskAckCommand = JSONUtils.parseObject( + command.getBody(), DBTaskAckCommand.class); + + if (taskAckCommand == null){ + return; + } + + if (taskAckCommand.getStatus() == ExecutionStatus.SUCCESS.getCode()){ + ResponceCache.get().removeAckCache(taskAckCommand.getTaskInstanceId()); + } + } + + +} diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/DBTaskResponseProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/DBTaskResponseProcessor.java new file mode 100644 index 0000000000..e382245b63 --- /dev/null +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/DBTaskResponseProcessor.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.worker.processor; + +import io.netty.channel.Channel; +import org.apache.dolphinscheduler.common.enums.ExecutionStatus; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.Preconditions; +import org.apache.dolphinscheduler.remote.command.Command; +import org.apache.dolphinscheduler.remote.command.CommandType; +import org.apache.dolphinscheduler.remote.command.DBTaskResponseCommand; +import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; +import org.apache.dolphinscheduler.server.worker.cache.ResponceCache; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * db task response processor + */ +public class DBTaskResponseProcessor implements NettyRequestProcessor { + + private final Logger logger = LoggerFactory.getLogger(DBTaskResponseProcessor.class); + + + @Override + public void process(Channel channel, Command command) { + Preconditions.checkArgument(CommandType.DB_TASK_RESPONSE == command.getType(), + String.format("invalid command type : %s", command.getType())); + + DBTaskResponseCommand taskResponseCommand = JSONUtils.parseObject( + command.getBody(), DBTaskResponseCommand.class); + + if (taskResponseCommand == null){ + return; + } + + if (taskResponseCommand.getStatus() == ExecutionStatus.SUCCESS.getCode()){ + ResponceCache.get().removeResponseCache(taskResponseCommand.getTaskInstanceId()); + } + } + + +} diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/NettyRemoteChannel.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/NettyRemoteChannel.java index b1b67affcc..6e2fdeb5d9 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/NettyRemoteChannel.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/NettyRemoteChannel.java @@ -50,6 +50,12 @@ public class NettyRemoteChannel { this.opaque = opaque; } + public NettyRemoteChannel(Channel channel) { + this.channel = channel; + this.host = ChannelUtils.toAddress(channel); + this.opaque = -1; + } + public Channel getChannel() { return channel; } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackService.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackService.java index 4aeb6feffb..eda4da6dd9 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackService.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackService.java @@ -25,6 +25,7 @@ import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.remote.NettyRemotingClient; import org.apache.dolphinscheduler.remote.command.Command; +import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.config.NettyClientConfig; import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.server.registry.ZookeeperRegistryCenter; @@ -66,6 +67,8 @@ public class TaskCallbackService { public TaskCallbackService(){ final NettyClientConfig clientConfig = new NettyClientConfig(); this.nettyRemotingClient = new NettyRemotingClient(clientConfig); + this.nettyRemotingClient.registerProcessor(CommandType.DB_TASK_ACK, new DBTaskAckProcessor()); + this.nettyRemotingClient.registerProcessor(CommandType.DB_TASK_RESPONSE, new DBTaskResponseProcessor()); } /** @@ -83,20 +86,21 @@ public class TaskCallbackService { * @return callback channel */ private NettyRemoteChannel getRemoteChannel(int taskInstanceId){ + Channel newChannel; NettyRemoteChannel nettyRemoteChannel = REMOTE_CHANNELS.get(taskInstanceId); - if(nettyRemoteChannel == null){ - throw new IllegalArgumentException("nettyRemoteChannel is empty, should call addRemoteChannel first"); - } - if(nettyRemoteChannel.isActive()){ - return nettyRemoteChannel; - } - Channel newChannel = nettyRemotingClient.getChannel(nettyRemoteChannel.getHost()); - if(newChannel != null){ - return getRemoteChannel(newChannel, nettyRemoteChannel.getOpaque(), taskInstanceId); + if(nettyRemoteChannel != null){ + if(nettyRemoteChannel.isActive()){ + return nettyRemoteChannel; + } + newChannel = nettyRemotingClient.getChannel(nettyRemoteChannel.getHost()); + if(newChannel != null){ + return getRemoteChannel(newChannel, nettyRemoteChannel.getOpaque(), taskInstanceId); + } + logger.warn("original master : {} for task : {} is not reachable, random select master", + nettyRemoteChannel.getHost(), + taskInstanceId); } - logger.warn("original master : {} for task : {} is not reachable, random select master", - nettyRemoteChannel.getHost(), - taskInstanceId); + Set masterNodes = null; int ntries = 0; while (Stopper.isRunning()) { @@ -116,7 +120,7 @@ public class TaskCallbackService { for (String masterNode : masterNodes) { newChannel = nettyRemotingClient.getChannel(Host.of(masterNode)); if (newChannel != null) { - return getRemoteChannel(newChannel, nettyRemoteChannel.getOpaque(), taskInstanceId); + return getRemoteChannel(newChannel,taskInstanceId); } } masterNodes = null; @@ -138,6 +142,12 @@ public class TaskCallbackService { return remoteChannel; } + private NettyRemoteChannel getRemoteChannel(Channel newChannel, int taskInstanceId){ + NettyRemoteChannel remoteChannel = new NettyRemoteChannel(newChannel); + addRemoteChannel(taskInstanceId, remoteChannel); + return remoteChannel; + } + /** * remove callback channels * @param taskInstanceId taskInstanceId diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java index 13a9dca9d6..6080baf3cb 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.server.worker.processor; +import org.apache.dolphinscheduler.common.enums.Event; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.TaskType; import org.apache.dolphinscheduler.common.thread.ThreadUtils; @@ -34,19 +35,18 @@ import org.apache.dolphinscheduler.remote.command.TaskExecuteRequestCommand; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; import org.apache.dolphinscheduler.server.utils.LogUtils; +import org.apache.dolphinscheduler.server.worker.cache.ResponceCache; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.runner.TaskExecuteThread; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import java.util.Date; import java.util.Optional; -import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.github.rholder.retry.RetryException; import io.netty.channel.Channel; @@ -100,15 +100,17 @@ public class TaskExecuteProcessor implements NettyRequestProcessor { logger.error("task execution context is null"); return; } - - taskExecutionContext.setHost(NetUtils.getHost() + ":" + workerConfig.getListenPort()); - // custom logger Logger taskLogger = LoggerFactory.getLogger(LoggerUtils.buildTaskId(LoggerUtils.TASK_LOGGER_INFO_PREFIX, taskExecutionContext.getProcessDefineId(), taskExecutionContext.getProcessInstanceId(), taskExecutionContext.getTaskInstanceId())); + taskExecutionContext.setHost(NetUtils.getHost() + ":" + workerConfig.getListenPort()); + taskExecutionContext.setStartTime(new Date()); + taskExecutionContext.setLogPath(LogUtils.getTaskLogPath(taskExecutionContext)); + taskExecutionContext.setCurrentExecutionStatus(ExecutionStatus.RUNNING_EXECUTION); + // local execute path String execLocalPath = getExecLocalPath(taskExecutionContext); logger.info("task instance local execute path : {} ", execLocalPath); @@ -126,27 +128,17 @@ public class TaskExecuteProcessor implements NettyRequestProcessor { taskCallbackService.addRemoteChannel(taskExecutionContext.getTaskInstanceId(), new NettyRemoteChannel(channel, command.getOpaque())); - if (DateUtils.getRemainTime(taskExecutionContext.getFirstSubmitTime(), taskExecutionContext.getDelayTime() * 60L) > 0) { - taskExecutionContext.setCurrentExecutionStatus(ExecutionStatus.DELAY_EXECUTION); - taskExecutionContext.setStartTime(null); - } else { - taskExecutionContext.setCurrentExecutionStatus(ExecutionStatus.RUNNING_EXECUTION); - taskExecutionContext.setStartTime(new Date()); - } + this.doAck(taskExecutionContext); - // tell master the status of this task (RUNNING_EXECUTION or DELAY_EXECUTION) - final Command ackCommand = buildAckCommand(taskExecutionContext).convert2Command(); + // submit task + workerExecService.submit(new TaskExecuteThread(taskExecutionContext, taskCallbackService, taskLogger)); + } - try { - RetryerUtils.retryCall(() -> { - taskCallbackService.sendAck(taskExecutionContext.getTaskInstanceId(),ackCommand); - return Boolean.TRUE; - }); - // submit task - workerExecService.submit(new TaskExecuteThread(taskExecutionContext, taskCallbackService, taskLogger)); - } catch (ExecutionException | RetryException e) { - logger.error(e.getMessage(), e); - } + private void doAck(TaskExecutionContext taskExecutionContext){ + // tell master that task is in executing + TaskExecuteAckCommand ackCommand = buildAckCommand(taskExecutionContext); + ResponceCache.get().cache(taskExecutionContext.getTaskInstanceId(),ackCommand.convert2Command(),Event.ACK); + taskCallbackService.sendAck(taskExecutionContext.getTaskInstanceId(), ackCommand.convert2Command()); } /** diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/RetryReportTaskStatusThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/RetryReportTaskStatusThread.java new file mode 100644 index 0000000000..ec79238d39 --- /dev/null +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/RetryReportTaskStatusThread.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.worker.runner; + +import org.apache.dolphinscheduler.common.thread.Stopper; + +import org.apache.dolphinscheduler.common.thread.ThreadUtils; +import org.apache.dolphinscheduler.remote.command.Command; +import org.apache.dolphinscheduler.server.worker.cache.ResponceCache; +import org.apache.dolphinscheduler.server.worker.processor.TaskCallbackService; +import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import java.util.Map; + +/** + * Retry Report Task Status Thread + */ +@Component +public class RetryReportTaskStatusThread implements Runnable { + + private final Logger logger = LoggerFactory.getLogger(RetryReportTaskStatusThread.class); + + /** + * every 5 minutes + */ + private static long RETRY_REPORT_TASK_STATUS_INTERVAL = 5 * 60 * 1000L; + /** + * task callback service + */ + private final TaskCallbackService taskCallbackService; + + public void start(){ + Thread thread = new Thread(this,"RetryReportTaskStatusThread"); + thread.start(); + } + + public RetryReportTaskStatusThread(){ + this.taskCallbackService = SpringApplicationContext.getBean(TaskCallbackService.class); + } + + /** + * retry ack/response + */ + @Override + public void run() { + ResponceCache responceCache = ResponceCache.get(); + + while (Stopper.isRunning()){ + + // sleep 5 minutes + ThreadUtils.sleep(RETRY_REPORT_TASK_STATUS_INTERVAL); + + try { + if (!responceCache.getAckCache().isEmpty()){ + Map ackCache = responceCache.getAckCache(); + for (Map.Entry entry : ackCache.entrySet()){ + Integer taskInstanceId = entry.getKey(); + Command ackCommand = entry.getValue(); + taskCallbackService.sendAck(taskInstanceId,ackCommand); + } + } + + if (!responceCache.getResponseCache().isEmpty()){ + Map responseCache = responceCache.getResponseCache(); + for (Map.Entry entry : responseCache.entrySet()){ + Integer taskInstanceId = entry.getKey(); + Command responseCommand = entry.getValue(); + taskCallbackService.sendResult(taskInstanceId,responseCommand); + } + } + }catch (Exception e){ + logger.warn("retry report task status error", e); + } + } + } +} diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java index 58f743303c..6baeae9b7b 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.server.worker.runner; import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.Event; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.TaskType; import org.apache.dolphinscheduler.common.model.TaskNode; @@ -32,6 +33,7 @@ import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.TaskExecuteAckCommand; import org.apache.dolphinscheduler.remote.command.TaskExecuteResponseCommand; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; +import org.apache.dolphinscheduler.server.worker.cache.ResponceCache; import org.apache.dolphinscheduler.server.worker.cache.TaskExecutionContextCacheManager; import org.apache.dolphinscheduler.server.worker.cache.impl.TaskExecutionContextCacheManagerImpl; import org.apache.dolphinscheduler.server.worker.processor.TaskCallbackService; @@ -165,14 +167,10 @@ public class TaskExecuteThread implements Runnable { responseCommand.setProcessId(task.getProcessId()); responseCommand.setAppIds(task.getAppIds()); } finally { - try { - taskExecutionContext.setCurrentExecutionStatus(ExecutionStatus.of(responseCommand.getStatus())); - taskExecutionContextCacheManager.removeByTaskInstanceId(taskExecutionContext.getTaskInstanceId()); - taskCallbackService.sendResult(taskExecutionContext.getTaskInstanceId(), responseCommand.convert2Command()); - } catch (Exception e) { - ThreadUtils.sleep(Constants.SLEEP_TIME_MILLIS); - taskCallbackService.sendResult(taskExecutionContext.getTaskInstanceId(), responseCommand.convert2Command()); - } + taskExecutionContextCacheManager.removeByTaskInstanceId(taskExecutionContext.getTaskInstanceId()); + ResponceCache.get().cache(taskExecutionContext.getTaskInstanceId(),responseCommand.convert2Command(),Event.RESULT); + taskCallbackService.sendResult(taskExecutionContext.getTaskInstanceId(), responseCommand.convert2Command()); + } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java index dddd1a64b7..89af95278c 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java @@ -16,46 +16,34 @@ */ package org.apache.dolphinscheduler.server.worker.task; -import static org.apache.dolphinscheduler.common.Constants.EXIT_CODE_FAILURE; -import static org.apache.dolphinscheduler.common.Constants.EXIT_CODE_SUCCESS; - import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.common.thread.ThreadUtils; -import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.HadoopUtils; -import org.apache.dolphinscheduler.common.utils.LoggerUtils; -import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; -import org.apache.dolphinscheduler.common.utils.process.ProcessBuilderForWin32; +import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; import org.apache.dolphinscheduler.server.utils.ProcessUtils; import org.apache.dolphinscheduler.server.worker.cache.TaskExecutionContextCacheManager; import org.apache.dolphinscheduler.server.worker.cache.impl.TaskExecutionContextCacheManagerImpl; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; +import org.apache.dolphinscheduler.service.process.ProcessService; +import org.slf4j.Logger; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStreamReader; +import java.io.*; import java.lang.reflect.Field; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; +import java.util.*; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.slf4j.Logger; - -import com.sun.jna.platform.win32.Kernel32; -import com.sun.jna.platform.win32.WinNT; +import static org.apache.dolphinscheduler.common.Constants.EXIT_CODE_FAILURE; +import static org.apache.dolphinscheduler.common.Constants.EXIT_CODE_SUCCESS; /** * abstract command executor @@ -117,45 +105,24 @@ public abstract class AbstractCommandExecutor { // setting up user to run commands List command = new LinkedList<>(); - if (OSUtils.isWindows()) { - //init process builder - ProcessBuilderForWin32 processBuilder = new ProcessBuilderForWin32(); - // setting up a working directory - processBuilder.directory(new File(taskExecutionContext.getExecutePath())); - // setting up a username and password - processBuilder.user(taskExecutionContext.getTenantCode(), StringUtils.EMPTY); - // merge error information to standard output stream - processBuilder.redirectErrorStream(true); - - // setting up user to run commands - command.add(commandInterpreter()); - command.add("/c"); - command.addAll(commandOptions()); - command.add(commandFile); - - // setting commands - processBuilder.command(command); - process = processBuilder.start(); - } else { - //init process builder - ProcessBuilder processBuilder = new ProcessBuilder(); - // setting up a working directory - processBuilder.directory(new File(taskExecutionContext.getExecutePath())); - // merge error information to standard output stream - processBuilder.redirectErrorStream(true); - - // setting up user to run commands - command.add("sudo"); - command.add("-u"); - command.add(taskExecutionContext.getTenantCode()); - command.add(commandInterpreter()); - command.addAll(commandOptions()); - command.add(commandFile); - - // setting commands - processBuilder.command(command); - process = processBuilder.start(); - } + //init process builder + ProcessBuilder processBuilder = new ProcessBuilder(); + // setting up a working directory + processBuilder.directory(new File(taskExecutionContext.getExecutePath())); + // merge error information to standard output stream + processBuilder.redirectErrorStream(true); + + // setting up user to run commands + command.add("sudo"); + command.add("-u"); + command.add(taskExecutionContext.getTenantCode()); + command.add(commandInterpreter()); + command.addAll(commandOptions()); + command.add(commandFile); + + // setting commands + processBuilder.command(command); + process = processBuilder.start(); // print command printCommand(command); @@ -238,7 +205,7 @@ public abstract class AbstractCommandExecutor { public String getVarPool() { return varPool.toString(); } - + /** * cancel application * @throws Exception exception @@ -316,20 +283,30 @@ public abstract class AbstractCommandExecutor { * @param commands process builder */ private void printCommand(List commands) { - String cmdStr = ProcessUtils.buildCommandStr(commands); - logger.info("task run command:\n{}", cmdStr); + String cmdStr; + + try { + cmdStr = ProcessUtils.buildCommandStr(commands); + logger.info("task run command:\n{}", cmdStr); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } } /** * clear */ private void clear() { + + List markerList = new ArrayList<>(); + markerList.add(ch.qos.logback.classic.ClassicConstants.FINALIZE_SESSION_MARKER.toString()); + if (!logBuffer.isEmpty()) { // log handle logHandler.accept(logBuffer); - logBuffer.clear(); } + logHandler.accept(markerList); } /** @@ -479,12 +456,13 @@ public abstract class AbstractCommandExecutor { /** - * get remain time?s? + * get remain time（s） * * @return remain time */ private long getRemaintime() { - long remainTime = DateUtils.getRemainTime(taskExecutionContext.getStartTime(), taskExecutionContext.getTaskTimeout()); + long usedTime = (System.currentTimeMillis() - taskExecutionContext.getStartTime().getTime()) / 1000; + long remainTime = taskExecutionContext.getTaskTimeout() - usedTime; if (remainTime < 0) { throw new RuntimeException("task execution time out"); @@ -506,12 +484,7 @@ public abstract class AbstractCommandExecutor { Field f = process.getClass().getDeclaredField(Constants.PID); f.setAccessible(true); - if (OSUtils.isWindows()) { - WinNT.HANDLE handle = (WinNT.HANDLE) f.get(process); - processId = Kernel32.INSTANCE.GetProcessId(handle); - } else { - processId = f.getInt(process); - } + processId = f.getInt(process); } catch (Throwable e) { logger.error(e.getMessage(), e); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractTask.java index 1a66349817..fe60f4a82c 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractTask.java @@ -17,7 +17,10 @@ package org.apache.dolphinscheduler.server.worker.task; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.enums.*; +import org.apache.dolphinscheduler.common.enums.CommandType; +import org.apache.dolphinscheduler.common.enums.ExecutionStatus; +import org.apache.dolphinscheduler.common.enums.TaskRecordStatus; +import org.apache.dolphinscheduler.common.enums.TaskType; import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.conditions.ConditionsParameters; @@ -30,18 +33,18 @@ import org.apache.dolphinscheduler.common.task.shell.ShellParameters; import org.apache.dolphinscheduler.common.task.spark.SparkParameters; import org.apache.dolphinscheduler.common.task.sql.SqlParameters; import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters; -import org.apache.dolphinscheduler.common.utils.*; +import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.dao.TaskRecordDao; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; import org.apache.dolphinscheduler.server.utils.ParamUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; -import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; +import static ch.qos.logback.classic.ClassicConstants.FINALIZE_SESSION_MARKER; + /** * executive task */ @@ -51,7 +54,7 @@ public abstract class AbstractTask { * varPool string */ protected String varPool; - + /** * taskExecutionContext **/ @@ -123,17 +126,20 @@ public abstract class AbstractTask { */ public void logHandle(List logs) { // note that the "new line" is added here to facilitate log parsing - logger.info(" -> {}", String.join("\n\t", logs)); + if (logs.contains(FINALIZE_SESSION_MARKER.toString())) { + logger.info(FINALIZE_SESSION_MARKER, FINALIZE_SESSION_MARKER.toString()); + } else { + logger.info(" -> {}", String.join("\n\t", logs)); + } } public void setVarPool(String varPool) { this.varPool = varPool; } - public String getVarPool() { return varPool; } - + /** * get exit status code * @return exit status code diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java index 49bfb5f9a8..f6d4d0d4bb 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java @@ -83,10 +83,10 @@ public class ZKMasterClient extends AbstractZKClient { // self tolerant if (getActiveMasterNum() == 1) { - failoverWorker(null, true); - failoverMaster(null); + removeZKNodePath(null, ZKNodeType.MASTER, true); + removeZKNodePath(null, ZKNodeType.WORKER, true); } - + registerListener(); } catch (Exception e) { logger.error("master start up exception", e); } finally { @@ -133,9 +133,16 @@ public class ZKMasterClient extends AbstractZKClient { mutex = new InterProcessMutex(getZkClient(), failoverPath); mutex.acquire(); - String serverHost = getHostByEventDataPath(path); - // handle dead server - handleDeadServer(path, zkNodeType, Constants.ADD_ZK_OP); + String serverHost = null; + if(StringUtils.isNotEmpty(path)){ + serverHost = getHostByEventDataPath(path); + if(StringUtils.isEmpty(serverHost)){ + logger.error("server down error: unknown path: {}", path); + return; + } + // handle dead server + handleDeadServer(path, zkNodeType, Constants.ADD_ZK_OP); + } //failover server if (failover) { failoverServerWhenDown(serverHost, zkNodeType); @@ -336,8 +343,11 @@ public class ZKMasterClient extends AbstractZKClient { List needFailoverProcessInstanceList = processService.queryNeedFailoverProcessInstances(masterHost); + logger.info("failover process list size:{} ", needFailoverProcessInstanceList.size()); //updateProcessInstance host is null and insert into command for (ProcessInstance processInstance : needFailoverProcessInstanceList) { + logger.info("failover process instance id: {} host:{}", + processInstance.getId(), processInstance.getHost()); if (Constants.NULL.equals(processInstance.getHost())) { continue; } diff --git a/dolphinscheduler-server/src/main/resources/logback-master.xml b/dolphinscheduler-server/src/main/resources/logback-master.xml index 7410c01f05..2b986ddad2 100644 --- a/dolphinscheduler-server/src/main/resources/logback-master.xml +++ b/dolphinscheduler-server/src/main/resources/logback-master.xml @@ -74,7 +74,6 @@ - diff --git a/dolphinscheduler-server/src/main/resources/logback-worker.xml b/dolphinscheduler-server/src/main/resources/logback-worker.xml index be1d0acde5..a09202fb7e 100644 --- a/dolphinscheduler-server/src/main/resources/logback-worker.xml +++ b/dolphinscheduler-server/src/main/resources/logback-worker.xml @@ -75,7 +75,6 @@ - diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/DependentTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/DependentTaskTest.java index 351be8aa65..0f6239af0e 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/DependentTaskTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/DependentTaskTest.java @@ -217,6 +217,7 @@ public class DependentTaskTest { taskInstance.setId(252612); taskInstance.setName("C"); taskInstance.setProcessInstanceId(10111); + taskInstance.setTaskJson("{}"); taskInstance.setState(ExecutionStatus.SUBMITTED_SUCCESS); return taskInstance; } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterCommandTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterCommandTest.java index d541f43a3b..d402afcee2 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterCommandTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterCommandTest.java @@ -111,29 +111,4 @@ public class MasterCommandTest { } - @Test - public void testDagHelper(){ - - ProcessDefinition processDefinition = processDefinitionMapper.selectById(19); - - try { - ProcessDag processDag = DagHelper.generateFlowDag(processDefinition.getProcessDefinitionJson(), - new ArrayList<>(), new ArrayList<>(), TaskDependType.TASK_POST); - - DAG dag = DagHelper.buildDagGraph(processDag); - Collection start = DagHelper.getStartVertex("1", dag, null); - - System.out.println(start.toString()); - - Map forbidden = DagHelper.getForbiddenTaskNodeMaps(processDefinition.getProcessDefinitionJson()); - System.out.println(forbidden); - } catch (Exception e) { - e.printStackTrace(); - } - - } - - - - } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelectorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelectorTest.java index 9e41cd68bf..31eac07270 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelectorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelectorTest.java @@ -40,41 +40,81 @@ public class RoundRobinSelectorTest { public void testSelect1() { RoundRobinSelector selector = new RoundRobinSelector(); Host result; - result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, System.currentTimeMillis(), "kris"), new Host("192.168.1.2", 80, 10, System.currentTimeMillis(), "kris"))); + result = selector.select(Arrays.asList( + new Host("192.168.1.1", 80, 20, System.currentTimeMillis(), "kris"), + new Host("192.168.1.2", 80, 10, System.currentTimeMillis(), "kris"))); Assert.assertEquals("192.168.1.1", result.getIp()); - result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, System.currentTimeMillis(), "kris"), new Host("192.168.1.2", 80, 10, System.currentTimeMillis(), "kris"))); + + result = selector.select(Arrays.asList( + new Host("192.168.1.1", 80, 20, System.currentTimeMillis(), "kris"), + new Host("192.168.1.2", 80, 10, System.currentTimeMillis(), "kris"))); Assert.assertEquals("192.168.1.2", result.getIp()); - result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, System.currentTimeMillis(), "kris"), new Host("192.168.1.2", 80, 10, System.currentTimeMillis(), "kris"))); + + result = selector.select(Arrays.asList( + new Host("192.168.1.1", 80, 20, System.currentTimeMillis(), "kris"), + new Host("192.168.1.2", 80, 10, System.currentTimeMillis(), "kris"))); Assert.assertEquals("192.168.1.1", result.getIp()); + // add new host - result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, System.currentTimeMillis(), "kris"), new Host("192.168.1.2", 80, 10, System.currentTimeMillis(), "kris"))); + result = selector.select(Arrays.asList( + new Host("192.168.1.1", 80, 20, System.currentTimeMillis(), "kris"), + new Host("192.168.1.2", 80, 10, System.currentTimeMillis(), "kris"))); Assert.assertEquals("192.168.1.1", result.getIp()); - result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, System.currentTimeMillis(), "kris"), new Host("192.168.1.2", 80, 10, System.currentTimeMillis(), "kris"))); + + result = selector.select(Arrays.asList( + new Host("192.168.1.1", 80, 20, System.currentTimeMillis(), "kris"), + new Host("192.168.1.2", 80, 10, System.currentTimeMillis(), "kris"))); Assert.assertEquals("192.168.1.2", result.getIp()); - result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, System.currentTimeMillis(), "kris"), new Host("192.168.1.2", 80, 10, System.currentTimeMillis(), "kris"), - new Host("192.168.1.3", 80, 10, System.currentTimeMillis(), "kris"))); + + result = selector.select(Arrays.asList( + new Host("192.168.1.1", 80, 20, System.currentTimeMillis(), "kris"), + new Host("192.168.1.2", 80, 10, System.currentTimeMillis(), "kris"), + new Host("192.168.1.3", 80, 10, System.currentTimeMillis(), "kris"))); Assert.assertEquals("192.168.1.1", result.getIp()); - result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, System.currentTimeMillis(), "kris"), new Host("192.168.1.2", 80, 10, System.currentTimeMillis(), "kris"), - new Host("192.168.1.3", 80, 10, System.currentTimeMillis(), "kris"))); + + result = selector.select(Arrays.asList( + new Host("192.168.1.1", 80, 20, System.currentTimeMillis(), "kris"), + new Host("192.168.1.2", 80, 10, System.currentTimeMillis(), "kris"), + new Host("192.168.1.3", 80, 10, System.currentTimeMillis(), "kris"))); Assert.assertEquals("192.168.1.3", result.getIp()); - result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, System.currentTimeMillis(), "kris"), new Host("192.168.1.2", 80, 10, System.currentTimeMillis(), "kris"), - new Host("192.168.1.3", 80, 10, System.currentTimeMillis(), "kris"))); + + result = selector.select(Arrays.asList( + new Host("192.168.1.1", 80, 20, System.currentTimeMillis(), "kris"), + new Host("192.168.1.2", 80, 10, System.currentTimeMillis(), "kris"), + new Host("192.168.1.3", 80, 10, System.currentTimeMillis(), "kris"))); Assert.assertEquals("192.168.1.1", result.getIp()); - result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, System.currentTimeMillis(), "kris"), new Host("192.168.1.2", 80, 10, System.currentTimeMillis(), "kris"), - new Host("192.168.1.3", 80, 10, System.currentTimeMillis(), "kris"))); + + result = selector.select(Arrays.asList( + new Host("192.168.1.1", 80, 20, System.currentTimeMillis(), "kris"), + new Host("192.168.1.2", 80, 10, System.currentTimeMillis(), "kris"), + new Host("192.168.1.3", 80, 10, System.currentTimeMillis(), "kris"))); Assert.assertEquals("192.168.1.2", result.getIp()); - result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, System.currentTimeMillis(), "kris"), new Host("192.168.1.2", 80, 10, System.currentTimeMillis(), "kris"), - new Host("192.168.1.3", 80, 10, System.currentTimeMillis(), "kris"))); + result = selector.select(Arrays.asList( + new Host("192.168.1.1", 80, 20, System.currentTimeMillis(), "kris"), + new Host("192.168.1.2", 80, 10, System.currentTimeMillis(), "kris"), + new Host("192.168.1.3", 80, 10, System.currentTimeMillis(), "kris"))); Assert.assertEquals("192.168.1.1", result.getIp()); - result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, System.currentTimeMillis(), "kris"), new Host("192.168.1.2", 80, 10, System.currentTimeMillis(), "kris"), - new Host("192.168.1.3", 80, 10, System.currentTimeMillis(), "kris"))); + + result = selector.select(Arrays.asList( + new Host("192.168.1.1", 80, 20, System.currentTimeMillis(), "kris"), + new Host("192.168.1.2", 80, 10, System.currentTimeMillis(), "kris"), + new Host("192.168.1.3", 80, 10, System.currentTimeMillis(), "kris"))); Assert.assertEquals("192.168.1.3", result.getIp()); + // remove host3 - result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, System.currentTimeMillis(), "kris"), new Host("192.168.1.2", 80, 10, System.currentTimeMillis(), "kris"))); + result = selector.select(Arrays.asList( + new Host("192.168.1.1", 80, 20, System.currentTimeMillis(), "kris"), + new Host("192.168.1.2", 80, 10, System.currentTimeMillis(), "kris"))); Assert.assertEquals("192.168.1.1", result.getIp()); - result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, System.currentTimeMillis(), "kris"), new Host("192.168.1.2", 80, 10, System.currentTimeMillis(), "kris"))); + + result = selector.select(Arrays.asList( + new Host("192.168.1.1", 80, 20, System.currentTimeMillis(), "kris"), + new Host("192.168.1.2", 80, 10, System.currentTimeMillis(), "kris"))); Assert.assertEquals("192.168.1.2", result.getIp()); - result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, System.currentTimeMillis(), "kris"), new Host("192.168.1.2", 80, 10, System.currentTimeMillis(), "kris"))); + + result = selector.select(Arrays.asList( + new Host("192.168.1.1", 80, 20, System.currentTimeMillis(), "kris"), + new Host("192.168.1.2", 80, 10, System.currentTimeMillis(), "kris"))); Assert.assertEquals("192.168.1.1", result.getIp()); } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessorTest.java index f4805a7561..76ffe7904a 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessorTest.java @@ -85,17 +85,17 @@ public class TaskAckProcessorTest { @Test public void testProcess() { - Command command = taskExecuteAckCommand.convert2Command(); - Assert.assertEquals(CommandType.TASK_EXECUTE_ACK,command.getType()); - InetSocketAddress socketAddress = new InetSocketAddress("localhost",12345); - PowerMockito.when(channel.remoteAddress()).thenReturn(socketAddress); - PowerMockito.mockStatic(TaskResponseEvent.class); - - PowerMockito.when(TaskResponseEvent.newAck(Mockito.any(), Mockito.any(), Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.anyInt())) - .thenReturn(taskResponseEvent); - TaskInstance taskInstance = PowerMockito.mock(TaskInstance.class); - PowerMockito.when(processService.findTaskInstanceById(Mockito.any())).thenReturn(taskInstance); - - taskAckProcessor.process(channel,command); +// Command command = taskExecuteAckCommand.convert2Command(); +// Assert.assertEquals(CommandType.TASK_EXECUTE_ACK,command.getType()); +// InetSocketAddress socketAddress = new InetSocketAddress("localhost",12345); +// PowerMockito.when(channel.remoteAddress()).thenReturn(socketAddress); +// PowerMockito.mockStatic(TaskResponseEvent.class); +// +// PowerMockito.when(TaskResponseEvent.newAck(Mockito.any(), Mockito.any(), Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.anyInt(), channel)) +// .thenReturn(taskResponseEvent); +// TaskInstance taskInstance = PowerMockito.mock(TaskInstance.class); +// PowerMockito.when(processService.findTaskInstanceById(Mockito.any())).thenReturn(taskInstance); +// +// taskAckProcessor.process(channel,command); } } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseServiceTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseServiceTest.java index a2b1b4ecc2..a9fdb58520 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseServiceTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseServiceTest.java @@ -46,7 +46,7 @@ public class TaskResponseServiceTest { @Test public void testAdd(){ TaskResponseEvent taskResponseEvent = TaskResponseEvent.newAck(ExecutionStatus.RUNNING_EXECUTION, new Date(), - "", "", "", 1); + "", "", "", 1,null); taskResponseService.addResponse(taskResponseEvent); Assert.assertTrue(taskResponseService.getEventQueue().size() == 1); try { @@ -60,7 +60,7 @@ public class TaskResponseServiceTest { @Test public void testStop(){ TaskResponseEvent taskResponseEvent = TaskResponseEvent.newAck(ExecutionStatus.RUNNING_EXECUTION, new Date(), - "", "", "", 1); + "", "", "", 1,null); taskResponseService.addResponse(taskResponseEvent); taskResponseService.stop(); Assert.assertTrue(taskResponseService.getEventQueue().size() == 0); diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThreadTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThreadTest.java index f6fdfaab63..405ad435cc 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThreadTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThreadTest.java @@ -57,7 +57,7 @@ public class MasterTaskExecThreadTest { this.zookeeperRegistryCenter = PowerMockito.mock(ZookeeperRegistryCenter.class); PowerMockito.when(SpringApplicationContext.getBean(ZookeeperRegistryCenter.class)) .thenReturn(this.zookeeperRegistryCenter); - this.masterTaskExecThread = new MasterTaskExecThread(null); + this.masterTaskExecThread = new MasterTaskExecThread(getTaskInstance()); } @Test @@ -114,6 +114,7 @@ public class MasterTaskExecThreadTest { taskInstance.setTaskType("SHELL"); taskInstance.setId(252612); taskInstance.setName("C"); + taskInstance.setTaskJson("{}"); taskInstance.setProcessInstanceId(10111); taskInstance.setState(ExecutionStatus.SUBMITTED_SUCCESS); return taskInstance; diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackServiceTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackServiceTest.java index 8d58459e85..2e00a160c3 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackServiceTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackServiceTest.java @@ -152,12 +152,12 @@ public class TaskCallbackServiceTest { nettyRemotingClient.close(); } - @Test(expected = IllegalArgumentException.class) - public void testSendAckWithIllegalArgumentException(){ - TaskExecuteAckCommand ackCommand = Mockito.mock(TaskExecuteAckCommand.class); - taskCallbackService.sendAck(1, ackCommand.convert2Command()); - Stopper.stop(); - } +// @Test(expected = IllegalArgumentException.class) +// public void testSendAckWithIllegalArgumentException(){ +// TaskExecuteAckCommand ackCommand = Mockito.mock(TaskExecuteAckCommand.class); +// taskCallbackService.sendAck(1, ackCommand.convert2Command()); +// Stopper.stop(); +// } @Test public void testPause(){ diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThreadTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThreadTest.java index 2e7e531f30..f0c68d1088 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThreadTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThreadTest.java @@ -119,8 +119,11 @@ public class TaskExecuteThreadTest { taskExecutionContext.setCurrentExecutionStatus(ExecutionStatus.RUNNING_EXECUTION); TaskExecuteThread taskExecuteThread = new TaskExecuteThread(taskExecutionContext, taskCallbackService, taskLogger); taskExecuteThread.run(); + taskExecutionContext.getCurrentExecutionStatus(); - Assert.assertEquals(ExecutionStatus.SUCCESS, taskExecutionContext.getCurrentExecutionStatus()); + taskExecuteThread.run(); + + Assert.assertEquals(ExecutionStatus.RUNNING_EXECUTION, taskExecutionContext.getCurrentExecutionStatus()); } @Test @@ -132,7 +135,7 @@ public class TaskExecuteThreadTest { TaskExecuteThread taskExecuteThread = new TaskExecuteThread(taskExecutionContext, taskCallbackService, taskLogger); taskExecuteThread.run(); - Assert.assertEquals(ExecutionStatus.SUCCESS, taskExecutionContext.getCurrentExecutionStatus()); + Assert.assertEquals(ExecutionStatus.RUNNING_EXECUTION, taskExecutionContext.getCurrentExecutionStatus()); } private class SimpleTask extends AbstractTask { diff --git a/dolphinscheduler-service/pom.xml b/dolphinscheduler-service/pom.xml index f0e8f408d7..ef41d907c2 100644 --- a/dolphinscheduler-service/pom.xml +++ b/dolphinscheduler-service/pom.xml @@ -20,7 +20,7 @@ dolphinscheduler org.apache.dolphinscheduler - 1.3.2-SNAPSHOT + 1.3.4-SNAPSHOT 4.0.0 diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java index 10069aea10..7160168188 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java @@ -79,6 +79,8 @@ import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.service.log.LogClientService; import org.apache.dolphinscheduler.service.quartz.cron.CronUtils; +import org.apache.commons.lang.ArrayUtils; + import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; @@ -162,7 +164,7 @@ public class ProcessService { * @param command found command * @return process instance */ - @Transactional(rollbackFor = RuntimeException.class) + @Transactional(rollbackFor = Exception.class) public ProcessInstance handleCommand(Logger logger, String host, int validThreadNum, Command command) { ProcessInstance processInstance = constructProcessInstance(command, host); //cannot construct process instance, return null; @@ -188,7 +190,7 @@ public class ProcessService { * @param command command * @param message message */ - @Transactional(rollbackFor = RuntimeException.class) + @Transactional(rollbackFor = Exception.class) public void moveToErrorCommand(Command command, String message) { ErrorCommand errorCommand = new ErrorCommand(command, message); this.errorCommandMapper.insert(errorCommand); @@ -358,32 +360,39 @@ public class ProcessService { * remove task log file * @param processInstanceId processInstanceId */ - public void removeTaskLogFile(Integer processInstanceId) { + public void removeTaskLogFile(Integer processInstanceId){ - LogClientService logClient = new LogClientService(); + LogClientService logClient = null; - List taskInstanceList = findValidTaskListByProcessId(processInstanceId); + try { + logClient = new LogClientService(); + List taskInstanceList = findValidTaskListByProcessId(processInstanceId); - if (CollectionUtils.isEmpty(taskInstanceList)) { - return; - } + if (CollectionUtils.isEmpty(taskInstanceList)) { + return; + } - for (TaskInstance taskInstance : taskInstanceList) { - String taskLogPath = taskInstance.getLogPath(); - if (StringUtils.isEmpty(taskInstance.getHost())) { - continue; + for (TaskInstance taskInstance : taskInstanceList) { + String taskLogPath = taskInstance.getLogPath(); + if (StringUtils.isEmpty(taskInstance.getHost())) { + continue; + } + int port = Constants.RPC_PORT; + String ip = ""; + try { + ip = Host.of(taskInstance.getHost()).getIp(); + } catch (Exception e) { + // compatible old version + ip = taskInstance.getHost(); + } + + // remove task log from loggerserver + logClient.removeTaskLog(ip, port, taskLogPath); } - int port = Constants.RPC_PORT; - String ip = ""; - try { - ip = Host.of(taskInstance.getHost()).getIp(); - } catch (Exception e) { - // compatible old version - ip = taskInstance.getHost(); + }finally { + if (logClient != null) { + logClient.close(); } - - // remove task log from loggerserver - logClient.removeTaskLog(ip,port,taskLogPath); } } @@ -457,6 +466,7 @@ public class ProcessService { processInstance.getWarningType(), processInstance.getWarningGroupId(), processInstance.getScheduleTime(), + processInstance.getWorkerGroup(), processInstance.getProcessInstancePriority() ); saveCommand(command); @@ -878,8 +888,8 @@ public class ProcessService { * @param taskInstance taskInstance * @return task instance */ - @Transactional(rollbackFor = RuntimeException.class) - public TaskInstance submitTask(TaskInstance taskInstance) { + @Transactional(rollbackFor = Exception.class) + public TaskInstance submitTask(TaskInstance taskInstance){ ProcessInstance processInstance = this.findProcessInstanceDetailById(taskInstance.getProcessInstanceId()); logger.info("start submit task : {}, instance id:{}, state: {}", taskInstance.getName(), taskInstance.getProcessInstanceId(), processInstance.getState()); @@ -957,9 +967,8 @@ public class ProcessService { /** * create sub work process command - * * @param parentProcessInstance parentProcessInstance - * @param task task + * @param task task */ public void createSubWorkProcess(ProcessInstance parentProcessInstance, TaskInstance task) { if (!task.isSubProcess()) { @@ -1031,6 +1040,7 @@ public class ProcessService { parentProcessInstance.getWarningType(), parentProcessInstance.getWarningGroupId(), parentProcessInstance.getScheduleTime(), + task.getWorkerGroup(), parentProcessInstance.getProcessInstancePriority() ); } @@ -1641,8 +1651,10 @@ public class ProcessService { * @param resourceType resource type * @return tenant code */ - public String queryTenantCodeByResName(String resName,ResourceType resourceType) { - return resourceMapper.queryTenantCodeByResourceName(resName, resourceType.ordinal()); + public String queryTenantCodeByResName(String resName,ResourceType resourceType){ + // in order to query tenant code successful although the version is older + String fullName = resName.startsWith("/") ? resName : String.format("/%s",resName); + return resourceMapper.queryTenantCodeByResourceName(fullName, resourceType.ordinal()); } /** @@ -1679,7 +1691,7 @@ public class ProcessService { */ public List getCycleDependencies(int masterId,int[] ids,Date scheduledFireTime) throws Exception { List cycleDependencyList = new ArrayList(); - if (ids == null || ids.length == 0) { + if(ArrayUtils.isEmpty(ids)){ logger.warn("ids[] is empty!is invalid!"); return cycleDependencyList; } diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZookeeperCachedOperator.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZookeeperCachedOperator.java index 4fe941503e..6dfce79a3a 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZookeeperCachedOperator.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZookeeperCachedOperator.java @@ -16,6 +16,8 @@ */ package org.apache.dolphinscheduler.service.zk; +import org.apache.dolphinscheduler.common.thread.ThreadUtils; + import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.ChildData; import org.apache.curator.framework.recipes.cache.TreeCache; @@ -39,14 +41,6 @@ public class ZookeeperCachedOperator extends ZookeeperOperator { */ @Override protected void registerListener() { - treeCache = new TreeCache(getZkClient(), getZookeeperConfig().getDsRoot() + "/nodes"); - logger.info("add listener to zk path: {}", getZookeeperConfig().getDsRoot()); - try { - treeCache.start(); - } catch (Exception e) { - logger.error("add listener to zk path: {} failed", getZookeeperConfig().getDsRoot()); - throw new RuntimeException(e); - } treeCache.getListenable().addListener((client, event) -> { String path = null == event.getData() ? "" : event.getData().getPath(); @@ -55,7 +49,18 @@ public class ZookeeperCachedOperator extends ZookeeperOperator { } dataChanged(client, event, path); }); + } + @Override + protected void treeCacheStart() { + treeCache = new TreeCache(zkClient, getZookeeperConfig().getDsRoot() + "/nodes"); + logger.info("add listener to zk path: {}", getZookeeperConfig().getDsRoot()); + try { + treeCache.start(); + } catch (Exception e) { + logger.error("add listener to zk path: {} failed", getZookeeperConfig().getDsRoot()); + throw new RuntimeException(e); + } } //for sub class @@ -80,11 +85,7 @@ public class ZookeeperCachedOperator extends ZookeeperOperator { @Override public void close() { treeCache.close(); - try { - Thread.sleep(500); - } catch (InterruptedException ignore) { - Thread.currentThread().interrupt(); - } + ThreadUtils.sleep(500); super.close(); } } diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZookeeperOperator.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZookeeperOperator.java index 7aeb7289c4..e7b049f8bf 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZookeeperOperator.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZookeeperOperator.java @@ -50,11 +50,15 @@ public class ZookeeperOperator implements InitializingBean { private final Logger logger = LoggerFactory.getLogger(ZookeeperOperator.class); @Autowired - private CuratorZookeeperClient zookeeperClient; + private ZookeeperConfig zookeeperConfig; + + protected CuratorFramework zkClient; @Override public void afterPropertiesSet() throws Exception { - registerListener(); + this.zkClient = buildClient(); + initStateLister(); + treeCacheStart(); } /** @@ -62,9 +66,62 @@ public class ZookeeperOperator implements InitializingBean { */ protected void registerListener(){} + protected void treeCacheStart(){} + + public void initStateLister() { + checkNotNull(zkClient); + + zkClient.getConnectionStateListenable().addListener((client, newState) -> { + if(newState == ConnectionState.LOST){ + logger.error("connection lost from zookeeper"); + } else if(newState == ConnectionState.RECONNECTED){ + logger.info("reconnected to zookeeper"); + } else if(newState == ConnectionState.SUSPENDED){ + logger.warn("connection SUSPENDED to zookeeper"); + } + }); + } + + private CuratorFramework buildClient() { + logger.info("zookeeper registry center init, server lists is: {}.", zookeeperConfig.getServerList()); + + CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder().ensembleProvider(new DefaultEnsembleProvider(checkNotNull(zookeeperConfig.getServerList(),"zookeeper quorum can't be null"))) + .retryPolicy(new ExponentialBackoffRetry(zookeeperConfig.getBaseSleepTimeMs(), zookeeperConfig.getMaxRetries(), zookeeperConfig.getMaxSleepMs())); + + //these has default value + if (0 != zookeeperConfig.getSessionTimeoutMs()) { + builder.sessionTimeoutMs(zookeeperConfig.getSessionTimeoutMs()); + } + if (0 != zookeeperConfig.getConnectionTimeoutMs()) { + builder.connectionTimeoutMs(zookeeperConfig.getConnectionTimeoutMs()); + } + if (StringUtils.isNotBlank(zookeeperConfig.getDigest())) { + builder.authorization("digest", zookeeperConfig.getDigest().getBytes(StandardCharsets.UTF_8)).aclProvider(new ACLProvider() { + + @Override + public List getDefaultAcl() { + return ZooDefs.Ids.CREATOR_ALL_ACL; + } + + @Override + public List getAclForPath(final String path) { + return ZooDefs.Ids.CREATOR_ALL_ACL; + } + }); + } + zkClient = builder.build(); + zkClient.start(); + try { + zkClient.blockUntilConnected(); + } catch (final Exception ex) { + throw new RuntimeException(ex); + } + return zkClient; + } + public String get(final String key) { try { - return new String(zookeeperClient.getZkClient().getData().forPath(key), StandardCharsets.UTF_8); + return new String(zkClient.getData().forPath(key), StandardCharsets.UTF_8); } catch (Exception ex) { logger.error("get key : {}", key, ex); } @@ -74,7 +131,7 @@ public class ZookeeperOperator implements InitializingBean { public List getChildrenKeys(final String key) { List values; try { - values = zookeeperClient.getZkClient().getChildren().forPath(key); + values = zkClient.getChildren().forPath(key); return values; } catch (InterruptedException ex) { logger.error("getChildrenKeys key : {} InterruptedException", key); @@ -88,7 +145,7 @@ public class ZookeeperOperator implements InitializingBean { public boolean hasChildren(final String key){ Stat stat ; try { - stat = zookeeperClient.getZkClient().checkExists().forPath(key); + stat = zkClient.checkExists().forPath(key); return stat.getNumChildren() >= 1; } catch (Exception ex) { throw new IllegalStateException(ex); @@ -97,7 +154,7 @@ public class ZookeeperOperator implements InitializingBean { public boolean isExisted(final String key) { try { - return zookeeperClient.getZkClient().checkExists().forPath(key) != null; + return zkClient.checkExists().forPath(key) != null; } catch (Exception ex) { logger.error("isExisted key : {}", key, ex); } @@ -107,7 +164,7 @@ public class ZookeeperOperator implements InitializingBean { public void persist(final String key, final String value) { try { if (!isExisted(key)) { - zookeeperClient.getZkClient().create().creatingParentsIfNeeded().withMode(CreateMode.PERSISTENT).forPath(key, value.getBytes(StandardCharsets.UTF_8)); + zkClient.create().creatingParentsIfNeeded().withMode(CreateMode.PERSISTENT).forPath(key, value.getBytes(StandardCharsets.UTF_8)); } else { update(key, value); } @@ -118,11 +175,7 @@ public class ZookeeperOperator implements InitializingBean { public void update(final String key, final String value) { try { - - CuratorOp check = zookeeperClient.getZkClient().transactionOp().check().forPath(key); - CuratorOp setData = zookeeperClient.getZkClient().transactionOp().setData().forPath(key, value.getBytes(StandardCharsets.UTF_8)); - zookeeperClient.getZkClient().transaction().forOperations(check, setData); - + zkClient.inTransaction().check().forPath(key).and().setData().forPath(key, value.getBytes(StandardCharsets.UTF_8)).and().commit(); } catch (Exception ex) { logger.error("update key : {} , value : {}", key, value, ex); } @@ -132,12 +185,12 @@ public class ZookeeperOperator implements InitializingBean { try { if (isExisted(key)) { try { - zookeeperClient.getZkClient().delete().deletingChildrenIfNeeded().forPath(key); + zkClient.delete().deletingChildrenIfNeeded().forPath(key); } catch (KeeperException.NoNodeException ignore) { //NOP } } - zookeeperClient.getZkClient().create().creatingParentsIfNeeded().withMode(CreateMode.EPHEMERAL).forPath(key, value.getBytes(StandardCharsets.UTF_8)); + zkClient.create().creatingParentsIfNeeded().withMode(CreateMode.EPHEMERAL).forPath(key, value.getBytes(StandardCharsets.UTF_8)); } catch (final Exception ex) { logger.error("persistEphemeral key : {} , value : {}", key, value, ex); } @@ -149,7 +202,7 @@ public class ZookeeperOperator implements InitializingBean { persistEphemeral(key, value); } else { if (!isExisted(key)) { - zookeeperClient.getZkClient().create().creatingParentsIfNeeded().withMode(CreateMode.EPHEMERAL).forPath(key, value.getBytes(StandardCharsets.UTF_8)); + zkClient.create().creatingParentsIfNeeded().withMode(CreateMode.EPHEMERAL).forPath(key, value.getBytes(StandardCharsets.UTF_8)); } } } catch (final Exception ex) { @@ -159,7 +212,7 @@ public class ZookeeperOperator implements InitializingBean { public void persistEphemeralSequential(final String key, String value) { try { - zookeeperClient.getZkClient().create().creatingParentsIfNeeded().withMode(CreateMode.EPHEMERAL_SEQUENTIAL).forPath(key, value.getBytes(StandardCharsets.UTF_8)); + zkClient.create().creatingParentsIfNeeded().withMode(CreateMode.EPHEMERAL_SEQUENTIAL).forPath(key, value.getBytes(StandardCharsets.UTF_8)); } catch (final Exception ex) { logger.error("persistEphemeralSequential key : {}", key, ex); } @@ -168,7 +221,7 @@ public class ZookeeperOperator implements InitializingBean { public void remove(final String key) { try { if (isExisted(key)) { - zookeeperClient.getZkClient().delete().deletingChildrenIfNeeded().forPath(key); + zkClient.delete().deletingChildrenIfNeeded().forPath(key); } } catch (KeeperException.NoNodeException ignore) { //NOP @@ -178,14 +231,14 @@ public class ZookeeperOperator implements InitializingBean { } public CuratorFramework getZkClient() { - return zookeeperClient.getZkClient(); + return zkClient; } public ZookeeperConfig getZookeeperConfig() { - return zookeeperClient.getZookeeperConfig(); + return zookeeperConfig; } public void close() { - CloseableUtils.closeQuietly(zookeeperClient.getZkClient()); + CloseableUtils.closeQuietly(zkClient); } } \ No newline at end of file diff --git a/dolphinscheduler-ui/build/webpack.config.prod.js b/dolphinscheduler-ui/build/webpack.config.prod.js index 4bb90d54d1..1024ac6724 100644 --- a/dolphinscheduler-ui/build/webpack.config.prod.js +++ b/dolphinscheduler-ui/build/webpack.config.prod.js @@ -51,12 +51,7 @@ const config = merge.smart(baseConfig, { minimizer: [ new TerserPlugin({ terserOptions: { - compress: { - warnings: false, - drop_console: true, - drop_debugger: true, - pure_funcs: ['console.log'] - } + compress: {} }, cache: true, parallel: true, diff --git a/dolphinscheduler-ui/pom.xml b/dolphinscheduler-ui/pom.xml index 13644bad91..8e644de265 100644 --- a/dolphinscheduler-ui/pom.xml +++ b/dolphinscheduler-ui/pom.xml @@ -20,7 +20,7 @@ dolphinscheduler org.apache.dolphinscheduler - 1.3.2-SNAPSHOT + 1.3.4-SNAPSHOT 4.0.0 diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.js b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.js index 74671cca57..937b7391ef 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.js @@ -53,7 +53,9 @@ Dag.prototype.setConfig = function (o) { */ Dag.prototype.create = function () { const self = this - jsPlumb.ready(() => { + let plumbIns = jsPlumb.getInstance() + plumbIns.reset() + plumbIns.ready(() => { JSP.init({ dag: this.dag, instance: this.instance, @@ -164,7 +166,9 @@ Dag.prototype.backfill = function (arg) { }) }) } else { - jsPlumb.ready(() => { + let plumbIns = jsPlumb.getInstance() + plumbIns.reset() + plumbIns.ready(() => { JSP.init({ dag: this.dag, instance: this.instance, diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/log.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/log.vue index 7874b53885..838f429bd3 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/log.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/log.vue @@ -303,25 +303,27 @@ color: #0097e0; font-size: 12px; margin-left: 10px; - i { + em { + text-decoration: none !important; vertical-align: middle; } } .clock { - >i { + >em { font-size: 20px; vertical-align: middle; transform: scale(1); } } .refresh-log { - >i { + >em { + text-decoration: none; font-size: 20px; vertical-align: middle; transform: scale(1); } &.active { - >i { + >em { -webkit-transition-property: -webkit-transform; -webkit-transition-duration: 1s; -moz-transition-property: -moz-transform; @@ -368,5 +370,16 @@ } } } - + @-webkit-keyframes rotateloading{from{-webkit-transform: rotate(0deg)} + to{-webkit-transform: rotate(360deg)} + } + @-moz-keyframes rotateloading{from{-moz-transform: rotate(0deg)} + to{-moz-transform: rotate(359deg)} + } + @-o-keyframes rotateloading{from{-o-transform: rotate(0deg)} + to{-o-transform: rotate(359deg)} + } + @keyframes rotateloading{from{transform: rotate(0deg)} + to{transform: rotate(359deg)} + } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/processStateCount.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/processStateCount.vue index d121034932..974d57ff95 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/processStateCount.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/processStateCount.vue @@ -19,7 +19,7 @@
-
+
@@ -31,7 +31,10 @@ {{$index+1}} - {{item.value}} + + {{item.value}} + {{item.value}} + {{item.key}} @@ -49,6 +52,8 @@ import { mapActions } from 'vuex' import { pie } from './chartConfig' import Chart from '@/module/ana-charts' + import echarts from 'echarts' + import store from '@/conf/home/store' import mNoData from '@/module/components/noData/noData' import { stateType } from '@/conf/home/pages/projects/pages/_source/instanceConditions/common' export default { @@ -57,7 +62,8 @@ return { isSpin: true, msg: '', - processStateList: [] + processStateList: [], + currentName: '' } }, props: { @@ -83,7 +89,7 @@ value: v.count } }) - const myChart = Chart.pie('#process-state-pie', this.processStateList, { title: '' }) + const myChart = Chart.pie('#process-state-pie', this.processStateList, { title: '' }) myChart.echart.setOption(pie) // 首页不允许跳转 if (this.searchParams.projectId) { @@ -108,11 +114,15 @@ this.isSpin = false }) } + }, + '$store.state.projects.sideBar': function() { + echarts.init(document.getElementById('process-state-pie')).resize() } }, beforeCreate () { }, created () { + this.currentName = this.$router.currentRoute.name }, beforeMount () { }, @@ -132,7 +142,4 @@ diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/taskCtatusCount.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/taskCtatusCount.vue index 3e56c344f7..60e9413b2e 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/taskCtatusCount.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/taskCtatusCount.vue @@ -19,7 +19,7 @@
-
+
@@ -32,8 +32,9 @@ {{$index+1}} - - {{item.value}} + {{item.value}} + + {{item.value}} {{item.key}} @@ -53,6 +54,8 @@ import { mapActions } from 'vuex' import { pie } from './chartConfig' import Chart from '@/module/ana-charts' + import echarts from 'echarts' + import store from '@/conf/home/store' import mNoData from '@/module/components/noData/noData' import { stateType } from '@/conf/home/pages/projects/pages/_source/instanceConditions/common' @@ -62,7 +65,8 @@ return { isSpin: true, msg: '', - taskCtatusList: [] + taskCtatusList: [], + currentName: '' } }, props: { @@ -115,11 +119,15 @@ this.isSpin = false }) } + }, + '$store.state.projects.sideBar': function() { + echarts.init(document.getElementById('task-status-pie')).resize() } }, beforeCreate () { }, created () { + this.currentName = this.$router.currentRoute.name }, beforeMount () { }, @@ -139,7 +147,4 @@ diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/taskInstance/_source/list.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/taskInstance/_source/list.vue index a6c7de1a1f..9ad0938009 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/taskInstance/_source/list.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/taskInstance/_source/list.vue @@ -108,6 +108,7 @@ data-toggle="tooltip" :title="$t('View log')" icon="ans-icon-log" + :disabled="item.taskType==='SUB_PROCESS'? true: false" @click="_refreshLog(item)"> diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/details/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/details/index.vue index b4ee720d12..06626b6dcf 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/details/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/details/index.vue @@ -232,7 +232,7 @@ position: absolute; right: 0; top: 0; - >i { + >em { font-size: 20px; color: #2d8cf0; cursor: pointer; diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/function/_source/list.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/function/_source/list.vue index d77f55722e..8320a17516 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/function/_source/list.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/function/_source/list.vue @@ -94,11 +94,11 @@ v-ps