diff --git a/.gitignore b/.gitignore index 7a99e2e4b0..7cf1d4d7db 100644 --- a/.gitignore +++ b/.gitignore @@ -4,25 +4,27 @@ .zip .gz .DS_Store +.idea .idea/ -dist/ -all-dependencies.txt -self-modules.txt -third-party-dependencies.txt -**/target/ +.idea/* +.target +.target/ +**/**/target/** +target/* +*/target +*/target/* .settings .nbproject .classpath .project -**/*.iml +*.iml *.ipr *.iws *.tgz .*.swp -.factorypath .vim .tmp -**/node_modules +node_modules npm-debug.log .vscode logs/* @@ -39,10 +41,110 @@ dolphinscheduler-alert/logs/ dolphinscheduler-alert/src/main/resources/alert.properties_bak dolphinscheduler-alert/src/main/resources/logback.xml dolphinscheduler-server/src/main/resources/logback.xml -dolphinscheduler-ui/dist/ +dolphinscheduler-ui/dist dolphinscheduler-ui/node -dolphinscheduler-dao/src/main/resources/dao/data_source.properties +dolphinscheduler-ui/dist/css/common.16ac5d9.css +dolphinscheduler-ui/dist/css/home/index.b444b91.css +dolphinscheduler-ui/dist/css/login/index.5866c64.css +dolphinscheduler-ui/dist/js/0.ac94e5d.js +dolphinscheduler-ui/dist/js/0.ac94e5d.js.map +dolphinscheduler-ui/dist/js/1.0b043a3.js +dolphinscheduler-ui/dist/js/1.0b043a3.js.map +dolphinscheduler-ui/dist/js/10.1bce3dc.js +dolphinscheduler-ui/dist/js/10.1bce3dc.js.map +dolphinscheduler-ui/dist/js/11.79f04d8.js +dolphinscheduler-ui/dist/js/11.79f04d8.js.map +dolphinscheduler-ui/dist/js/12.420daa5.js +dolphinscheduler-ui/dist/js/12.420daa5.js.map +dolphinscheduler-ui/dist/js/13.e5bae1c.js +dolphinscheduler-ui/dist/js/13.e5bae1c.js.map +dolphinscheduler-ui/dist/js/14.f2a0dca.js +dolphinscheduler-ui/dist/js/14.f2a0dca.js.map +dolphinscheduler-ui/dist/js/15.45373e8.js +dolphinscheduler-ui/dist/js/15.45373e8.js.map +dolphinscheduler-ui/dist/js/16.fecb0fc.js +dolphinscheduler-ui/dist/js/16.fecb0fc.js.map +dolphinscheduler-ui/dist/js/17.84be279.js +dolphinscheduler-ui/dist/js/17.84be279.js.map +dolphinscheduler-ui/dist/js/18.307ea70.js +dolphinscheduler-ui/dist/js/18.307ea70.js.map +dolphinscheduler-ui/dist/js/19.144db9c.js +dolphinscheduler-ui/dist/js/19.144db9c.js.map +dolphinscheduler-ui/dist/js/2.8b4ef29.js +dolphinscheduler-ui/dist/js/2.8b4ef29.js.map +dolphinscheduler-ui/dist/js/20.4c527e9.js +dolphinscheduler-ui/dist/js/20.4c527e9.js.map +dolphinscheduler-ui/dist/js/21.831b2a2.js +dolphinscheduler-ui/dist/js/21.831b2a2.js.map +dolphinscheduler-ui/dist/js/22.2b4bb2a.js +dolphinscheduler-ui/dist/js/22.2b4bb2a.js.map +dolphinscheduler-ui/dist/js/23.81467ef.js +dolphinscheduler-ui/dist/js/23.81467ef.js.map +dolphinscheduler-ui/dist/js/24.54a00e4.js +dolphinscheduler-ui/dist/js/24.54a00e4.js.map +dolphinscheduler-ui/dist/js/25.8d7bd36.js +dolphinscheduler-ui/dist/js/25.8d7bd36.js.map +dolphinscheduler-ui/dist/js/26.2ec5e78.js +dolphinscheduler-ui/dist/js/26.2ec5e78.js.map +dolphinscheduler-ui/dist/js/27.3ab48c2.js +dolphinscheduler-ui/dist/js/27.3ab48c2.js.map +dolphinscheduler-ui/dist/js/28.363088a.js +dolphinscheduler-ui/dist/js/28.363088a.js.map +dolphinscheduler-ui/dist/js/29.6c5853a.js +dolphinscheduler-ui/dist/js/29.6c5853a.js.map +dolphinscheduler-ui/dist/js/3.a0edb5b.js +dolphinscheduler-ui/dist/js/3.a0edb5b.js.map +dolphinscheduler-ui/dist/js/30.940fdd3.js +dolphinscheduler-ui/dist/js/30.940fdd3.js.map +dolphinscheduler-ui/dist/js/31.168a460.js +dolphinscheduler-ui/dist/js/31.168a460.js.map +dolphinscheduler-ui/dist/js/32.8df6594.js +dolphinscheduler-ui/dist/js/32.8df6594.js.map +dolphinscheduler-ui/dist/js/33.4480bbe.js +dolphinscheduler-ui/dist/js/33.4480bbe.js.map +dolphinscheduler-ui/dist/js/34.b407fe1.js +dolphinscheduler-ui/dist/js/34.b407fe1.js.map +dolphinscheduler-ui/dist/js/35.f340b0a.js +dolphinscheduler-ui/dist/js/35.f340b0a.js.map +dolphinscheduler-ui/dist/js/36.8880c2d.js +dolphinscheduler-ui/dist/js/36.8880c2d.js.map +dolphinscheduler-ui/dist/js/37.ea2a25d.js +dolphinscheduler-ui/dist/js/37.ea2a25d.js.map +dolphinscheduler-ui/dist/js/38.98a59ee.js +dolphinscheduler-ui/dist/js/38.98a59ee.js.map +dolphinscheduler-ui/dist/js/39.a5e958a.js +dolphinscheduler-ui/dist/js/39.a5e958a.js.map +dolphinscheduler-ui/dist/js/4.4ca44db.js +dolphinscheduler-ui/dist/js/4.4ca44db.js.map +dolphinscheduler-ui/dist/js/40.e187b1e.js +dolphinscheduler-ui/dist/js/40.e187b1e.js.map +dolphinscheduler-ui/dist/js/41.0e89182.js +dolphinscheduler-ui/dist/js/41.0e89182.js.map +dolphinscheduler-ui/dist/js/42.341047c.js +dolphinscheduler-ui/dist/js/42.341047c.js.map +dolphinscheduler-ui/dist/js/43.27b8228.js +dolphinscheduler-ui/dist/js/43.27b8228.js.map +dolphinscheduler-ui/dist/js/44.e8869bc.js +dolphinscheduler-ui/dist/js/44.e8869bc.js.map +dolphinscheduler-ui/dist/js/45.8d54901.js +dolphinscheduler-ui/dist/js/45.8d54901.js.map +dolphinscheduler-ui/dist/js/5.e1ed7f3.js +dolphinscheduler-ui/dist/js/5.e1ed7f3.js.map +dolphinscheduler-ui/dist/js/6.241ba07.js +dolphinscheduler-ui/dist/js/6.241ba07.js.map +dolphinscheduler-ui/dist/js/7.ab2e297.js +dolphinscheduler-ui/dist/js/7.ab2e297.js.map +dolphinscheduler-ui/dist/js/8.83ff814.js +dolphinscheduler-ui/dist/js/8.83ff814.js.map +dolphinscheduler-ui/dist/js/9.39cb29f.js +dolphinscheduler-ui/dist/js/9.39cb29f.js.map +dolphinscheduler-ui/dist/js/common.733e342.js +dolphinscheduler-ui/dist/js/common.733e342.js.map +dolphinscheduler-ui/dist/js/home/index.78a5d12.js +dolphinscheduler-ui/dist/js/home/index.78a5d12.js.map +dolphinscheduler-ui/dist/js/login/index.291b8e3.js +dolphinscheduler-ui/dist/js/login/index.291b8e3.js.map +dolphinscheduler-ui/dist/lib/external/ +dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/taskInstance/index.vue +/dolphinscheduler-dao/src/main/resources/dao/data_source.properties -.mvn/wrapper/*.jar - -!/zookeeper_data/ diff --git a/.mvn/wrapper/maven-wrapper.jar b/.mvn/wrapper/maven-wrapper.jar new file mode 100644 index 0000000000..2cc7d4a55c Binary files /dev/null and b/.mvn/wrapper/maven-wrapper.jar differ diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/configuration/dolphin-application.xml b/ambari_plugin/common-services/DOLPHIN/1.2.1/configuration/dolphin-application.xml deleted file mode 100644 index 6e50a1b649..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/1.2.1/configuration/dolphin-application.xml +++ /dev/null @@ -1,467 +0,0 @@ - - - - spring.datasource.initialSize - 5 - - Init connection number - - - int - - - - - spring.datasource.minIdle - 5 - - Min connection number - - - int - - - - - spring.datasource.maxActive - 50 - - Max connection number - - - int - - - - - spring.datasource.maxWait - 60000 - - Max wait time for get a connection in milliseconds. - If configuring maxWait, fair locks are enabled by default and concurrency efficiency decreases. - If necessary, unfair locks can be used by configuring the useUnfairLock attribute to true. - - - int - - - - - spring.datasource.timeBetweenEvictionRunsMillis - 60000 - - Milliseconds for check to close free connections - - - int - - - - - spring.datasource.timeBetweenConnectErrorMillis - 60000 - - The Destroy thread detects the connection interval and closes the physical connection in milliseconds - if the connection idle time is greater than or equal to minEvictableIdleTimeMillis. - - - int - - - - - spring.datasource.minEvictableIdleTimeMillis - 300000 - - The longest time a connection remains idle without being evicted, in milliseconds - - - int - - - - - spring.datasource.validationQuery - SELECT 1 - - The SQL used to check whether the connection is valid requires a query statement. - If validation Query is null, testOnBorrow, testOnReturn, and testWhileIdle will not work. - - - - - spring.datasource.validationQueryTimeout - 3 - - int - - - Check whether the connection is valid for timeout, in seconds - - - - - spring.datasource.testWhileIdle - true - - boolean - - - When applying for a connection, - if it is detected that the connection is idle longer than time Between Eviction Runs Millis, - validation Query is performed to check whether the connection is valid - - - - - spring.datasource.testOnBorrow - true - - boolean - - - Execute validation to check if the connection is valid when applying for a connection - - - - - spring.datasource.testOnReturn - false - - boolean - - - Execute validation to check if the connection is valid when the connection is returned - - - - - spring.datasource.defaultAutoCommit - true - - boolean - - - - - - - spring.datasource.keepAlive - false - - boolean - - - - - - - - spring.datasource.poolPreparedStatements - true - - boolean - - - Open PSCache, specify count PSCache for every connection - - - - - spring.datasource.maxPoolPreparedStatementPerConnectionSize - 20 - - int - - - - - - spring.datasource.spring.datasource.filters - stat,wall,log4j - - - - - spring.datasource.connectionProperties - druid.stat.mergeSql=true;druid.stat.slowSqlMillis=5000 - - - - - - mybatis-plus.mapper-locations - classpath*:/org.apache.dolphinscheduler.dao.mapper/*.xml - - - - - mybatis-plus.typeEnumsPackage - org.apache.dolphinscheduler.*.enums - - - - - mybatis-plus.typeAliasesPackage - org.apache.dolphinscheduler.dao.entity - - Entity scan, where multiple packages are separated by a comma or semicolon - - - - - mybatis-plus.global-config.db-config.id-type - AUTO - - value-list - - - AUTO - - - - INPUT - - - - ID_WORKER - - - - UUID - - - - 1 - - - Primary key type AUTO:" database ID AUTO ", - INPUT:" user INPUT ID", - ID_WORKER:" global unique ID (numeric type unique ID)", - UUID:" global unique ID UUID"; - - - - - mybatis-plus.global-config.db-config.field-strategy - NOT_NULL - - value-list - - - IGNORED - - - - NOT_NULL - - - - NOT_EMPTY - - - - 1 - - - Field policy IGNORED:" ignore judgment ", - NOT_NULL:" not NULL judgment "), - NOT_EMPTY:" not NULL judgment" - - - - - mybatis-plus.global-config.db-config.column-underline - true - - boolean - - - - - - mybatis-plus.global-config.db-config.logic-delete-value - 1 - - int - - - - - - mybatis-plus.global-config.db-config.logic-not-delete-value - 0 - - int - - - - - - mybatis-plus.global-config.db-config.banner - true - - boolean - - - - - - - mybatis-plus.configuration.map-underscore-to-camel-case - true - - boolean - - - - - - mybatis-plus.configuration.cache-enabled - false - - boolean - - - - - - mybatis-plus.configuration.call-setters-on-nulls - true - - boolean - - - - - - mybatis-plus.configuration.jdbc-type-for-null - null - - - - - master.exec.threads - 100 - - int - - - - - - master.exec.task.num - 20 - - int - - - - - - master.heartbeat.interval - 10 - - int - - - - - - master.task.commit.retryTimes - 5 - - int - - - - - - master.task.commit.interval - 1000 - - int - - - - - - master.max.cpuload.avg - 100 - - int - - - - - - master.reserved.memory - 0.1 - - float - - - - - - worker.exec.threads - 100 - - int - - - - - - worker.heartbeat.interval - 10 - - int - - - - - - worker.fetch.task.num - 3 - - int - - - - - - worker.max.cpuload.avg - 100 - - int - - - - - - worker.reserved.memory - 0.1 - - float - - - - - - \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/alerts.json b/ambari_plugin/common-services/DOLPHIN/1.3.3/alerts.json similarity index 95% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/alerts.json rename to ambari_plugin/common-services/DOLPHIN/1.3.3/alerts.json index 769245b366..184f021ac3 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.2.1/alerts.json +++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/alerts.json @@ -65,7 +65,7 @@ "enabled": true, "source": { "type": "SCRIPT", - "path": "DOLPHIN/1.2.1/package/alerts/alert_dolphin_scheduler_status.py", + "path": "DOLPHIN/1.3.3/package/alerts/alert_dolphin_scheduler_status.py", "parameters": [ { @@ -98,7 +98,7 @@ "enabled": true, "source": { "type": "SCRIPT", - "path": "DOLPHIN/1.2.1/package/alerts/alert_dolphin_scheduler_status.py", + "path": "DOLPHIN/1.3.3/package/alerts/alert_dolphin_scheduler_status.py", "parameters": [ { @@ -131,7 +131,7 @@ "enabled": true, "source": { "type": "SCRIPT", - "path": "DOLPHIN/1.2.1/package/alerts/alert_dolphin_scheduler_status.py", + "path": "DOLPHIN/1.3.3/package/alerts/alert_dolphin_scheduler_status.py", "parameters": [ { diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/configuration/dolphin-alert.xml b/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-alert.xml similarity index 95% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/configuration/dolphin-alert.xml rename to ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-alert.xml index 5b82230148..32abcc791d 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.2.1/configuration/dolphin-alert.xml +++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-alert.xml @@ -90,13 +90,6 @@ - - xls.file.path - /tmp/xls - - - - enterprise.wechat.enable false diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/configuration/dolphin-application-api.xml b/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-application-api.xml similarity index 82% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/configuration/dolphin-application-api.xml rename to ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-application-api.xml index ea4cb82afd..766c0f477d 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.2.1/configuration/dolphin-application-api.xml +++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-application-api.xml @@ -34,6 +34,12 @@ + + server.servlet.context-path + /dolphinscheduler/ + + + spring.servlet.multipart.max-file-size 1024 @@ -68,4 +74,14 @@ UTF-8 + + spring.messages.basename + i18n/messages + + + + security.authentication.type + PASSWORD + + \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/configuration/dolphin-common.xml b/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-common.xml similarity index 100% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/configuration/dolphin-common.xml rename to ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-common.xml diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-datasource.xml b/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-datasource.xml new file mode 100644 index 0000000000..02d8de0482 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-datasource.xml @@ -0,0 +1,206 @@ + + + + spring.datasource.initialSize + 5 + + Init connection number + + + int + + + + + spring.datasource.minIdle + 5 + + Min connection number + + + int + + + + + spring.datasource.maxActive + 50 + + Max connection number + + + int + + + + + spring.datasource.maxWait + 60000 + + Max wait time for get a connection in milliseconds. + If configuring maxWait, fair locks are enabled by default and concurrency efficiency decreases. + If necessary, unfair locks can be used by configuring the useUnfairLock attribute to true. + + + int + + + + + spring.datasource.timeBetweenEvictionRunsMillis + 60000 + + Milliseconds for check to close free connections + + + int + + + + + spring.datasource.timeBetweenConnectErrorMillis + 60000 + + The Destroy thread detects the connection interval and closes the physical connection in milliseconds + if the connection idle time is greater than or equal to minEvictableIdleTimeMillis. + + + int + + + + + spring.datasource.minEvictableIdleTimeMillis + 300000 + + The longest time a connection remains idle without being evicted, in milliseconds + + + int + + + + + spring.datasource.validationQuery + SELECT 1 + + The SQL used to check whether the connection is valid requires a query statement. + If validation Query is null, testOnBorrow, testOnReturn, and testWhileIdle will not work. + + + + + spring.datasource.validationQueryTimeout + 3 + + int + + + Check whether the connection is valid for timeout, in seconds + + + + + spring.datasource.testWhileIdle + true + + boolean + + + When applying for a connection, + if it is detected that the connection is idle longer than time Between Eviction Runs Millis, + validation Query is performed to check whether the connection is valid + + + + + spring.datasource.testOnBorrow + true + + boolean + + + Execute validation to check if the connection is valid when applying for a connection + + + + + spring.datasource.testOnReturn + false + + boolean + + + Execute validation to check if the connection is valid when the connection is returned + + + + + spring.datasource.defaultAutoCommit + true + + boolean + + + + + + + spring.datasource.keepAlive + false + + boolean + + + + + + + + spring.datasource.poolPreparedStatements + true + + boolean + + + Open PSCache, specify count PSCache for every connection + + + + + spring.datasource.maxPoolPreparedStatementPerConnectionSize + 20 + + int + + + + + + spring.datasource.spring.datasource.filters + stat,wall,log4j + + + + + spring.datasource.connectionProperties + druid.stat.mergeSql=true;druid.stat.slowSqlMillis=5000 + + + + \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/configuration/dolphin-env.xml b/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-env.xml similarity index 100% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/configuration/dolphin-env.xml rename to ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-env.xml diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-master.xml b/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-master.xml new file mode 100644 index 0000000000..c8eec047fc --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-master.xml @@ -0,0 +1,88 @@ + + + + master.exec.threads + 100 + + int + + master execute thread num + + + + master.exec.task.num + 20 + + int + + master execute task number in parallel + + + + master.heartbeat.interval + 10 + + int + + master heartbeat interval + + + + master.task.commit.retryTimes + 5 + + int + + master commit task retry times + + + + master.task.commit.interval + 1000 + + int + + master commit task interval + + + + master.max.cpuload.avg + 100 + + int + + only less than cpu avg load, master server can work. default value : the number of cpu cores * 2 + + + + master.reserved.memory + 0.3 + only larger than reserved memory, master server can work. default value : physical memory * 1/10, unit is G. + + + + + master.listen.port + 5678 + + int + + master listen port + + + \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/configuration/dolphin-quartz.xml b/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-quartz.xml similarity index 91% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/configuration/dolphin-quartz.xml rename to ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-quartz.xml index 82b59d8827..7a0c68b051 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.2.1/configuration/dolphin-quartz.xml +++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-quartz.xml @@ -106,26 +106,21 @@ - org.quartz.jobStore.dataSource - myDs - - - - org.quartz.dataSource.myDs.connectionProvider.class - org.apache.dolphinscheduler.server.quartz.DruidConnectionProvider + org.quartz.jobStore.acquireTriggersWithinLock + true + + boolean + - org.quartz.dataSource.myDs.maxConnections - 10 - - int - + org.quartz.jobStore.dataSource + myDs - org.quartz.dataSource.myDs.validationQuery - select 1 + org.quartz.dataSource.myDs.connectionProvider.class + org.apache.dolphinscheduler.service.quartz.DruidConnectionProvider \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-worker.xml b/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-worker.xml new file mode 100644 index 0000000000..1ae7a1a765 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-worker.xml @@ -0,0 +1,67 @@ + + + + worker.exec.threads + 100 + + int + + worker execute thread num + + + + worker.heartbeat.interval + 10 + + int + + worker heartbeat interval + + + + worker.max.cpuload.avg + 100 + + int + + only less than cpu avg load, worker server can work. default value : the number of cpu cores * 2 + + + + worker.reserved.memory + 0.3 + only larger than reserved memory, worker server can work. default value : physical memory * 1/10, unit is G. + + + + + worker.listen.port + 1234 + + int + + worker listen port + + + + worker.groups + default + default worker group + + + \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-zookeeper.xml b/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-zookeeper.xml new file mode 100644 index 0000000000..e89962d900 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-zookeeper.xml @@ -0,0 +1,76 @@ + + + + zookeeper.dolphinscheduler.root + /dolphinscheduler + + dolphinscheduler root directory + + + + + zookeeper.session.timeout + 300 + + int + + + + + + + zookeeper.connection.timeout + 300 + + int + + + + + + + zookeeper.retry.base.sleep + 100 + + int + + + + + + + zookeeper.retry.max.sleep + 30000 + + int + + + + + + + zookeeper.retry.maxtime + 5 + + int + + + + + + \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/metainfo.xml b/ambari_plugin/common-services/DOLPHIN/1.3.3/metainfo.xml similarity index 98% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/metainfo.xml rename to ambari_plugin/common-services/DOLPHIN/1.3.3/metainfo.xml index 0d2bbe3163..074306d5cb 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.2.1/metainfo.xml +++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/metainfo.xml @@ -22,7 +22,7 @@ DOLPHIN Dolphin Scheduler 分布式易扩展的可视化DAG工作流任务调度系统 - 1.2.1 + 1.3.3 DOLPHIN_MASTER @@ -103,7 +103,7 @@ any - apache-dolphinscheduler-incubating-1.2.1* + apache-dolphinscheduler-incubating* @@ -134,4 +134,4 @@ - + \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/alerts/alert_dolphin_scheduler_status.py b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/alerts/alert_dolphin_scheduler_status.py similarity index 100% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/package/alerts/alert_dolphin_scheduler_status.py rename to ambari_plugin/common-services/DOLPHIN/1.3.3/package/alerts/alert_dolphin_scheduler_status.py diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/dolphin_alert_service.py b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_alert_service.py similarity index 92% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/dolphin_alert_service.py rename to ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_alert_service.py index 62255a3432..e78c38d272 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/dolphin_alert_service.py +++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_alert_service.py @@ -26,7 +26,8 @@ class DolphinAlertService(Script): import params env.set_params(params) self.install_packages(env) - Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True) + Execute(('chmod', '-R', '777', params.dolphin_home)) + Execute(('chown', '-R', params.dolphin_user + ":" + params.dolphin_group, params.dolphin_home)) def configure(self, env): import params diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/dolphin_api_service.py b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_api_service.py similarity index 93% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/dolphin_api_service.py rename to ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_api_service.py index bdc18fb602..5a28924a9a 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/dolphin_api_service.py +++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_api_service.py @@ -26,7 +26,8 @@ class DolphinApiService(Script): import params env.set_params(params) self.install_packages(env) - Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True) + Execute(('chmod', '-R', '777', params.dolphin_home)) + Execute(('chown', '-R', params.dolphin_user + ":" + params.dolphin_group, params.dolphin_home)) def configure(self, env): import params diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/dolphin_env.py b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_env.py similarity index 78% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/dolphin_env.py rename to ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_env.py index 235605894f..1661d76c75 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/dolphin_env.py +++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_env.py @@ -42,31 +42,12 @@ def dolphin_env(): create_parents=True ) - - Directory(params.dolphin_alert_map['xls.file.path'], - mode=0777, - owner=params.dolphin_user, - group=params.dolphin_group, - create_parents=True - ) Directory(params.dolphin_common_map['data.basedir.path'], mode=0777, owner=params.dolphin_user, group=params.dolphin_group, create_parents=True ) - Directory(params.dolphin_common_map['data.download.basedir.path'], - mode=0777, - owner=params.dolphin_user, - group=params.dolphin_group, - create_parents=True - ) - Directory(params.dolphin_common_map['process.exec.basepath'], - mode=0777, - owner=params.dolphin_user, - group=params.dolphin_group, - create_parents=True - ) File(format(params.dolphin_env_path), @@ -79,11 +60,25 @@ def dolphin_env(): File(format(params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh"), mode=0755, - content=Template("dolphin-daemon.j2"), + content=Template("dolphin-daemon.sh.j2"), owner=params.dolphin_user, group=params.dolphin_group ) + File(format(params.dolphin_conf_dir + "/master.properties"), + mode=0755, + content=Template("master.properties.j2"), + owner=params.dolphin_user, + group=params.dolphin_group + ) + + File(format(params.dolphin_conf_dir + "/worker.properties"), + mode=0755, + content=Template("worker.properties.j2"), + owner=params.dolphin_user, + group=params.dolphin_group + ) + File(format(params.dolphin_conf_dir + "/alert.properties"), mode=0755, @@ -92,9 +87,9 @@ def dolphin_env(): group=params.dolphin_group ) - File(format(params.dolphin_conf_dir + "/application.properties"), + File(format(params.dolphin_conf_dir + "/datasource.properties"), mode=0755, - content=Template("application.properties.j2"), + content=Template("datasource.properties.j2"), owner=params.dolphin_user, group=params.dolphin_group ) @@ -119,3 +114,10 @@ def dolphin_env(): owner=params.dolphin_user, group=params.dolphin_group ) + + File(format(params.dolphin_conf_dir + "/zookeeper.properties"), + mode=0755, + content=Template("zookeeper.properties.j2"), + owner=params.dolphin_user, + group=params.dolphin_group + ) diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/dolphin_logger_service.py b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_logger_service.py similarity index 92% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/dolphin_logger_service.py rename to ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_logger_service.py index f1c19bd66f..fb47e132e1 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/dolphin_logger_service.py +++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_logger_service.py @@ -26,8 +26,8 @@ class DolphinLoggerService(Script): import params env.set_params(params) self.install_packages(env) - Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True) - + Execute(('chmod', '-R', '777', params.dolphin_home)) + Execute(('chown', '-R', params.dolphin_user + ":" + params.dolphin_group, params.dolphin_home)) def configure(self, env): import params params.pika_slave = True diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/dolphin_master_service.py b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_master_service.py similarity index 92% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/dolphin_master_service.py rename to ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_master_service.py index 6ee7ecfcf3..8d64935d26 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/dolphin_master_service.py +++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_master_service.py @@ -27,7 +27,8 @@ class DolphinMasterService(Script): import params env.set_params(params) self.install_packages(env) - Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True) + Execute(('chmod', '-R', '777', params.dolphin_home)) + Execute(('chown', '-R', params.dolphin_user + ":" + params.dolphin_group, params.dolphin_home)) def configure(self, env): import params diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/dolphin_worker_service.py b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_worker_service.py similarity index 92% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/dolphin_worker_service.py rename to ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_worker_service.py index 2d145ee730..1f542c06c2 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/dolphin_worker_service.py +++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_worker_service.py @@ -26,7 +26,8 @@ class DolphinWorkerService(Script): import params env.set_params(params) self.install_packages(env) - Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True) + Execute(('chmod', '-R', '777', params.dolphin_home)) + Execute(('chown', '-R', params.dolphin_user + ":" + params.dolphin_group, params.dolphin_home)) def configure(self, env): import params diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/params.py b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/params.py similarity index 68% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/params.py rename to ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/params.py index 049b2cf3ae..5a9994f559 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/params.py +++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/params.py @@ -54,11 +54,8 @@ dolphin_env_content = dolphin_env_map['dolphinscheduler-env-content'] # database config dolphin_database_config = {} dolphin_database_config['dolphin_database_type'] = dolphin_env_map['dolphin.database.type'] -dolphin_database_config['dolphin_database_host'] = dolphin_env_map['dolphin.database.host'] -dolphin_database_config['dolphin_database_port'] = dolphin_env_map['dolphin.database.port'] dolphin_database_config['dolphin_database_username'] = dolphin_env_map['dolphin.database.username'] dolphin_database_config['dolphin_database_password'] = dolphin_env_map['dolphin.database.password'] - if 'mysql' == dolphin_database_config['dolphin_database_type']: dolphin_database_config['dolphin_database_driver'] = 'com.mysql.jdbc.Driver' dolphin_database_config['driverDelegateClass'] = 'org.quartz.impl.jdbcjobstore.StdJDBCDelegate' @@ -72,6 +69,10 @@ else: + ':' + dolphin_env_map['dolphin.database.port'] \ + '/dolphinscheduler' + + + + # application-alert.properties dolphin_alert_map = {} wechat_push_url = 'https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=$token' @@ -79,27 +80,22 @@ wechat_token_url = 'https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid=$corpId& wechat_team_send_msg = '{\"toparty\":\"{toParty}\",\"agentid\":\"{agentId}\",\"msgtype\":\"text\",\"text\":{\"content\":\"{msg}\"},\"safe\":\"0\"}' wechat_user_send_msg = '{\"touser\":\"{toUser}\",\"agentid\":\"{agentId}\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"{msg}\"}}' -dolphin_alert_map['enterprise.wechat.push.ur'] = wechat_push_url -dolphin_alert_map['enterprise.wechat.token.url'] = wechat_token_url -dolphin_alert_map['enterprise.wechat.team.send.msg'] = wechat_team_send_msg -dolphin_alert_map['enterprise.wechat.user.send.msg'] = wechat_user_send_msg -dolphin_alert_map.update(config['configurations']['dolphin-alert']) +dolphin_alert_config_map = config['configurations']['dolphin-alert'] + +if dolphin_alert_config_map['enterprise.wechat.enable']: + dolphin_alert_map['enterprise.wechat.push.ur'] = wechat_push_url + dolphin_alert_map['enterprise.wechat.token.url'] = wechat_token_url + dolphin_alert_map['enterprise.wechat.team.send.msg'] = wechat_team_send_msg + dolphin_alert_map['enterprise.wechat.user.send.msg'] = wechat_user_send_msg + +dolphin_alert_map.update(dolphin_alert_config_map) + + # application-api.properties dolphin_app_api_map = {} -dolphin_app_api_map['logging.config'] = 'classpath:apiserver_logback.xml' -dolphin_app_api_map['spring.messages.basename'] = 'i18n/messages' -dolphin_app_api_map['server.servlet.context-path'] = '/dolphinscheduler/' dolphin_app_api_map.update(config['configurations']['dolphin-application-api']) -# application-dao.properties -dolphin_application_map = {} -dolphin_application_map['spring.datasource.type'] = 'com.alibaba.druid.pool.DruidDataSource' -dolphin_application_map['spring.datasource.driver-class-name'] = dolphin_database_config['dolphin_database_driver'] -dolphin_application_map['spring.datasource.url'] = dolphin_database_config['dolphin_database_url'] -dolphin_application_map['spring.datasource.username'] = dolphin_database_config['dolphin_database_username'] -dolphin_application_map['spring.datasource.password'] = dolphin_database_config['dolphin_database_password'] -dolphin_application_map.update(config['configurations']['dolphin-application']) # common.properties dolphin_common_map = {} @@ -118,33 +114,42 @@ else: dolphin_common_map_tmp = config['configurations']['dolphin-common'] data_basedir_path = dolphin_common_map_tmp['data.basedir.path'] -process_exec_basepath = data_basedir_path + '/exec' -data_download_basedir_path = data_basedir_path + '/download' -dolphin_common_map['process.exec.basepath'] = process_exec_basepath -dolphin_common_map['data.download.basedir.path'] = data_download_basedir_path dolphin_common_map['dolphinscheduler.env.path'] = dolphin_env_path +dolphin_common_map.update(config['configurations']['dolphin-common']) -zookeeperHosts = default("/clusterHostInfo/zookeeper_hosts", []) -if len(zookeeperHosts) > 0 and "clientPort" in config['configurations']['zoo.cfg']: - clientPort = config['configurations']['zoo.cfg']['clientPort'] - zookeeperPort = ":" + clientPort + "," - dolphin_common_map['zookeeper.quorum'] = zookeeperPort.join(zookeeperHosts) + ":" + clientPort +# datasource.properties +dolphin_datasource_map = {} +dolphin_datasource_map['spring.datasource.type'] = 'com.alibaba.druid.pool.DruidDataSource' +dolphin_datasource_map['spring.datasource.driver-class-name'] = dolphin_database_config['dolphin_database_driver'] +dolphin_datasource_map['spring.datasource.url'] = dolphin_database_config['dolphin_database_url'] +dolphin_datasource_map['spring.datasource.username'] = dolphin_database_config['dolphin_database_username'] +dolphin_datasource_map['spring.datasource.password'] = dolphin_database_config['dolphin_database_password'] +dolphin_datasource_map.update(config['configurations']['dolphin-datasource']) -dolphin_common_map.update(config['configurations']['dolphin-common']) +# master.properties +dolphin_master_map = config['configurations']['dolphin-master'] # quartz.properties dolphin_quartz_map = {} dolphin_quartz_map['org.quartz.jobStore.driverDelegateClass'] = dolphin_database_config['driverDelegateClass'] -dolphin_quartz_map['org.quartz.dataSource.myDs.driver'] = dolphin_database_config['dolphin_database_driver'] -dolphin_quartz_map['org.quartz.dataSource.myDs.URL'] = dolphin_database_config['dolphin_database_url'] -dolphin_quartz_map['org.quartz.dataSource.myDs.user'] = dolphin_database_config['dolphin_database_username'] -dolphin_quartz_map['org.quartz.dataSource.myDs.password'] = dolphin_database_config['dolphin_database_password'] dolphin_quartz_map.update(config['configurations']['dolphin-quartz']) -# if 'ganglia_server_host' in config['clusterHostInfo'] and \ -# len(config['clusterHostInfo']['ganglia_server_host'])>0: -# ganglia_installed = True -# ganglia_server = config['clusterHostInfo']['ganglia_server_host'][0] -# ganglia_report_interval = 60 -# else: -# ganglia_installed = False +# worker.properties +dolphin_worker_map = config['configurations']['dolphin-worker'] + +# zookeeper.properties +dolphin_zookeeper_map={} +zookeeperHosts = default("/clusterHostInfo/zookeeper_hosts", []) +if len(zookeeperHosts) > 0 and "clientPort" in config['configurations']['zoo.cfg']: + clientPort = config['configurations']['zoo.cfg']['clientPort'] + zookeeperPort = ":" + clientPort + "," + dolphin_zookeeper_map['zookeeper.quorum'] = zookeeperPort.join(zookeeperHosts) + ":" + clientPort +dolphin_zookeeper_map.update(config['configurations']['dolphin-zookeeper']) +if 'spring.servlet.multipart.max-file-size' in dolphin_app_api_map: + file_size = dolphin_app_api_map['spring.servlet.multipart.max-file-size'] + dolphin_app_api_map['spring.servlet.multipart.max-file-size'] = file_size + "MB" +if 'spring.servlet.multipart.max-request-size' in dolphin_app_api_map: + request_size = dolphin_app_api_map['spring.servlet.multipart.max-request-size'] + dolphin_app_api_map['spring.servlet.multipart.max-request-size'] = request_size + "MB" + + diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/service_check.py b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/service_check.py similarity index 100% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/service_check.py rename to ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/service_check.py diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/status_params.py b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/status_params.py similarity index 100% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/status_params.py rename to ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/status_params.py diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/templates/alert.properties.j2 b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/alert.properties.j2 similarity index 100% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/package/templates/alert.properties.j2 rename to ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/alert.properties.j2 diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/templates/application-api.properties.j2 b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/application-api.properties.j2 similarity index 100% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/package/templates/application-api.properties.j2 rename to ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/application-api.properties.j2 diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/templates/common.properties.j2 b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/common.properties.j2 similarity index 100% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/package/templates/common.properties.j2 rename to ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/common.properties.j2 diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/datasource.properties.j2 b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/datasource.properties.j2 new file mode 100644 index 0000000000..40aed83543 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/datasource.properties.j2 @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{% for key, value in dolphin_datasource_map.iteritems() -%} + {{key}}={{value}} +{% endfor %} \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/templates/dolphin-daemon.j2 b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/dolphin-daemon.sh.j2 similarity index 83% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/package/templates/dolphin-daemon.j2 rename to ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/dolphin-daemon.sh.j2 index 1dc4bac0ab..c5cc11fb62 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/templates/dolphin-daemon.j2 +++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/dolphin-daemon.sh.j2 @@ -48,22 +48,19 @@ pid={{dolphin_pidfile_dir}}/$command.pid cd $DOLPHINSCHEDULER_HOME if [ "$command" = "api-server" ]; then - LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/apiserver_logback.xml -Dspring.profiles.active=api" + LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/logback-api.xml -Dspring.profiles.active=api" CLASS=org.apache.dolphinscheduler.api.ApiApplicationServer elif [ "$command" = "master-server" ]; then - LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/master_logback.xml -Ddruid.mysql.usePingMethod=false" + LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/logback-master.xml -Ddruid.mysql.usePingMethod=false" CLASS=org.apache.dolphinscheduler.server.master.MasterServer elif [ "$command" = "worker-server" ]; then - LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/worker_logback.xml -Ddruid.mysql.usePingMethod=false" + LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/logback-worker.xml -Ddruid.mysql.usePingMethod=false" CLASS=org.apache.dolphinscheduler.server.worker.WorkerServer elif [ "$command" = "alert-server" ]; then - LOG_FILE="-Dlogback.configurationFile={{dolphin_conf_dir}}/alert_logback.xml" + LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/logback-alert.xml" CLASS=org.apache.dolphinscheduler.alert.AlertServer elif [ "$command" = "logger-server" ]; then - CLASS=org.apache.dolphinscheduler.server.rpc.LoggerServer -elif [ "$command" = "combined-server" ]; then - LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/combined_logback.xml -Dspring.profiles.active=api -Dserver.is-combined-server=true" - CLASS=org.apache.dolphinscheduler.api.CombinedApplicationServer + CLASS=org.apache.dolphinscheduler.server.log.LoggerServer else echo "Error: No command named \`$command' was found." exit 1 diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/master.properties.j2 b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/master.properties.j2 new file mode 100644 index 0000000000..d9b85e14cf --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/master.properties.j2 @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{% for key, value in dolphin_master_map.iteritems() -%} + {{key}}={{value}} +{% endfor %} \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/templates/quartz.properties.j2 b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/quartz.properties.j2 similarity index 100% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/package/templates/quartz.properties.j2 rename to ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/quartz.properties.j2 diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/worker.properties.j2 b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/worker.properties.j2 new file mode 100644 index 0000000000..a008b74084 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/worker.properties.j2 @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{% for key, value in dolphin_worker_map.iteritems() -%} + {{key}}={{value}} +{% endfor %} \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/zookeeper.properties.j2 b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/zookeeper.properties.j2 new file mode 100644 index 0000000000..9eb14eaef3 --- /dev/null +++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/zookeeper.properties.j2 @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{% for key, value in dolphin_zookeeper_map.iteritems() -%} + {{key}}={{value}} +{% endfor %} \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/quicklinks/quicklinks.json b/ambari_plugin/common-services/DOLPHIN/1.3.3/quicklinks/quicklinks.json old mode 100755 new mode 100644 similarity index 100% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/quicklinks/quicklinks.json rename to ambari_plugin/common-services/DOLPHIN/1.3.3/quicklinks/quicklinks.json diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/themes/theme.json b/ambari_plugin/common-services/DOLPHIN/1.3.3/themes/theme.json similarity index 86% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/themes/theme.json rename to ambari_plugin/common-services/DOLPHIN/1.3.3/themes/theme.json index 23e46076aa..953e2323f8 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.2.1/themes/theme.json +++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/themes/theme.json @@ -151,18 +151,40 @@ "subsection-name": "env-row1-col2" }, { - "config": "dolphin-common/res.upload.startup.type", + "config": "dolphin-common/resource.storage.type", "subsection-name": "dynamic-row1-col1" }, + { + "config": "dolphin-common/resource.upload.path", + "subsection-name": "dynamic-row1-col1", + "depends-on": [ + { + "configs":[ + "dolphin-common/resource.storage.type" + ], + "if": "${dolphin-common/resource.storage.type} === HDFS || ${dolphin-common/resource.storage.type} === S3", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, { "config": "dolphin-common/hdfs.root.user", "subsection-name": "dynamic-row1-col1", "depends-on": [ { "configs":[ - "dolphin-common/res.upload.startup.type" + "dolphin-common/resource.storage.type" ], - "if": "${dolphin-common/res.upload.startup.type} === HDFS", + "if": "${dolphin-common/resource.storage.type} === HDFS", "then": { "property_value_attributes": { "visible": true @@ -182,9 +204,9 @@ "depends-on": [ { "configs":[ - "dolphin-common/res.upload.startup.type" + "dolphin-common/resource.storage.type" ], - "if": "${dolphin-common/res.upload.startup.type} === HDFS", + "if": "${dolphin-common/resource.storage.type} === HDFS", "then": { "property_value_attributes": { "visible": true @@ -204,9 +226,9 @@ "depends-on": [ { "configs":[ - "dolphin-common/res.upload.startup.type" + "dolphin-common/resource.storage.type" ], - "if": "${dolphin-common/res.upload.startup.type} === HDFS", + "if": "${dolphin-common/resource.storage.type} === HDFS", "then": { "property_value_attributes": { "visible": true @@ -226,9 +248,9 @@ "depends-on": [ { "configs":[ - "dolphin-common/res.upload.startup.type" + "dolphin-common/resource.storage.type" ], - "if": "${dolphin-common/res.upload.startup.type} === S3", + "if": "${dolphin-common/resource.storage.type} === S3", "then": { "property_value_attributes": { "visible": true @@ -248,9 +270,9 @@ "depends-on": [ { "configs":[ - "dolphin-common/res.upload.startup.type" + "dolphin-common/resource.storage.type" ], - "if": "${dolphin-common/res.upload.startup.type} === S3", + "if": "${dolphin-common/resource.storage.type} === S3", "then": { "property_value_attributes": { "visible": true @@ -270,9 +292,9 @@ "depends-on": [ { "configs":[ - "dolphin-common/res.upload.startup.type" + "dolphin-common/resource.storage.type" ], - "if": "${dolphin-common/res.upload.startup.type} === S3", + "if": "${dolphin-common/resource.storage.type} === S3", "then": { "property_value_attributes": { "visible": true @@ -356,6 +378,28 @@ } ] }, + { + "config": "dolphin-common/kerberos.expire.time", + "subsection-name": "dynamic-row1-col2", + "depends-on": [ + { + "configs":[ + "dolphin-common/hadoop.security.authentication.startup.state" + ], + "if": "${dolphin-common/hadoop.security.authentication.startup.state}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, { "config": "dolphin-alert/enterprise.wechat.enable", "subsection-name": "dynamic-row1-col3" @@ -505,11 +549,17 @@ } }, { - "config": "dolphin-common/res.upload.startup.type", + "config": "dolphin-common/resource.storage.type", "widget": { "type": "combo" } }, + { + "config": "dolphin-common/resource.upload.path", + "widget": { + "type": "text-field" + } + }, { "config": "dolphin-common/hdfs.root.user", "widget": { @@ -570,6 +620,12 @@ "type": "text-field" } }, + { + "config": "dolphin-common/kerberos.expire.time", + "widget": { + "type": "text-field" + } + }, { "config": "dolphin-alert/enterprise.wechat.enable", "widget": { diff --git a/docker/build/Dockerfile b/docker/build/Dockerfile index 7b0f09c539..086deae70a 100644 --- a/docker/build/Dockerfile +++ b/docker/build/Dockerfile @@ -42,7 +42,7 @@ ADD ./apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz RUN mv /opt/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin/ /opt/dolphinscheduler/ ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler -#4. install database, if use mysql as your backend database, the `mysql-client` package should be installed +#4. install database, if use mysql as your backend database, you should append `mysql-client` at the end of the sentence RUN apk add --update --no-cache postgresql postgresql-contrib #5. modify nginx diff --git a/docker/build/startup.sh b/docker/build/startup.sh index ea4fbc0cd1..0511788d48 100644 --- a/docker/build/startup.sh +++ b/docker/build/startup.sh @@ -38,13 +38,13 @@ initDatabase() { echo "connect ${DATABASE_TYPE} service" if [ ${DATABASE_TYPE} = "mysql" ]; then v=$(mysql -h${DATABASE_HOST} -P${DATABASE_PORT} -u${DATABASE_USERNAME} --password=${DATABASE_PASSWORD} -D ${DATABASE_DATABASE} -e "select 1" 2>&1) - if [ "$(echo '${v}' | grep 'ERROR' | wc -l)" -eq 1 ]; then + if [ "$(echo ${v} | grep 'ERROR' | wc -l)" -eq 1 ]; then echo "Error: Can't connect to database...${v}" exit 1 fi else v=$(sudo -u postgres PGPASSWORD=${DATABASE_PASSWORD} psql -h ${DATABASE_HOST} -p ${DATABASE_PORT} -U ${DATABASE_USERNAME} -d ${DATABASE_DATABASE} -tAc "select 1") - if [ "$(echo '${v}' | grep 'FATAL' | wc -l)" -eq 1 ]; then + if [ "$(echo ${v} | grep 'FATAL' | wc -l)" -eq 1 ]; then echo "Error: Can't connect to database...${v}" exit 1 fi diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/templates/application.properties.j2 b/docker/kubernetes/dolphinscheduler/requirements.yaml similarity index 76% rename from ambari_plugin/common-services/DOLPHIN/1.2.1/package/templates/application.properties.j2 rename to docker/kubernetes/dolphinscheduler/requirements.yaml index 7bb9f8aff3..e219975995 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/templates/application.properties.j2 +++ b/docker/kubernetes/dolphinscheduler/requirements.yaml @@ -14,7 +14,12 @@ # See the License for the specific language governing permissions and # limitations under the License. # - -{% for key, value in dolphin_application_map.iteritems() -%} - {{key}}={{value}} -{% endfor %} \ No newline at end of file +dependencies: +- name: postgresql + version: 8.x.x + repository: https://charts.bitnami.com/bitnami + condition: postgresql.enabled +- name: zookeeper + version: 5.x.x + repository: https://charts.bitnami.com/bitnami + condition: redis.enabled \ No newline at end of file diff --git a/dolphinscheduler-alert/pom.xml b/dolphinscheduler-alert/pom.xml index 215916ddf7..a44d101ffe 100644 --- a/dolphinscheduler-alert/pom.xml +++ b/dolphinscheduler-alert/pom.xml @@ -21,7 +21,7 @@ org.apache.dolphinscheduler dolphinscheduler - 1.3.2-SNAPSHOT + 1.3.4-SNAPSHOT dolphinscheduler-alert ${project.artifactId} diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java index ef1022755f..b3cb5f7c4e 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java @@ -104,11 +104,7 @@ public class EnterpriseWeChatUtils { } Map map = JSONUtils.toMap(resp); - if (map != null) { - return map.get("access_token"); - } else { - return null; - } + return map == null ? null : map.get("access_token"); } finally { httpClient.close(); } diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPluginTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPluginTest.java index c2f4006752..cdbcf276d3 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPluginTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPluginTest.java @@ -74,7 +74,7 @@ public class EmailAlertPluginTest { alertInfo.setAlertData(alertData); List list = new ArrayList(){{ add("xx@xx.com"); }}; alertInfo.addProp("receivers", list); - Map ret = plugin.process(alertInfo); - assertFalse(Boolean.parseBoolean(String.valueOf(ret.get(Constants.STATUS)))); +// Map ret = plugin.process(alertInfo); +// assertFalse(Boolean.parseBoolean(String.valueOf(ret.get(Constants.STATUS)))); } } \ No newline at end of file diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index 76dd8980b7..2af0ed2ffa 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -21,7 +21,7 @@ org.apache.dolphinscheduler dolphinscheduler - 1.3.2-SNAPSHOT + 1.3.4-SNAPSHOT dolphinscheduler-api ${project.artifactId} @@ -162,6 +162,12 @@ org.apache.hadoop hadoop-client + + + org.slf4j + slf4j-log4j12 + + diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AccessTokenController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AccessTokenController.java index 2457177cdf..17faad04bc 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AccessTokenController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AccessTokenController.java @@ -86,7 +86,7 @@ public class AccessTokenController extends BaseController { logger.info("login user {}, create token , userId : {} , token expire time : {} , token : {}", loginUser.getUserName(), userId, expireTime, token); - Map result = accessTokenService.createToken(userId, expireTime, token); + Map result = accessTokenService.createToken(loginUser, userId, expireTime, token); return returnDataList(result); } @@ -106,7 +106,7 @@ public class AccessTokenController extends BaseController { @RequestParam(value = "userId") int userId, @RequestParam(value = "expireTime") String expireTime) { logger.info("login user {}, generate token , userId : {} , token expire time : {}", loginUser, userId, expireTime); - Map result = accessTokenService.generateToken(userId, expireTime); + Map result = accessTokenService.generateToken(loginUser, userId, expireTime); return returnDataList(result); } @@ -185,7 +185,7 @@ public class AccessTokenController extends BaseController { logger.info("login user {}, update token , userId : {} , token expire time : {} , token : {}", loginUser.getUserName(), userId, expireTime, token); - Map result = accessTokenService.updateToken(id, userId, expireTime, token); + Map result = accessTokenService.updateToken(loginUser, id, userId, expireTime, token); return returnDataList(result); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectController.java index dac97bca9d..1bf5003946 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectController.java @@ -228,21 +228,20 @@ public class ProjectController extends BaseController { } /** - * query user created project + * query authorized and user created project * * @param loginUser login user - * @return projects which the user create + * @return projects which the user create and authorized */ - @ApiOperation(value = "queryProjectCreatedByUser", notes = "QUERY_USER_CREATED_PROJECT_NOTES") - - @GetMapping(value = "/login-user-created-project") + @ApiOperation(value = "queryProjectCreatedAndAuthorizedByUser", notes = "QUERY_AUTHORIZED_AND_USER_CREATED_PROJECT_NOTES") + @GetMapping(value = "/created-and-authorized-project") @ResponseStatus(HttpStatus.OK) - @ApiException(QUERY_USER_CREATED_PROJECT_ERROR) - public Result queryProjectCreatedByUser(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser) { - logger.info("login user {}, query authorized project by user id: {}.", + @ApiException(QUERY_AUTHORIZED_AND_USER_CREATED_PROJECT_ERROR) + public Result queryProjectCreatedAndAuthorizedByUser(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser) { + logger.info("login user {}, query authorized and user created project by user id: {}.", StringUtils.replaceNRTtoUnderline(loginUser.getUserName()), StringUtils.replaceNRTtoUnderline(String.valueOf(loginUser.getId()))); - Map result = projectService.queryProjectCreatedByUser(loginUser); + Map result = projectService.queryProjectCreatedAndAuthorizedByUser(loginUser); return returnDataList(result); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java index a0ec666ed7..07859f7006 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java @@ -14,23 +14,48 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.api.controller; +import static org.apache.dolphinscheduler.api.enums.Status.AUTHORIZED_FILE_RESOURCE_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.AUTHORIZED_UDF_FUNCTION_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.AUTHORIZE_RESOURCE_TREE; +import static org.apache.dolphinscheduler.api.enums.Status.CREATE_RESOURCE_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.CREATE_RESOURCE_FILE_ON_LINE_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.CREATE_UDF_FUNCTION_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.DELETE_RESOURCE_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.DELETE_UDF_FUNCTION_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.DOWNLOAD_RESOURCE_FILE_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.EDIT_RESOURCE_FILE_ON_LINE_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.QUERY_DATASOURCE_BY_TYPE_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.QUERY_RESOURCES_LIST_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.QUERY_RESOURCES_LIST_PAGING; +import static org.apache.dolphinscheduler.api.enums.Status.QUERY_UDF_FUNCTION_LIST_PAGING_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.RESOURCE_FILE_IS_EMPTY; +import static org.apache.dolphinscheduler.api.enums.Status.RESOURCE_NOT_EXIST; +import static org.apache.dolphinscheduler.api.enums.Status.UNAUTHORIZED_UDF_FUNCTION_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_RESOURCE_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_UDF_FUNCTION_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.VERIFY_RESOURCE_BY_NAME_AND_TYPE_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.VERIFY_UDF_FUNCTION_NAME_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.VIEW_RESOURCE_FILE_ON_LINE_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.VIEW_UDF_FUNCTION_ERROR; + import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.ResourcesService; import org.apache.dolphinscheduler.api.service.UdfFuncService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.ProgramType; import org.apache.dolphinscheduler.common.enums.ResourceType; import org.apache.dolphinscheduler.common.enums.UdfType; import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.dao.entity.User; -import io.swagger.annotations.Api; -import io.swagger.annotations.ApiImplicitParam; -import io.swagger.annotations.ApiImplicitParams; -import io.swagger.annotations.ApiOperation; -import org.apache.commons.lang.StringUtils; + +import java.util.Map; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -38,13 +63,21 @@ import org.springframework.core.io.Resource; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.*; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestAttribute; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; import org.springframework.web.multipart.MultipartFile; -import springfox.documentation.annotations.ApiIgnore; - -import java.util.Map; -import static org.apache.dolphinscheduler.api.enums.Status.*; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiImplicitParam; +import io.swagger.annotations.ApiImplicitParams; +import io.swagger.annotations.ApiOperation; +import springfox.documentation.annotations.ApiIgnore; /** * resources controller @@ -56,37 +89,28 @@ public class ResourcesController extends BaseController { private static final Logger logger = LoggerFactory.getLogger(ResourcesController.class); - @Autowired private ResourcesService resourceService; @Autowired private UdfFuncService udfFuncService; /** - * create directory * - * @param loginUser login user - * @param alias alias - * @param description description - * @param type type - * @return create result code - */ - - /** * @param loginUser login user * @param type type * @param alias alias * @param description description * @param pid parent id * @param currentDir current directory - * @return + * @return create result code */ @ApiOperation(value = "createDirctory", notes = "CREATE_RESOURCE_NOTES") @ApiImplicitParams({ @ApiImplicitParam(name = "type", value = "RESOURCE_TYPE", required = true, dataType = "ResourceType"), @ApiImplicitParam(name = "name", value = "RESOURCE_NAME", required = true, dataType = "String"), @ApiImplicitParam(name = "description", value = "RESOURCE_DESC", dataType = "String"), - @ApiImplicitParam(name = "file", value = "RESOURCE_FILE", required = true, dataType = "MultipartFile") + @ApiImplicitParam(name = "pid", value = "RESOURCE_PID", required = true, dataType = "Int"), + @ApiImplicitParam(name = "currentDir", value = "RESOURCE_CURRENTDIR", required = true, dataType = "String") }) @PostMapping(value = "/directory/create") @ApiException(CREATE_RESOURCE_ERROR) @@ -103,12 +127,13 @@ public class ResourcesController extends BaseController { /** * create resource - * - * @param loginUser login user - * @param alias alias - * @param description description - * @param type type - * @param file file + * @param loginUser + * @param type + * @param alias + * @param description + * @param file + * @param pid + * @param currentDir * @return create result code */ @ApiOperation(value = "createResource", notes = "CREATE_RESOURCE_NOTES") @@ -116,7 +141,9 @@ public class ResourcesController extends BaseController { @ApiImplicitParam(name = "type", value = "RESOURCE_TYPE", required = true, dataType = "ResourceType"), @ApiImplicitParam(name = "name", value = "RESOURCE_NAME", required = true, dataType = "String"), @ApiImplicitParam(name = "description", value = "RESOURCE_DESC", dataType = "String"), - @ApiImplicitParam(name = "file", value = "RESOURCE_FILE", required = true, dataType = "MultipartFile") + @ApiImplicitParam(name = "file", value = "RESOURCE_FILE", required = true, dataType = "MultipartFile"), + @ApiImplicitParam(name = "pid", value = "RESOURCE_PID", required = true, dataType = "Int"), + @ApiImplicitParam(name = "currentDir", value = "RESOURCE_CURRENTDIR", required = true, dataType = "String") }) @PostMapping(value = "/create") @ApiException(CREATE_RESOURCE_ERROR) @@ -135,11 +162,12 @@ public class ResourcesController extends BaseController { /** * update resource * - * @param loginUser login user - * @param alias alias - * @param resourceId resource id - * @param type resource type + * @param loginUser login user + * @param alias alias + * @param resourceId resource id + * @param type resource type * @param description description + * @param file resource file * @return update result code */ @ApiOperation(value = "updateResource", notes = "UPDATE_RESOURCE_NOTES") @@ -147,7 +175,8 @@ public class ResourcesController extends BaseController { @ApiImplicitParam(name = "id", value = "RESOURCE_ID", required = true, dataType = "Int", example = "100"), @ApiImplicitParam(name = "type", value = "RESOURCE_TYPE", required = true, dataType = "ResourceType"), @ApiImplicitParam(name = "name", value = "RESOURCE_NAME", required = true, dataType = "String"), - @ApiImplicitParam(name = "description", value = "RESOURCE_DESC", dataType = "String") + @ApiImplicitParam(name = "description", value = "RESOURCE_DESC", dataType = "String"), + @ApiImplicitParam(name = "file", value = "RESOURCE_FILE", required = true, dataType = "MultipartFile") }) @PostMapping(value = "/update") @ApiException(UPDATE_RESOURCE_ERROR) @@ -155,17 +184,18 @@ public class ResourcesController extends BaseController { @RequestParam(value = "id") int resourceId, @RequestParam(value = "type") ResourceType type, @RequestParam(value = "name") String alias, - @RequestParam(value = "description", required = false) String description) { - logger.info("login user {}, update resource, type: {}, resource alias: {}, desc: {}", - loginUser.getUserName(), type, alias, description); - return resourceService.updateResource(loginUser, resourceId, alias, description, type); + @RequestParam(value = "description", required = false) String description, + @RequestParam(value = "file" ,required = false) MultipartFile file) { + logger.info("login user {}, update resource, type: {}, resource alias: {}, desc: {}, file: {}", + loginUser.getUserName(), type, alias, description, file); + return resourceService.updateResource(loginUser, resourceId, alias, description, type, file); } /** * query resources list * * @param loginUser login user - * @param type resource type + * @param type resource type * @return resource list */ @ApiOperation(value = "queryResourceList", notes = "QUERY_RESOURCE_LIST_NOTES") @@ -187,10 +217,10 @@ public class ResourcesController extends BaseController { * query resources list paging * * @param loginUser login user - * @param type resource type + * @param type resource type * @param searchVal search value - * @param pageNo page number - * @param pageSize page size + * @param pageNo page number + * @param pageSize page size * @return resource list page */ @ApiOperation(value = "queryResourceListPaging", notes = "QUERY_RESOURCE_LIST_PAGING_NOTES") @@ -227,7 +257,7 @@ public class ResourcesController extends BaseController { /** * delete resource * - * @param loginUser login user + * @param loginUser login user * @param resourceId resource id * @return delete result code */ @@ -251,8 +281,8 @@ public class ResourcesController extends BaseController { * verify resource by alias and type * * @param loginUser login user - * @param fullName resource full name - * @param type resource type + * @param fullName resource full name + * @param type resource type * @return true if the resource name not exists, otherwise return false */ @ApiOperation(value = "verifyResourceName", notes = "VERIFY_RESOURCE_NAME_NOTES") @@ -277,10 +307,10 @@ public class ResourcesController extends BaseController { * query resources jar list * * @param loginUser login user - * @param type resource type + * @param type resource type * @return resource list */ - @ApiOperation(value = "queryResourceJarList", notes = "QUERY_RESOURCE_LIST_NOTES") + @ApiOperation(value = "queryResourceByProgramType", notes = "QUERY_RESOURCE_LIST_NOTES") @ApiImplicitParams({ @ApiImplicitParam(name = "type", value = "RESOURCE_TYPE", required = true, dataType = "ResourceType") }) @@ -288,10 +318,14 @@ public class ResourcesController extends BaseController { @ResponseStatus(HttpStatus.OK) @ApiException(QUERY_RESOURCES_LIST_ERROR) public Result queryResourceJarList(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @RequestParam(value = "type") ResourceType type + @RequestParam(value = "type") ResourceType type, + @RequestParam(value = "programType",required = false) ProgramType programType ) { - logger.info("query resource list, login user:{}, resource type:{}", loginUser.getUserName(), type.toString()); - Map result = resourceService.queryResourceJarList(loginUser, type); + String programTypeName = programType == null ? "" : programType.name(); + String userName = loginUser.getUserName(); + userName = userName.replaceAll("[\n|\r|\t]", "_"); + logger.info("query resource list, login user:{}, resource type:{}, program type:{}", userName,programTypeName); + Map result = resourceService.queryResourceByProgramType(loginUser, type,programType); return returnDataList(result); } @@ -299,14 +333,16 @@ public class ResourcesController extends BaseController { * query resource by full name and type * * @param loginUser login user - * @param fullName resource full name - * @param type resource type + * @param fullName resource full name + * @param type resource type + * @param id resource id * @return true if the resource name not exists, otherwise return false */ @ApiOperation(value = "queryResource", notes = "QUERY_BY_RESOURCE_NAME") @ApiImplicitParams({ @ApiImplicitParam(name = "type", value = "RESOURCE_TYPE", required = true, dataType = "ResourceType"), - @ApiImplicitParam(name = "fullName", value = "RESOURCE_FULL_NAME", required = true, dataType = "String") + @ApiImplicitParam(name = "fullName", value = "RESOURCE_FULL_NAME", required = true, dataType = "String"), + @ApiImplicitParam(name = "id", value = "RESOURCE_ID", required = false, dataType = "Int") }) @GetMapping(value = "/queryResource") @ResponseStatus(HttpStatus.OK) @@ -325,10 +361,10 @@ public class ResourcesController extends BaseController { /** * view resource file online * - * @param loginUser login user - * @param resourceId resource id + * @param loginUser login user + * @param resourceId resource id * @param skipLineNum skip line number - * @param limit limit + * @param limit limit * @return resource content */ @ApiOperation(value = "viewResource", notes = "VIEW_RESOURCE_BY_ID_NOTES") @@ -352,13 +388,14 @@ public class ResourcesController extends BaseController { /** * create resource file online - * - * @param loginUser login user - * @param type resource type - * @param fileName file name - * @param fileSuffix file suffix - * @param description description - * @param content content + * @param loginUser + * @param type + * @param fileName + * @param fileSuffix + * @param description + * @param content + * @param pid + * @param currentDir * @return create result code */ @ApiOperation(value = "onlineCreateResource", notes = "ONLINE_CREATE_RESOURCE_NOTES") @@ -367,7 +404,9 @@ public class ResourcesController extends BaseController { @ApiImplicitParam(name = "fileName", value = "RESOURCE_NAME", required = true, dataType = "String"), @ApiImplicitParam(name = "suffix", value = "SUFFIX", required = true, dataType = "String"), @ApiImplicitParam(name = "description", value = "RESOURCE_DESC", dataType = "String"), - @ApiImplicitParam(name = "content", value = "CONTENT", required = true, dataType = "String") + @ApiImplicitParam(name = "content", value = "CONTENT", required = true, dataType = "String"), + @ApiImplicitParam(name = "pid", value = "RESOURCE_PID", required = true, dataType = "Int"), + @ApiImplicitParam(name = "currentDir", value = "RESOURCE_CURRENTDIR", required = true, dataType = "String") }) @PostMapping(value = "/online-create") @ApiException(CREATE_RESOURCE_FILE_ON_LINE_ERROR) @@ -392,9 +431,9 @@ public class ResourcesController extends BaseController { /** * edit resource file online * - * @param loginUser login user + * @param loginUser login user * @param resourceId resource id - * @param content content + * @param content content * @return update result code */ @ApiOperation(value = "updateResourceContent", notes = "UPDATE_RESOURCE_NOTES") @@ -420,7 +459,7 @@ public class ResourcesController extends BaseController { /** * download resource file * - * @param loginUser login user + * @param loginUser login user * @param resourceId resource id * @return resource content */ @@ -449,14 +488,14 @@ public class ResourcesController extends BaseController { /** * create udf function * - * @param loginUser login user - * @param type udf type - * @param funcName function name - * @param argTypes argument types - * @param database database + * @param loginUser login user + * @param type udf type + * @param funcName function name + * @param argTypes argument types + * @param database database * @param description description - * @param className class name - * @param resourceId resource id + * @param className class name + * @param resourceId resource id * @return create result code */ @ApiOperation(value = "createUdfFunc", notes = "CREATE_UDF_FUNCTION_NOTES") @@ -490,12 +529,12 @@ public class ResourcesController extends BaseController { * view udf function * * @param loginUser login user - * @param id resource id + * @param id resource id * @return udf function detail */ @ApiOperation(value = "viewUIUdfFunction", notes = "VIEW_UDF_FUNCTION_NOTES") @ApiImplicitParams({ - @ApiImplicitParam(name = "resourceId", value = "RESOURCE_ID", required = true, dataType = "Int", example = "100") + @ApiImplicitParam(name = "id", value = "RESOURCE_ID", required = true, dataType = "Int", example = "100") }) @GetMapping(value = "/udf-func/update-ui") @@ -512,26 +551,27 @@ public class ResourcesController extends BaseController { /** * update udf function * - * @param loginUser login user - * @param type resource type - * @param funcName function name - * @param argTypes argument types - * @param database data base + * @param loginUser login user + * @param type resource type + * @param funcName function name + * @param argTypes argument types + * @param database data base * @param description description - * @param resourceId resource id - * @param className class name - * @param udfFuncId udf function id + * @param resourceId resource id + * @param className class name + * @param udfFuncId udf function id * @return update result code */ @ApiOperation(value = "updateUdfFunc", notes = "UPDATE_UDF_FUNCTION_NOTES") @ApiImplicitParams({ + @ApiImplicitParam(name = "id", value = "UDF_ID", required = true, dataType = "Int"), @ApiImplicitParam(name = "type", value = "UDF_TYPE", required = true, dataType = "UdfType"), @ApiImplicitParam(name = "funcName", value = "FUNC_NAME", required = true, dataType = "String"), - @ApiImplicitParam(name = "suffix", value = "CLASS_NAME", required = true, dataType = "String"), + @ApiImplicitParam(name = "className", value = "CLASS_NAME", required = true, dataType = "String"), @ApiImplicitParam(name = "argTypes", value = "ARG_TYPES", dataType = "String"), @ApiImplicitParam(name = "database", value = "DATABASE_NAME", dataType = "String"), @ApiImplicitParam(name = "description", value = "UDF_DESC", dataType = "String"), - @ApiImplicitParam(name = "id", value = "RESOURCE_ID", required = true, dataType = "Int", example = "100") + @ApiImplicitParam(name = "resourceId", value = "RESOURCE_ID", required = true, dataType = "Int", example = "100") }) @PostMapping(value = "/udf-func/update") @@ -556,8 +596,8 @@ public class ResourcesController extends BaseController { * * @param loginUser login user * @param searchVal search value - * @param pageNo page number - * @param pageSize page size + * @param pageNo page number + * @param pageSize page size * @return udf function list page */ @ApiOperation(value = "queryUdfFuncListPaging", notes = "QUERY_UDF_FUNCTION_LIST_PAGING_NOTES") @@ -569,7 +609,7 @@ public class ResourcesController extends BaseController { @GetMapping(value = "/udf-func/list-paging") @ResponseStatus(HttpStatus.OK) @ApiException(QUERY_UDF_FUNCTION_LIST_PAGING_ERROR) - public Result queryUdfFuncList(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + public Result queryUdfFuncListPaging(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam("pageNo") Integer pageNo, @RequestParam(value = "searchVal", required = false) String searchVal, @RequestParam("pageSize") Integer pageSize @@ -586,23 +626,25 @@ public class ResourcesController extends BaseController { } /** - * query resource list by type + * query udf func list by type * * @param loginUser login user - * @param type resource type + * @param type resource type * @return resource list */ - @ApiOperation(value = "queryResourceList", notes = "QUERY_RESOURCE_LIST_NOTES") + @ApiOperation(value = "queryUdfFuncList", notes = "QUERY_UDF_FUNC_LIST_NOTES") @ApiImplicitParams({ @ApiImplicitParam(name = "type", value = "UDF_TYPE", required = true, dataType = "UdfType") }) @GetMapping(value = "/udf-func/list") @ResponseStatus(HttpStatus.OK) @ApiException(QUERY_DATASOURCE_BY_TYPE_ERROR) - public Result queryResourceList(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + public Result queryUdfFuncList(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam("type") UdfType type) { - logger.info("query datasource list, user:{}, type:{}", loginUser.getUserName(), type); - Map result = udfFuncService.queryResourceList(loginUser, type.ordinal()); + String userName = loginUser.getUserName(); + userName = userName.replaceAll("[\n|\r|\t]", "_"); + logger.info("query udf func list, user:{}, type:{}", userName, type); + Map result = udfFuncService.queryUdfFuncList(loginUser, type.ordinal()); return returnDataList(result); } @@ -610,7 +652,7 @@ public class ResourcesController extends BaseController { * verify udf function name can use or not * * @param loginUser login user - * @param name name + * @param name name * @return true if the name can user, otherwise return false */ @ApiOperation(value = "verifyUdfFuncName", notes = "VERIFY_UDF_FUNCTION_NAME_NOTES") @@ -655,7 +697,7 @@ public class ResourcesController extends BaseController { * authorized file resource list * * @param loginUser login user - * @param userId user id + * @param userId user id * @return authorized result */ @ApiOperation(value = "authorizedFile", notes = "AUTHORIZED_FILE_NOTES") @@ -677,7 +719,7 @@ public class ResourcesController extends BaseController { * unauthorized file resource list * * @param loginUser login user - * @param userId user id + * @param userId user id * @return unauthorized result code */ @ApiOperation(value = "authorizeResourceTree", notes = "AUTHORIZE_RESOURCE_TREE_NOTES") @@ -699,7 +741,7 @@ public class ResourcesController extends BaseController { * unauthorized udf function * * @param loginUser login user - * @param userId user id + * @param userId user id * @return unauthorized result code */ @ApiOperation(value = "unauthUDFFunc", notes = "UNAUTHORIZED_UDF_FUNC_NOTES") @@ -722,7 +764,7 @@ public class ResourcesController extends BaseController { * authorized udf function * * @param loginUser login user - * @param userId user id + * @param userId user id * @return authorized result code */ @ApiOperation(value = "authUDFFunc", notes = "AUTHORIZED_UDF_FUNC_NOTES") diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskInstanceController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskInstanceController.java index c0ad88f481..56e7ef2087 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskInstanceController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskInstanceController.java @@ -14,8 +14,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.api.controller; +import static org.apache.dolphinscheduler.api.enums.Status.QUERY_TASK_LIST_PAGING_ERROR; import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.TaskInstanceService; @@ -23,18 +25,29 @@ import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.dao.entity.User; -import io.swagger.annotations.*; + +import java.util.Map; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; -import org.springframework.web.bind.annotation.*; -import springfox.documentation.annotations.ApiIgnore; - -import java.util.Map; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestAttribute; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; -import static org.apache.dolphinscheduler.api.enums.Status.QUERY_TASK_LIST_PAGING_ERROR; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiImplicitParam; +import io.swagger.annotations.ApiImplicitParams; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import springfox.documentation.annotations.ApiIgnore; /** * task instance controller @@ -69,6 +82,7 @@ public class TaskInstanceController extends BaseController { @ApiOperation(value = "queryTaskListPaging", notes = "QUERY_TASK_INSTANCE_LIST_PAGING_NOTES") @ApiImplicitParams({ @ApiImplicitParam(name = "processInstanceId", value = "PROCESS_INSTANCE_ID", required = false, dataType = "Int", example = "100"), + @ApiImplicitParam(name = "processInstanceName", value = "PROCESS_INSTANCE_NAME", required = false, type = "String"), @ApiImplicitParam(name = "searchVal", value = "SEARCH_VAL", type = "String"), @ApiImplicitParam(name = "taskName", value = "TASK_NAME", type = "String"), @ApiImplicitParam(name = "executorName", value = "EXECUTOR_NAME", type = "String"), @@ -85,6 +99,7 @@ public class TaskInstanceController extends BaseController { public Result queryTaskListPaging(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName, @RequestParam(value = "processInstanceId", required = false, defaultValue = "0") Integer processInstanceId, + @RequestParam(value = "processInstanceName", required = false) String processInstanceName, @RequestParam(value = "searchVal", required = false) String searchVal, @RequestParam(value = "taskName", required = false) String taskName, @RequestParam(value = "executorName", required = false) String executorName, @@ -95,11 +110,20 @@ public class TaskInstanceController extends BaseController { @RequestParam("pageNo") Integer pageNo, @RequestParam("pageSize") Integer pageSize) { - logger.info("query task instance list, project name:{},process instance:{}, search value:{},task name:{}, executor name: {},state type:{}, host:{}, start:{}, end:{}", - projectName, processInstanceId, searchVal, taskName, executorName, stateType, host, startTime, endTime); + logger.info("query task instance list, projectName:{}, processInstanceId:{}, processInstanceName:{}, search value:{}, taskName:{}, executorName: {}, stateType:{}, host:{}, start:{}, end:{}", + StringUtils.replaceNRTtoUnderline(projectName), + processInstanceId, + StringUtils.replaceNRTtoUnderline(processInstanceName), + StringUtils.replaceNRTtoUnderline(searchVal), + StringUtils.replaceNRTtoUnderline(taskName), + StringUtils.replaceNRTtoUnderline(executorName), + stateType, + StringUtils.replaceNRTtoUnderline(host), + StringUtils.replaceNRTtoUnderline(startTime), + StringUtils.replaceNRTtoUnderline(endTime)); searchVal = ParameterUtils.handleEscapes(searchVal); Map result = taskInstanceService.queryTaskListPaging( - loginUser, projectName, processInstanceId, taskName, executorName, startTime, endTime, searchVal, stateType, host, pageNo, pageSize); + loginUser, projectName, processInstanceId, processInstanceName, taskName, executorName, startTime, endTime, searchVal, stateType, host, pageNo, pageSize); return returnDataListPaging(result); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskRecordController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskRecordController.java index e20c845d42..4ff769dce6 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskRecordController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskRecordController.java @@ -22,11 +22,13 @@ import org.apache.dolphinscheduler.api.service.TaskRecordService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.dao.entity.User; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; import org.springframework.web.bind.annotation.*; + import springfox.documentation.annotations.ApiIgnore; import java.util.Map; @@ -59,7 +61,7 @@ public class TaskRecordController extends BaseController { * @param taskDate task date * @param startTime start time * @param endTime end time - * @param pageNo page numbere + * @param pageNo page number * @param pageSize page size * @return task record list */ diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TenantController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TenantController.java index 2676a774e7..4c5dacb696 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TenantController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TenantController.java @@ -29,6 +29,7 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.TenantService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; @@ -72,7 +73,6 @@ public class TenantController extends BaseController { * * @param loginUser login user * @param tenantCode tenant code - * @param tenantName tenant name * @param queueId queue id * @param description description * @return create result code @@ -80,7 +80,6 @@ public class TenantController extends BaseController { @ApiOperation(value = "createTenant", notes = "CREATE_TENANT_NOTES") @ApiImplicitParams({ @ApiImplicitParam(name = "tenantCode", value = "TENANT_CODE", required = true, dataType = "String"), - @ApiImplicitParam(name = "tenantName", value = "TENANT_NAME", required = true, dataType = "String"), @ApiImplicitParam(name = "queueId", value = "QUEUE_ID", required = true, dataType = "Int", example = "100"), @ApiImplicitParam(name = "description", value = "TENANT_DESC", dataType = "String") @@ -90,12 +89,13 @@ public class TenantController extends BaseController { @ApiException(CREATE_TENANT_ERROR) public Result createTenant(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam(value = "tenantCode") String tenantCode, - @RequestParam(value = "tenantName") String tenantName, @RequestParam(value = "queueId") int queueId, @RequestParam(value = "description", required = false) String description) throws Exception { - logger.info("login user {}, create tenant, tenantCode: {}, tenantName: {}, queueId: {}, desc: {}", - loginUser.getUserName(), tenantCode, tenantName, queueId, description); - Map result = tenantService.createTenant(loginUser, tenantCode, tenantName, queueId, description); + String userReplace = StringUtils.replaceNRTtoUnderline(loginUser.getUserName()); + String tenantCodeReplace = StringUtils.replaceNRTtoUnderline(tenantCode); + String descReplace = StringUtils.replaceNRTtoUnderline(description); + logger.info("login user {}, create tenant, tenantCode: {}, queueId: {}, desc: {}", userReplace, tenantCodeReplace, queueId, descReplace); + Map result = tenantService.createTenant(loginUser, tenantCode, queueId, description); return returnDataList(result); } @@ -157,7 +157,6 @@ public class TenantController extends BaseController { * @param loginUser login user * @param id tennat id * @param tenantCode tennat code - * @param tenantName tennat name * @param queueId queue id * @param description description * @return update result code @@ -166,7 +165,6 @@ public class TenantController extends BaseController { @ApiImplicitParams({ @ApiImplicitParam(name = "ID", value = "TENANT_ID", required = true, dataType = "Int", example = "100"), @ApiImplicitParam(name = "tenantCode", value = "TENANT_CODE", required = true, dataType = "String"), - @ApiImplicitParam(name = "tenantName", value = "TENANT_NAME", required = true, dataType = "String"), @ApiImplicitParam(name = "queueId", value = "QUEUE_ID", required = true, dataType = "Int", example = "100"), @ApiImplicitParam(name = "description", value = "TENANT_DESC", type = "String") @@ -177,12 +175,13 @@ public class TenantController extends BaseController { public Result updateTenant(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam(value = "id") int id, @RequestParam(value = "tenantCode") String tenantCode, - @RequestParam(value = "tenantName") String tenantName, @RequestParam(value = "queueId") int queueId, @RequestParam(value = "description", required = false) String description) throws Exception { - logger.info("login user {}, updateProcessInstance tenant, tenantCode: {}, tenantName: {}, queueId: {}, description: {}", - loginUser.getUserName(), tenantCode, tenantName, queueId, description); - Map result = tenantService.updateTenant(loginUser, id, tenantCode, tenantName, queueId, description); + String userReplace = StringUtils.replaceNRTtoUnderline(loginUser.getUserName()); + String tenantCodeReplace = StringUtils.replaceNRTtoUnderline(tenantCode); + String descReplace = StringUtils.replaceNRTtoUnderline(description); + logger.info("login user {}, create tenant, tenantCode: {}, queueId: {}, desc: {}", userReplace, tenantCodeReplace, queueId, descReplace); + Map result = tenantService.updateTenant(loginUser, id, tenantCode, queueId, description); return returnDataList(result); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/UsersController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/UsersController.java index 8d6f9fc820..9b16265e32 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/UsersController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/UsersController.java @@ -166,7 +166,7 @@ public class UsersController extends BaseController { @RequestParam(value = "state", required = false) int state) throws Exception { logger.info("login user {}, updateProcessInstance user, userName: {}, email: {}, tenantId: {}, userPassword: {}, phone: {}, user queue: {}, state: {}", loginUser.getUserName(), userName, email, tenantId, Constants.PASSWORD_DEFAULT, phone, queue, state); - Map result = usersService.updateUser(id, userName, userPassword, email, tenantId, phone, queue, state); + Map result = usersService.updateUser(loginUser, id, userName, userPassword, email, tenantId, phone, queue, state); return returnDataList(result); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/CommandStateCount.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/CommandStateCount.java index 3c3c31bfce..d116e46578 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/CommandStateCount.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/CommandStateCount.java @@ -57,4 +57,32 @@ public class CommandStateCount { public void setCommandState(CommandType commandState) { this.commandState = commandState; } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + CommandStateCount that = (CommandStateCount) o; + + if (errorCount != that.errorCount) { + return false; + } + if (normalCount != that.normalCount) { + return false; + } + return commandState == that.commandState; + } + + @Override + public int hashCode() { + int result = errorCount; + result = 31 * result + normalCount; + result = 31 * result + (commandState != null ? commandState.hashCode() : 0); + return result; + } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/TaskCountDto.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/TaskCountDto.java index 35aaaf34dd..6f42ba9f56 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/TaskCountDto.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/TaskCountDto.java @@ -19,8 +19,10 @@ package org.apache.dolphinscheduler.api.dto; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.dao.entity.ExecuteStatusCount; -import java.util.ArrayList; +import java.util.Arrays; import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; /** * task count dto @@ -43,77 +45,16 @@ public class TaskCountDto { } private void countTaskDtos(List taskInstanceStateCounts) { - int submittedSuccess = 0; - int runningExecution = 0; - int delayExecution = 0; - int readyPause = 0; - int pause = 0; - int readyStop = 0; - int stop = 0; - int failure = 0; - int success = 0; - int needFaultTolerance = 0; - int kill = 0; - int waittingThread = 0; + Map statusCountMap = taskInstanceStateCounts.stream() + .collect(Collectors.toMap(ExecuteStatusCount::getExecutionStatus, ExecuteStatusCount::getCount, Integer::sum)); - for (ExecuteStatusCount taskInstanceStateCount : taskInstanceStateCounts) { - ExecutionStatus status = taskInstanceStateCount.getExecutionStatus(); - totalCount += taskInstanceStateCount.getCount(); - switch (status) { - case SUBMITTED_SUCCESS: - submittedSuccess += taskInstanceStateCount.getCount(); - break; - case RUNNING_EXECUTION: - runningExecution += taskInstanceStateCount.getCount(); - break; - case DELAY_EXECUTION: - delayExecution += taskInstanceStateCount.getCount(); - break; - case READY_PAUSE: - readyPause += taskInstanceStateCount.getCount(); - break; - case PAUSE: - pause += taskInstanceStateCount.getCount(); - break; - case READY_STOP: - readyStop += taskInstanceStateCount.getCount(); - break; - case STOP: - stop += taskInstanceStateCount.getCount(); - break; - case FAILURE: - failure += taskInstanceStateCount.getCount(); - break; - case SUCCESS: - success += taskInstanceStateCount.getCount(); - break; - case NEED_FAULT_TOLERANCE: - needFaultTolerance += taskInstanceStateCount.getCount(); - break; - case KILL: - kill += taskInstanceStateCount.getCount(); - break; - case WAITTING_THREAD: - waittingThread += taskInstanceStateCount.getCount(); - break; + taskCountDtos = Arrays.stream(ExecutionStatus.values()) + .map(status -> new TaskStateCount(status, statusCountMap.getOrDefault(status, 0))) + .collect(Collectors.toList()); - default: - break; - } - } - this.taskCountDtos = new ArrayList<>(); - this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.SUBMITTED_SUCCESS, submittedSuccess)); - this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.RUNNING_EXECUTION, runningExecution)); - this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.DELAY_EXECUTION, delayExecution)); - this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.READY_PAUSE, readyPause)); - this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.PAUSE, pause)); - this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.READY_STOP, readyStop)); - this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.STOP, stop)); - this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.FAILURE, failure)); - this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.SUCCESS, success)); - this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.NEED_FAULT_TOLERANCE, needFaultTolerance)); - this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.KILL, kill)); - this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.WAITTING_THREAD, waittingThread)); + totalCount = taskCountDtos.stream() + .mapToInt(TaskStateCount::getCount) + .sum(); } public List getTaskCountDtos() { diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/TaskStateCount.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/TaskStateCount.java index a2fe348e40..460ea7da4b 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/TaskStateCount.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/TaskStateCount.java @@ -47,4 +47,28 @@ public class TaskStateCount { public void setTaskStateType(ExecutionStatus taskStateType) { this.taskStateType = taskStateType; } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + TaskStateCount that = (TaskStateCount) o; + + if (count != that.count) { + return false; + } + return taskStateType == that.taskStateType; + } + + @Override + public int hashCode() { + int result = count; + result = 31 * result + (taskStateType != null ? taskStateType.hashCode() : 0); + return result; + } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java index 43c03f09d8..6e23808c1f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java @@ -36,7 +36,7 @@ public enum Status { USER_NAME_NULL(10004, "user name is null", "用户名不能为空"), HDFS_OPERATION_ERROR(10006, "hdfs operation error", "hdfs操作错误"), TASK_INSTANCE_NOT_FOUND(10008, "task instance not found", "任务实例不存在"), - TENANT_NAME_EXIST(10009, "tenant code {0} already exists", "租户编码[{0}]已存在"), + TENANT_CODE_EXIST(10009, "tenant code {0} already exists", "租户编码[{0}]已存在"), USER_NOT_EXIST(10010, "user {0} not exists", "用户[{0}]不存在"), ALERT_GROUP_NOT_EXIST(10011, "alarm group not found", "告警组不存在"), ALERT_GROUP_EXIST(10012, "alarm group already exists", "告警组名称已存在"), @@ -133,7 +133,7 @@ public enum Status { QUERY_TASK_INSTANCE_LOG_ERROR(10103, "view task instance log error", "查询任务实例日志错误"), DOWNLOAD_TASK_INSTANCE_LOG_FILE_ERROR(10104, "download task instance log file error", "下载任务日志文件错误"), CREATE_PROCESS_DEFINITION(10105, "create process definition", "创建工作流错误"), - VERIFY_PROCESS_DEFINITION_NAME_UNIQUE_ERROR(10106, "verify process definition name unique error", "工作流名称已存在"), + VERIFY_PROCESS_DEFINITION_NAME_UNIQUE_ERROR(10106, "verify process definition name unique error", "工作流定义名称已存在"), UPDATE_PROCESS_DEFINITION_ERROR(10107, "update process definition error", "更新工作流定义错误"), RELEASE_PROCESS_DEFINITION_ERROR(10108, "release process definition error", "上线工作流错误"), QUERY_DATAIL_OF_PROCESS_DEFINITION_ERROR(10109, "query datail of process definition error", "查询工作流详细信息错误"), @@ -193,7 +193,9 @@ public enum Status { BATCH_COPY_PROCESS_DEFINITION_ERROR(10159, "batch copy process definition error", "复制工作流错误"), BATCH_MOVE_PROCESS_DEFINITION_ERROR(10160, "batch move process definition error", "移动工作流错误"), QUERY_WORKFLOW_LINEAGE_ERROR(10161, "query workflow lineage error", "查询血缘失败"), - DELETE_PROCESS_DEFINITION_BY_ID_FAIL(10162,"delete process definition by id fail, for there are {0} process instances in executing using it", "删除工作流定义失败，有[{0}]个运行中的工作流实例正在使用"), + QUERY_AUTHORIZED_AND_USER_CREATED_PROJECT_ERROR(10162, "query authorized and user created project error error", "查询授权的和用户创建的项目错误"), + DELETE_PROCESS_DEFINITION_BY_ID_FAIL(10163,"delete process definition by id fail, for there are {0} process instances in executing using it", "删除工作流定义失败，有[{0}]个运行中的工作流实例正在使用"), + CHECK_TENANT_CODE_ERROR(10164, "Please enter the English tenant code", "请输入英文租户编码"), UDF_FUNCTION_NOT_EXIST(20001, "UDF function not found", "UDF函数不存在"), UDF_FUNCTION_EXISTS(20002, "UDF function already exists", "UDF函数已存在"), diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AccessTokenService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AccessTokenService.java index 98eef47090..b1c320566f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AccessTokenService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AccessTokenService.java @@ -44,7 +44,8 @@ public interface AccessTokenService { * @param token token string * @return create result code */ - Map createToken(int userId, String expireTime, String token); + Map createToken(User loginUser, int userId, String expireTime, String token); + /** * generate token @@ -53,7 +54,7 @@ public interface AccessTokenService { * @param expireTime token expire time * @return token string */ - Map generateToken(int userId, String expireTime); + Map generateToken(User loginUser, int userId, String expireTime); /** * delete access token @@ -73,5 +74,5 @@ public interface AccessTokenService { * @param token token string * @return update result code */ - Map updateToken(int id, int userId, String expireTime, String token); + Map updateToken(User loginUser, int id, int userId, String expireTime, String token); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseService.java index 4b094ea494..e9b25250a8 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseService.java @@ -95,6 +95,25 @@ public class BaseService { } + /** + * check + * + * @param result result + * @param bool bool + * @param userNoOperationPerm status + * @return check result + */ + protected boolean check(Map result, boolean bool, Status userNoOperationPerm) { + //only admin can operate + if (bool) { + result.put(Constants.STATUS, userNoOperationPerm); + result.put(Constants.MSG, userNoOperationPerm.getMsg()); + return true; + } + return false; + } + + /** * get cookie info by name * diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java index 74c2f6908f..1fa0494d13 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java @@ -499,7 +499,13 @@ public class DataSourceService extends BaseService { String address = buildAddress(type, host, port, connectType); Map parameterMap = new LinkedHashMap(6); - String jdbcUrl = address + "/" + database; + String jdbcUrl; + if (DbType.SQLSERVER == type) { + jdbcUrl = address + ";databaseName=" + database; + } else { + jdbcUrl = address + "/" + database; + } + if (Constants.ORACLE.equals(type.name())) { parameterMap.put(Constants.ORACLE_DB_CONNECT_TYPE, connectType); } @@ -533,19 +539,17 @@ public class DataSourceService extends BaseService { (type == DbType.HIVE || type == DbType.SPARK)) { parameterMap.put(Constants.PRINCIPAL, principal); } - if (other != null && !"".equals(other)) { - Map map = JSONUtils.toMap(other); - if (map.size() > 0) { - StringBuilder otherSb = new StringBuilder(); - for (Map.Entry entry : map.entrySet()) { - otherSb.append(String.format("%s=%s%s", entry.getKey(), entry.getValue(), separator)); - } - if (!Constants.DB2.equals(type.name())) { - otherSb.deleteCharAt(otherSb.length() - 1); - } - parameterMap.put(Constants.OTHER, otherSb); - } + Map map = JSONUtils.toMap(other); + if (map != null) { + StringBuilder otherSb = new StringBuilder(); + for (Map.Entry entry: map.entrySet()) { + otherSb.append(String.format("%s=%s%s", entry.getKey(), entry.getValue(), separator)); + } + if (!Constants.DB2.equals(type.name())) { + otherSb.deleteCharAt(otherSb.length() - 1); + } + parameterMap.put(Constants.OTHER, otherSb); } if (logger.isDebugEnabled()) { diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java index c8d3c74da0..739fa19735 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java @@ -305,10 +305,10 @@ public class ProcessInstanceService extends BaseService { private void addDependResultForTaskList(List taskInstanceList) throws IOException { for (TaskInstance taskInstance : taskInstanceList) { if (taskInstance.getTaskType().equalsIgnoreCase(TaskType.DEPENDENT.toString())) { - Result logResult = loggerService.queryLog( + Result logResult = loggerService.queryLog( taskInstance.getId(), 0, 4098); if (logResult.getCode() == Status.SUCCESS.ordinal()) { - String log = (String) logResult.getData(); + String log = logResult.getData(); Map resultMap = parseLogForDependentResult(log); taskInstance.setDependentResult(JSONUtils.toJsonString(resultMap)); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProjectService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProjectService.java index ca0e1fc0ec..4753725d31 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProjectService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProjectService.java @@ -120,4 +120,11 @@ public interface ProjectService { */ Map queryAllProjectList(); + /** + * query authorized and user create project list by user id + * @param loginUser + * @return + */ + Map queryProjectCreatedAndAuthorizedByUser(User loginUser); + } \ No newline at end of file diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java index bd7598979d..dffc55edfe 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java @@ -29,6 +29,7 @@ import org.apache.dolphinscheduler.api.exceptions.ServiceException; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.ProgramType; import org.apache.dolphinscheduler.common.enums.ResourceType; import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.dao.entity.*; @@ -87,7 +88,7 @@ public class ResourcesService extends BaseService { * @param currentDir current directory * @return create directory result */ - @Transactional(rollbackFor = RuntimeException.class) + @Transactional(rollbackFor = Exception.class) public Result createDirectory(User loginUser, String name, String description, @@ -101,8 +102,11 @@ public class ResourcesService extends BaseService { putMsg(result, Status.HDFS_NOT_STARTUP); return result; } - String fullName = "/".equals(currentDir) ? String.format("%s%s",currentDir,name):String.format("%s/%s",currentDir,name); - + String fullName = currentDir.equals("/") ? String.format("%s%s",currentDir,name):String.format("%s/%s",currentDir,name); + result = verifyResourceName(fullName,type,loginUser); + if (!result.getCode().equals(Status.SUCCESS.getCode())) { + return result; + } if (pid != -1) { Resource parentResource = resourcesMapper.selectById(pid); @@ -165,7 +169,7 @@ public class ResourcesService extends BaseService { * @param currentDir current directory * @return create result code */ - @Transactional(rollbackFor = RuntimeException.class) + @Transactional(rollbackFor = Exception.class) public Result createResource(User loginUser, String name, String desc, @@ -230,7 +234,7 @@ public class ResourcesService extends BaseService { } // check resoure name exists - String fullName = "/".equals(currentDir) ? String.format("%s%s",currentDir,name):String.format("%s/%s",currentDir,name); + String fullName = currentDir.equals("/") ? String.format("%s%s",currentDir,name):String.format("%s/%s",currentDir,name); if (checkResourceExists(fullName, 0, type.ordinal())) { logger.error("resource {} has exist, can't recreate", name); putMsg(result, Status.RESOURCE_EXIST); @@ -288,14 +292,16 @@ public class ResourcesService extends BaseService { * @param name name * @param desc description * @param type resource type + * @param file resource file * @return update result code */ - @Transactional(rollbackFor = RuntimeException.class) + @Transactional(rollbackFor = Exception.class) public Result updateResource(User loginUser, int resourceId, String name, String desc, - ResourceType type) { + ResourceType type, + MultipartFile file) { Result result = new Result(); // if resource upload startup @@ -315,7 +321,7 @@ public class ResourcesService extends BaseService { return result; } - if (name.equals(resource.getAlias()) && desc.equals(resource.getDescription())) { + if (file == null && name.equals(resource.getAlias()) && desc.equals(resource.getDescription())) { putMsg(result, Status.SUCCESS); return result; } @@ -331,6 +337,42 @@ public class ResourcesService extends BaseService { return result; } + if (file != null) { + + // file is empty + if (file.isEmpty()) { + logger.error("file is empty: {}", file.getOriginalFilename()); + putMsg(result, Status.RESOURCE_FILE_IS_EMPTY); + return result; + } + + // file suffix + String fileSuffix = FileUtils.suffix(file.getOriginalFilename()); + String nameSuffix = FileUtils.suffix(name); + + // determine file suffix + if (!(StringUtils.isNotEmpty(fileSuffix) && fileSuffix.equalsIgnoreCase(nameSuffix))) { + /** + * rename file suffix and original suffix must be consistent + */ + logger.error("rename file suffix and original suffix must be consistent: {}", file.getOriginalFilename()); + putMsg(result, Status.RESOURCE_SUFFIX_FORBID_CHANGE); + return result; + } + + //If resource type is UDF, only jar packages are allowed to be uploaded, and the suffix must be .jar + if (Constants.UDF.equals(type.name()) && !JAR.equalsIgnoreCase(FileUtils.suffix(originFullName))) { + logger.error(Status.UDF_RESOURCE_SUFFIX_NOT_JAR.getMsg()); + putMsg(result, Status.UDF_RESOURCE_SUFFIX_NOT_JAR); + return result; + } + if (file.getSize() > Constants.MAX_FILE_SIZE) { + logger.error("file size is too large: {}", file.getOriginalFilename()); + putMsg(result, Status.RESOURCE_SIZE_EXCEED_LIMIT); + return result; + } + } + // query tenant by user id String tenantCode = getTenantCode(resource.getUserId(),result); if (StringUtils.isEmpty(tenantCode)){ @@ -380,31 +422,61 @@ public class ResourcesService extends BaseService { } // updateResource data - List childrenResource = listAllChildren(resource,false); Date now = new Date(); resource.setAlias(name); resource.setFullName(fullName); resource.setDescription(desc); resource.setUpdateTime(now); + if (file != null) { + resource.setFileName(file.getOriginalFilename()); + resource.setSize(file.getSize()); + } try { resourcesMapper.updateById(resource); - if (resource.isDirectory() && CollectionUtils.isNotEmpty(childrenResource)) { - String matcherFullName = Matcher.quoteReplacement(fullName); - List childResourceList = new ArrayList<>(); - List resourceList = resourcesMapper.listResourceByIds(childrenResource.toArray(new Integer[childrenResource.size()])); - childResourceList = resourceList.stream().map(t -> { - t.setFullName(t.getFullName().replaceFirst(originFullName, matcherFullName)); - t.setUpdateTime(now); - return t; - }).collect(Collectors.toList()); - resourcesMapper.batchUpdateResource(childResourceList); + if (resource.isDirectory()) { + List childrenResource = listAllChildren(resource,false); + if (CollectionUtils.isNotEmpty(childrenResource)) { + String matcherFullName = Matcher.quoteReplacement(fullName); + List childResourceList = new ArrayList<>(); + Integer[] childResIdArray = childrenResource.toArray(new Integer[childrenResource.size()]); + List resourceList = resourcesMapper.listResourceByIds(childResIdArray); + childResourceList = resourceList.stream().map(t -> { + t.setFullName(t.getFullName().replaceFirst(originFullName, matcherFullName)); + t.setUpdateTime(now); + return t; + }).collect(Collectors.toList()); + resourcesMapper.batchUpdateResource(childResourceList); + + if (ResourceType.UDF.equals(resource.getType())) { + List udfFuncs = udfFunctionMapper.listUdfByResourceId(childResIdArray); + if (CollectionUtils.isNotEmpty(udfFuncs)) { + udfFuncs = udfFuncs.stream().map(t -> { + t.setResourceName(t.getResourceName().replaceFirst(originFullName, matcherFullName)); + t.setUpdateTime(now); + return t; + }).collect(Collectors.toList()); + udfFunctionMapper.batchUpdateUdfFunc(udfFuncs); + } + } + } + } else if (ResourceType.UDF.equals(resource.getType())) { + List udfFuncs = udfFunctionMapper.listUdfByResourceId(new Integer[]{resourceId}); + if (CollectionUtils.isNotEmpty(udfFuncs)) { + udfFuncs = udfFuncs.stream().map(t -> { + t.setResourceName(fullName); + t.setUpdateTime(now); + return t; + }).collect(Collectors.toList()); + udfFunctionMapper.batchUpdateUdfFunc(udfFuncs); + } + } putMsg(result, Status.SUCCESS); Map dataMap = new BeanMap(resource); - Map resultMap = new HashMap<>(); + Map resultMap = new HashMap<>(5); for (Map.Entry entry: dataMap.entrySet()) { if (!Constants.CLASS.equalsIgnoreCase(entry.getKey().toString())) { resultMap.put(entry.getKey().toString(), entry.getValue()); @@ -415,11 +487,31 @@ public class ResourcesService extends BaseService { logger.error(Status.UPDATE_RESOURCE_ERROR.getMsg(), e); throw new ServiceException(Status.UPDATE_RESOURCE_ERROR); } + // if name unchanged, return directly without moving on HDFS - if (originResourceName.equals(name)) { + if (originResourceName.equals(name) && file == null) { + return result; + } + + if (file != null) { + // fail upload + if (!upload(loginUser, fullName, file, type)) { + logger.error("upload resource: {} file: {} failed.", name, file.getOriginalFilename()); + putMsg(result, Status.HDFS_OPERATION_ERROR); + throw new RuntimeException(String.format("upload resource: %s file: %s failed.", name, file.getOriginalFilename())); + } + if (!fullName.equals(originFullName)) { + try { + HadoopUtils.getInstance().delete(originHdfsFileName,false); + } catch (IOException e) { + logger.error(e.getMessage(),e); + throw new RuntimeException(String.format("delete resource: %s failed.", originFullName)); + } + } return result; } + // get the path of dest file in hdfs String destHdfsFileName = HadoopUtils.getHdfsFileName(resource.getType(),tenantCode,fullName); @@ -449,7 +541,7 @@ public class ResourcesService extends BaseService { */ public Map queryResourceListPaging(User loginUser, int direcotryId, ResourceType type, String searchVal, Integer pageNo, Integer pageSize) { - HashMap result = new HashMap<>(); + HashMap result = new HashMap<>(5); Page page = new Page(pageNo, pageSize); int userId = loginUser.getId(); if (isAdmin(loginUser)) { @@ -550,7 +642,7 @@ public class ResourcesService extends BaseService { */ public Map queryResourceList(User loginUser, ResourceType type) { - Map result = new HashMap<>(); + Map result = new HashMap<>(5); int userId = loginUser.getId(); if(isAdmin(loginUser)){ @@ -565,21 +657,33 @@ public class ResourcesService extends BaseService { } /** - * query resource list + * query resource list by program type * * @param loginUser login user * @param type resource type * @return resource list */ - public Map queryResourceJarList(User loginUser, ResourceType type) { + public Map queryResourceByProgramType(User loginUser, ResourceType type, ProgramType programType) { - Map result = new HashMap<>(); + Map result = new HashMap<>(5); + String suffix = ".jar"; int userId = loginUser.getId(); if(isAdmin(loginUser)){ userId = 0; } + if (programType != null) { + switch (programType) { + case JAVA: + break; + case SCALA: + break; + case PYTHON: + suffix = ".py"; + break; + } + } List allResourceList = resourcesMapper.queryResourceListAuthored(userId, type.ordinal(),0); - List resources = new ResourceFilter(".jar",new ArrayList<>(allResourceList)).filter(); + List resources = new ResourceFilter(suffix,new ArrayList<>(allResourceList)).filter(); Visitor resourceTreeVisitor = new ResourceTreeVisitor(resources); result.put(Constants.DATA_LIST, resourceTreeVisitor.visit().getChildren()); putMsg(result,Status.SUCCESS); @@ -829,7 +933,7 @@ public class ResourcesService extends BaseService { * @param content content * @return create result code */ - @Transactional(rollbackFor = RuntimeException.class) + @Transactional(rollbackFor = Exception.class) public Result onlineCreateResource(User loginUser, ResourceType type, String fileName, String fileSuffix, String desc, String content,int pid,String currentDirectory) { Result result = new Result(); // if resource upload startup @@ -852,12 +956,25 @@ public class ResourcesService extends BaseService { } String name = fileName.trim() + "." + nameSuffix; - String fullName = "/".equals(currentDirectory) ? String.format("%s%s",currentDirectory,name):String.format("%s/%s",currentDirectory,name); + String fullName = currentDirectory.equals("/") ? String.format("%s%s",currentDirectory,name):String.format("%s/%s",currentDirectory,name); result = verifyResourceName(fullName,type,loginUser); if (!result.getCode().equals(Status.SUCCESS.getCode())) { return result; } + if (pid != -1) { + Resource parentResource = resourcesMapper.selectById(pid); + + if (parentResource == null) { + putMsg(result, Status.PARENT_RESOURCE_NOT_EXIST); + return result; + } + + if (!hasPerm(loginUser, parentResource.getUserId())) { + putMsg(result, Status.USER_NO_OPERATION_PERM); + return result; + } + } // save data Date now = new Date(); @@ -891,7 +1008,7 @@ public class ResourcesService extends BaseService { * @param content content * @return update result cod */ - @Transactional(rollbackFor = RuntimeException.class) + @Transactional(rollbackFor = Exception.class) public Result updateResourceContent(int resourceId, String content) { Result result = new Result(); @@ -1096,7 +1213,7 @@ public class ResourcesService extends BaseService { * @return unauthorized result code */ public Map unauthorizedUDFFunction(User loginUser, Integer userId) { - Map result = new HashMap<>(); + Map result = new HashMap<>(5); //only admin can operate if (checkAdmin(loginUser, result)) { return result; @@ -1148,7 +1265,7 @@ public class ResourcesService extends BaseService { * @return authorized result */ public Map authorizedFile(User loginUser, Integer userId) { - Map result = new HashMap<>(); + Map result = new HashMap<>(5); if (checkAdmin(loginUser, result)){ return result; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskInstanceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskInstanceService.java index 695b76b2bc..012af8fd38 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskInstanceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskInstanceService.java @@ -14,8 +14,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.api.service; +package org.apache.dolphinscheduler.api.service; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.PageInfo; @@ -69,7 +69,6 @@ public class TaskInstanceService extends BaseService { @Autowired UsersService usersService; - /** * query task list by project, process instance, task name, task start time, task end time, task status, keyword paging * @@ -87,7 +86,7 @@ public class TaskInstanceService extends BaseService { * @return task list page */ public Map queryTaskListPaging(User loginUser, String projectName, - Integer processInstanceId, String taskName, String executorName, String startDate, + Integer processInstanceId, String processInstanceName, String taskName, String executorName, String startDate, String endDate, String searchVal, ExecutionStatus stateType, String host, Integer pageNo, Integer pageSize) { Map result = new HashMap<>(); @@ -124,7 +123,7 @@ public class TaskInstanceService extends BaseService { int executorId = usersService.getUserIdByName(executorName); IPage taskInstanceIPage = taskInstanceMapper.queryTaskInstanceListPaging( - page, project.getId(), processInstanceId, searchVal, taskName, executorId, statusArray, host, start, end + page, project.getId(), processInstanceId, processInstanceName, searchVal, taskName, executorId, statusArray, host, start, end ); Set exclusionSet = new HashSet<>(); exclusionSet.add(Constants.CLASS); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TenantService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TenantService.java index 8e83e22a3d..8ab84f9928 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TenantService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TenantService.java @@ -32,7 +32,6 @@ public interface TenantService { * * @param loginUser login user * @param tenantCode tenant code - * @param tenantName tenant name * @param queueId queue id * @param desc description * @return create result code @@ -40,7 +39,6 @@ public interface TenantService { */ Map createTenant(User loginUser, String tenantCode, - String tenantName, int queueId, String desc) throws Exception; @@ -61,13 +59,12 @@ public interface TenantService { * @param loginUser login user * @param id tennat id * @param tenantCode tennat code - * @param tenantName tennat name * @param queueId queue id * @param desc description * @return update result code * @throws Exception exception */ - Map updateTenant(User loginUser, int id, String tenantCode, String tenantName, int queueId, + Map updateTenant(User loginUser, int id, String tenantCode, int queueId, String desc) throws Exception; /** diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java index 04f641f279..cd962fdc70 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java @@ -148,7 +148,7 @@ public class UdfFuncService extends BaseService{ */ public Map queryUdfFuncDetail(int id) { - Map result = new HashMap<>(); + Map result = new HashMap<>(5); UdfFunc udfFunc = udfFuncMapper.selectById(id); if (udfFunc == null) { putMsg(result, Status.RESOURCE_NOT_EXIST); @@ -244,7 +244,7 @@ public class UdfFuncService extends BaseService{ * @return udf function list page */ public Map queryUdfFuncListPaging(User loginUser, String searchVal, Integer pageNo, Integer pageSize) { - Map result = new HashMap<>(); + Map result = new HashMap<>(5); PageInfo pageInfo = new PageInfo(pageNo, pageSize); @@ -276,15 +276,19 @@ public class UdfFuncService extends BaseService{ } /** - * query data resource by type + * query udf list * * @param loginUser login user - * @param type resource type - * @return resource list + * @param type udf type + * @return udf func list */ - public Map queryResourceList(User loginUser, Integer type) { - Map result = new HashMap<>(); - List udfFuncList = udfFuncMapper.getUdfFuncByType(loginUser.getId(), type); + public Map queryUdfFuncList(User loginUser, Integer type) { + Map result = new HashMap<>(5); + int userId = loginUser.getId(); + if (isAdmin(loginUser)) { + userId = 0; + } + List udfFuncList = udfFuncMapper.getUdfFuncByType(userId, type); result.put(Constants.DATA_LIST, udfFuncList); putMsg(result, Status.SUCCESS); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java index 89038ad09f..e47eb42cb5 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java @@ -103,7 +103,7 @@ public class UsersService extends BaseService { String queue, int state) throws Exception { - Map result = new HashMap<>(); + Map result = new HashMap<>(5); //check all user params String msg = this.checkUserParams(userName, userPassword, email, phone); @@ -231,7 +231,7 @@ public class UsersService extends BaseService { * @return user list page */ public Map queryUserList(User loginUser, String searchVal, Integer pageNo, Integer pageSize) { - Map result = new HashMap<>(); + Map result = new HashMap<>(5); if (check(result, !isAdmin(loginUser), Status.USER_NO_OPERATION_PERM)) { return result; @@ -253,6 +253,8 @@ public class UsersService extends BaseService { /** * updateProcessInstance user * + * + * @param loginUser * @param userId user id * @param userName user name * @param userPassword user password @@ -263,7 +265,7 @@ public class UsersService extends BaseService { * @return update result code * @throws Exception exception */ - public Map updateUser(int userId, + public Map updateUser(User loginUser, int userId, String userName, String userPassword, String email, @@ -271,16 +273,17 @@ public class UsersService extends BaseService { String phone, String queue, int state) throws Exception { - Map result = new HashMap<>(); + Map result = new HashMap<>(5); result.put(Constants.STATUS, false); + if (check(result, !hasPerm(loginUser, userId), Status.USER_NO_OPERATION_PERM)) { + return result; + } User user = userMapper.selectById(userId); - if (user == null) { putMsg(result, Status.USER_NOT_EXIST, userId); return result; } - if (StringUtils.isNotEmpty(userName)) { if (!CheckUtils.checkUserName(userName)){ @@ -394,7 +397,7 @@ public class UsersService extends BaseService { * @throws Exception exception when operate hdfs */ public Map deleteUserById(User loginUser, int id) throws Exception { - Map result = new HashMap<>(); + Map result = new HashMap<>(5); //only admin can operate if (!isAdmin(loginUser)) { putMsg(result, Status.USER_NO_OPERATION_PERM, id); @@ -434,7 +437,7 @@ public class UsersService extends BaseService { */ @Transactional(rollbackFor = RuntimeException.class) public Map grantProject(User loginUser, int userId, String projectIds) { - Map result = new HashMap<>(); + Map result = new HashMap<>(5); result.put(Constants.STATUS, false); //only admin can operate @@ -484,7 +487,7 @@ public class UsersService extends BaseService { */ @Transactional(rollbackFor = RuntimeException.class) public Map grantResources(User loginUser, int userId, String resourceIds) { - Map result = new HashMap<>(); + Map result = new HashMap<>(5); //only admin can operate if (check(result, !isAdmin(loginUser), Status.USER_NO_OPERATION_PERM)) { return result; @@ -581,7 +584,7 @@ public class UsersService extends BaseService { */ @Transactional(rollbackFor = RuntimeException.class) public Map grantUDFFunction(User loginUser, int userId, String udfIds) { - Map result = new HashMap<>(); + Map result = new HashMap<>(5); //only admin can operate if (check(result, !isAdmin(loginUser), Status.USER_NO_OPERATION_PERM)) { @@ -628,7 +631,7 @@ public class UsersService extends BaseService { */ @Transactional(rollbackFor = RuntimeException.class) public Map grantDataSource(User loginUser, int userId, String datasourceIds) { - Map result = new HashMap<>(); + Map result = new HashMap<>(5); result.put(Constants.STATUS, false); //only admin can operate @@ -708,7 +711,7 @@ public class UsersService extends BaseService { * @return user list */ public Map queryAllGeneralUsers(User loginUser) { - Map result = new HashMap<>(); + Map result = new HashMap<>(5); //only admin can operate if (check(result, !isAdmin(loginUser), Status.USER_NO_OPERATION_PERM)) { return result; @@ -729,7 +732,7 @@ public class UsersService extends BaseService { * @return user list */ public Map queryUserList(User loginUser) { - Map result = new HashMap<>(); + Map result = new HashMap<>(5); //only admin can operate if (check(result, !isAdmin(loginUser), Status.USER_NO_OPERATION_PERM)) { return result; @@ -773,7 +776,7 @@ public class UsersService extends BaseService { */ public Map unauthorizedUser(User loginUser, Integer alertgroupId) { - Map result = new HashMap<>(); + Map result = new HashMap<>(5); //only admin can operate if (check(result, !isAdmin(loginUser), Status.USER_NO_OPERATION_PERM)) { return result; @@ -809,7 +812,7 @@ public class UsersService extends BaseService { * @return authorized result code */ public Map authorizedUser(User loginUser, Integer alertgroupId) { - Map result = new HashMap<>(); + Map result = new HashMap<>(5); //only admin can operate if (check(result, !isAdmin(loginUser), Status.USER_NO_OPERATION_PERM)) { return result; @@ -821,24 +824,6 @@ public class UsersService extends BaseService { return result; } - /** - * check - * - * @param result result - * @param bool bool - * @param userNoOperationPerm status - * @return check result - */ - private boolean check(Map result, boolean bool, Status userNoOperationPerm) { - //only admin can operate - if (bool) { - result.put(Constants.STATUS, userNoOperationPerm); - result.put(Constants.MSG, userNoOperationPerm.getMsg()); - return true; - } - return false; - } - /** * @param tenantId tenant id * @return true if tenant exists, otherwise return false diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AccessTokenServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AccessTokenServiceImpl.java index 54151d902f..da85621041 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AccessTokenServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AccessTokenServiceImpl.java @@ -86,9 +86,14 @@ public class AccessTokenServiceImpl extends BaseService implements AccessTokenSe * @param token token string * @return create result code */ - public Map createToken(int userId, String expireTime, String token) { + public Map createToken(User loginUser, int userId, String expireTime, String token) { Map result = new HashMap<>(5); + if (!hasPerm(loginUser,userId)){ + putMsg(result, Status.USER_NO_OPERATION_PERM); + return result; + } + if (userId <= 0) { throw new IllegalArgumentException("User id should not less than or equals to 0."); } @@ -118,8 +123,12 @@ public class AccessTokenServiceImpl extends BaseService implements AccessTokenSe * @param expireTime token expire time * @return token string */ - public Map generateToken(int userId, String expireTime) { + public Map generateToken(User loginUser, int userId, String expireTime) { Map result = new HashMap<>(5); + if (!hasPerm(loginUser,userId)){ + putMsg(result, Status.USER_NO_OPERATION_PERM); + return result; + } String token = EncryptionUtils.getMd5(userId + expireTime + System.currentTimeMillis()); result.put(Constants.DATA_LIST, token); putMsg(result, Status.SUCCESS); @@ -144,8 +153,8 @@ public class AccessTokenServiceImpl extends BaseService implements AccessTokenSe return result; } - if (loginUser.getId() != accessToken.getUserId() && - loginUser.getUserType() != UserType.ADMIN_USER) { + + if (!hasPerm(loginUser,accessToken.getUserId())){ putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } @@ -164,9 +173,12 @@ public class AccessTokenServiceImpl extends BaseService implements AccessTokenSe * @param token token string * @return update result code */ - public Map updateToken(int id, int userId, String expireTime, String token) { + public Map updateToken(User loginUser, int id, int userId, String expireTime, String token) { Map result = new HashMap<>(5); - + if (!hasPerm(loginUser,userId)){ + putMsg(result, Status.USER_NO_OPERATION_PERM); + return result; + } AccessToken accessToken = accessTokenMapper.selectById(id); if (accessToken == null) { logger.error("access token not exist, access token id {}", id); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataAnalysisServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataAnalysisServiceImpl.java index 21313b96d3..c9560e1c50 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataAnalysisServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataAnalysisServiceImpl.java @@ -16,7 +16,6 @@ */ package org.apache.dolphinscheduler.api.service.impl; - import org.apache.dolphinscheduler.api.dto.CommandStateCount; import org.apache.dolphinscheduler.api.dto.DefineUserDto; import org.apache.dolphinscheduler.api.dto.TaskCountDto; @@ -45,14 +44,14 @@ import org.apache.dolphinscheduler.service.process.ProcessService; import java.text.MessageFormat; import java.util.ArrayList; +import java.util.Arrays; import java.util.Date; -import java.util.EnumMap; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -62,8 +61,6 @@ import org.springframework.stereotype.Service; @Service public class DataAnalysisServiceImpl extends BaseService implements DataAnalysisService { - private static final Logger logger = LoggerFactory.getLogger(DataAnalysisServiceImpl.class); - @Autowired private ProjectMapper projectMapper; @@ -88,10 +85,6 @@ public class DataAnalysisServiceImpl extends BaseService implements DataAnalysis @Autowired private ProcessService processService; - private static final String COMMAND_STATE = "commandState"; - - private static final String ERROR_COMMAND_STATE = "errorCommandState"; - /** * statistical task instance status data * @@ -137,16 +130,17 @@ public class DataAnalysisServiceImpl extends BaseService implements DataAnalysis return result; } - Date start; - Date end; - try { + Date start = null; + Date end = null; + if (StringUtils.isNotEmpty(startDate) && StringUtils.isNotEmpty(endDate)) { start = DateUtils.getScheduleDate(startDate); end = DateUtils.getScheduleDate(endDate); - } catch (Exception e) { - logger.error(e.getMessage(), e); - putErrorRequestParamsMsg(result); - return result; + if (Objects.isNull(start) || Objects.isNull(end)) { + putErrorRequestParamsMsg(result); + return result; + } } + Integer[] projectIdArray = getProjectIdsArrays(loginUser, projectId); List processInstanceStateCounts = instanceStateCounter.apply(start, end, projectIdArray); @@ -204,79 +198,39 @@ public class DataAnalysisServiceImpl extends BaseService implements DataAnalysis * statistics based on task status execution, failure, completion, wait, total */ Date start = null; + if (StringUtils.isNotEmpty(startDate)) { + start = DateUtils.getScheduleDate(startDate); + if (Objects.isNull(start)) { + putErrorRequestParamsMsg(result); + return result; + } + } Date end = null; - - if (startDate != null && endDate != null) { - try { - start = DateUtils.getScheduleDate(startDate); - end = DateUtils.getScheduleDate(endDate); - } catch (Exception e) { - logger.error(e.getMessage(), e); + if (StringUtils.isNotEmpty(endDate)) { + end = DateUtils.getScheduleDate(endDate); + if (Objects.isNull(end)) { putErrorRequestParamsMsg(result); return result; } } - Integer[] projectIdArray = getProjectIdsArrays(loginUser, projectId); - // count command state - List commandStateCounts = - commandMapper.countCommandState( - loginUser.getId(), - start, - end, - projectIdArray); + // count normal command state + Map normalCountCommandCounts = commandMapper.countCommandState(loginUser.getId(), start, end, projectIdArray) + .stream() + .collect(Collectors.toMap(CommandCount::getCommandType, CommandCount::getCount)); // count error command state - List errorCommandStateCounts = - errorCommandMapper.countCommandState( - start, end, projectIdArray); - - // enumMap - Map> dataMap = new EnumMap<>(CommandType.class); - - Map commonCommand = new HashMap<>(); - commonCommand.put(COMMAND_STATE, 0); - commonCommand.put(ERROR_COMMAND_STATE, 0); - - - // init data map - /** - * START_PROCESS, START_CURRENT_TASK_PROCESS, RECOVER_TOLERANCE_FAULT_PROCESS, RECOVER_SUSPENDED_PROCESS, - START_FAILURE_TASK_PROCESS,COMPLEMENT_DATA,SCHEDULER, REPEAT_RUNNING,PAUSE,STOP,RECOVER_WAITTING_THREAD; - */ - dataMap.put(CommandType.START_PROCESS, commonCommand); - dataMap.put(CommandType.START_CURRENT_TASK_PROCESS, commonCommand); - dataMap.put(CommandType.RECOVER_TOLERANCE_FAULT_PROCESS, commonCommand); - dataMap.put(CommandType.RECOVER_SUSPENDED_PROCESS, commonCommand); - dataMap.put(CommandType.START_FAILURE_TASK_PROCESS, commonCommand); - dataMap.put(CommandType.COMPLEMENT_DATA, commonCommand); - dataMap.put(CommandType.SCHEDULER, commonCommand); - dataMap.put(CommandType.REPEAT_RUNNING, commonCommand); - dataMap.put(CommandType.PAUSE, commonCommand); - dataMap.put(CommandType.STOP, commonCommand); - dataMap.put(CommandType.RECOVER_WAITTING_THREAD, commonCommand); - - // put command state - for (CommandCount executeStatusCount : commandStateCounts) { - Map commandStateCountsMap = new HashMap<>(dataMap.get(executeStatusCount.getCommandType())); - commandStateCountsMap.put(COMMAND_STATE, executeStatusCount.getCount()); - dataMap.put(executeStatusCount.getCommandType(), commandStateCountsMap); - } + Map errorCommandCounts = errorCommandMapper.countCommandState(start, end, projectIdArray) + .stream() + .collect(Collectors.toMap(CommandCount::getCommandType, CommandCount::getCount)); - // put error command state - for (CommandCount errorExecutionStatus : errorCommandStateCounts) { - Map errorCommandStateCountsMap = new HashMap<>(dataMap.get(errorExecutionStatus.getCommandType())); - errorCommandStateCountsMap.put(ERROR_COMMAND_STATE, errorExecutionStatus.getCount()); - dataMap.put(errorExecutionStatus.getCommandType(), errorCommandStateCountsMap); - } - - List list = new ArrayList<>(); - for (Map.Entry> next : dataMap.entrySet()) { - CommandStateCount commandStateCount = new CommandStateCount(next.getValue().get(ERROR_COMMAND_STATE), - next.getValue().get(COMMAND_STATE), next.getKey()); - list.add(commandStateCount); - } + List list = Arrays.stream(CommandType.values()) + .map(commandType -> new CommandStateCount( + errorCommandCounts.getOrDefault(commandType, 0), + normalCountCommandCounts.getOrDefault(commandType, 0), + commandType) + ).collect(Collectors.toList()); result.put(Constants.DATA_LIST, list); putMsg(result, Status.SUCCESS); @@ -311,59 +265,10 @@ public class DataAnalysisServiceImpl extends BaseService implements DataAnalysis return result; } - // TODO tasksQueueList and tasksKillList is never updated. - List tasksQueueList = new ArrayList<>(); - List tasksKillList = new ArrayList<>(); - + //TODO need to add detail data info Map dataMap = new HashMap<>(); - if (loginUser.getUserType() == UserType.ADMIN_USER) { - dataMap.put("taskQueue", tasksQueueList.size()); - dataMap.put("taskKill", tasksKillList.size()); - - result.put(Constants.DATA_LIST, dataMap); - putMsg(result, Status.SUCCESS); - return result; - } - - int[] tasksQueueIds = new int[tasksQueueList.size()]; - int[] tasksKillIds = new int[tasksKillList.size()]; - - int i = 0; - for (String taskQueueStr : tasksQueueList) { - if (StringUtils.isNotEmpty(taskQueueStr)) { - String[] splits = taskQueueStr.split("_"); - if (splits.length >= 4) { - tasksQueueIds[i++] = Integer.parseInt(splits[3]); - } - } - } - - i = 0; - for (String taskKillStr : tasksKillList) { - if (StringUtils.isNotEmpty(taskKillStr)) { - String[] splits = taskKillStr.split("-"); - if (splits.length == 2) { - tasksKillIds[i++] = Integer.parseInt(splits[1]); - } - } - } - Integer taskQueueCount = 0; - Integer taskKillCount = 0; - - Integer[] projectIds = getProjectIdsArrays(loginUser, projectId); - if (tasksQueueIds.length != 0) { - taskQueueCount = taskInstanceMapper.countTask( - projectIds, - tasksQueueIds); - } - - if (tasksKillIds.length != 0) { - taskKillCount = taskInstanceMapper.countTask(projectIds, tasksKillIds); - } - - dataMap.put("taskQueue", taskQueueCount); - dataMap.put("taskKill", taskKillCount); - + dataMap.put("taskQueue", 0); + dataMap.put("taskKill", 0); result.put(Constants.DATA_LIST, dataMap); putMsg(result, Status.SUCCESS); return result; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java index c71f2980f5..1574e7f0e7 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java @@ -130,7 +130,6 @@ public class LoggerServiceImpl implements LoggerService { logClient.getLogBytes(host, Constants.RPC_PORT, taskInstance.getLogPath())); } - /** * get host * diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java index f4ea1bf065..e86ea8301e 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java @@ -48,6 +48,7 @@ import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.model.TaskNodeRelation; import org.apache.dolphinscheduler.common.process.ProcessDag; import org.apache.dolphinscheduler.common.process.Property; +import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.common.utils.CollectionUtils; @@ -77,6 +78,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.HashSet; @@ -84,6 +86,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; @@ -164,7 +167,7 @@ public class ProcessDefinitionServiceImpl extends BaseService implements String processDefinitionJson, String desc, String locations, - String connects) throws JsonProcessingException { + String connects) { Map result = new HashMap<>(); Project project = projectMapper.queryByName(projectName); @@ -221,29 +224,38 @@ public class ProcessDefinitionServiceImpl extends BaseService implements // return processDefinition object with ID result.put(Constants.DATA_LIST, processDefineMapper.selectById(processDefine.getId())); putMsg(result, Status.SUCCESS); - result.put("processDefinitionId", processDefine.getId()); + result.put(PROCESSDEFINITIONID, processDefine.getId()); return result; } /** * get resource ids - * * @param processData process data * @return resource ids */ private String getResourceIds(ProcessData processData) { List tasks = processData.getTasks(); Set resourceIds = new HashSet<>(); + StringBuilder sb = new StringBuilder(); + if (CollectionUtils.isEmpty(tasks)) { + return sb.toString(); + } for (TaskNode taskNode : tasks) { String taskParameter = taskNode.getParams(); AbstractParameters params = TaskParametersUtils.getParameters(taskNode.getType(), taskParameter); + if (params == null) { + continue; + } if (CollectionUtils.isNotEmpty(params.getResourceFilesList())) { - Set tempSet = params.getResourceFilesList().stream().map(t -> t.getId()).collect(Collectors.toSet()); + Set tempSet = params.getResourceFilesList(). + stream() + .filter(t -> t.getId() != 0) + .map(ResourceInfo::getId) + .collect(Collectors.toSet()); resourceIds.addAll(tempSet); } } - StringBuilder sb = new StringBuilder(); for (int i : resourceIds) { if (sb.length() > 0) { sb.append(","); @@ -252,7 +264,6 @@ public class ProcessDefinitionServiceImpl extends BaseService implements } return sb.toString(); } - /** * query process definition list * @@ -262,7 +273,7 @@ public class ProcessDefinitionServiceImpl extends BaseService implements */ public Map queryProcessDefinitionList(User loginUser, String projectName) { - HashMap result = new HashMap<>(); + HashMap result = new HashMap<>(5); Project project = projectMapper.queryByName(projectName); Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); @@ -355,10 +366,15 @@ public class ProcessDefinitionServiceImpl extends BaseService implements * @param connects connects for nodes * @return update result code */ - public Map updateProcessDefinition(User loginUser, String projectName, int id, String name, - String processDefinitionJson, String desc, - String locations, String connects) { - Map result = new HashMap<>(); + public Map updateProcessDefinition(User loginUser, + String projectName, + int id, + String name, + String processDefinitionJson, + String desc, + String locations, + String connects) { + Map result = new HashMap<>(5); Project project = projectMapper.queryByName(projectName); Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); @@ -373,16 +389,24 @@ public class ProcessDefinitionServiceImpl extends BaseService implements return checkProcessJson; } ProcessDefinition processDefine = processService.findProcessDefineById(id); + // check process definition exists if (processDefine == null) { - // check process definition exists putMsg(result, Status.PROCESS_DEFINE_NOT_EXIST, id); return result; - } else if (processDefine.getReleaseState() == ReleaseState.ONLINE) { + } + if (processDefine.getReleaseState() == ReleaseState.ONLINE) { // online can not permit edit putMsg(result, Status.PROCESS_DEFINE_NOT_ALLOWED_EDIT, processDefine.getName()); return result; - } else { - putMsg(result, Status.SUCCESS); + } + + if (!name.equals(processDefine.getName())) { + // check whether the new process define name exist + ProcessDefinition definition = processDefineMapper.verifyByDefineName(project.getId(), name); + if (definition != null) { + putMsg(result, Status.VERIFY_PROCESS_DEFINITION_NAME_UNIQUE_ERROR, name); + return result; + } } Date now = new Date(); @@ -441,11 +465,11 @@ public class ProcessDefinitionServiceImpl extends BaseService implements if (resultEnum != Status.SUCCESS) { return checkResult; } - ProcessDefinition processDefinition = processDefineMapper.queryByDefineName(project.getId(), name); + ProcessDefinition processDefinition = processDefineMapper.verifyByDefineName(project.getId(), name); if (processDefinition == null) { putMsg(result, Status.SUCCESS); } else { - putMsg(result, Status.PROCESS_INSTANCE_EXIST, name); + putMsg(result, Status.VERIFY_PROCESS_DEFINITION_NAME_UNIQUE_ERROR, name); } return result; } @@ -461,7 +485,7 @@ public class ProcessDefinitionServiceImpl extends BaseService implements @Transactional(rollbackFor = RuntimeException.class) public Map deleteProcessDefinitionById(User loginUser, String projectName, Integer processDefinitionId) { - Map result = new HashMap<>(); + Map result = new HashMap<>(5); Project project = projectMapper.queryByName(projectName); Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); @@ -673,6 +697,17 @@ public class ProcessDefinitionServiceImpl extends BaseService implements } } + /** + * get export process metadata string + * @param processDefinitionId process definition id + * @param processDefinition process definition + * @return export process metadata string + */ + public String exportProcessMetaDataStr(Integer processDefinitionId, ProcessDefinition processDefinition) { + //create workflow json file + return JSONUtils.toJsonString(exportProcessMetaData(processDefinitionId, processDefinition)); + } + /** * get export process metadata string * @@ -690,6 +725,7 @@ public class ProcessDefinitionServiceImpl extends BaseService implements exportProcessMeta.setProjectName(processDefinition.getProjectName()); exportProcessMeta.setProcessDefinitionName(processDefinition.getName()); exportProcessMeta.setProcessDefinitionJson(processDefinition.getProcessDefinitionJson()); + exportProcessMeta.setProcessDefinitionDescription(processDefinition.getDescription()); exportProcessMeta.setProcessDefinitionLocations(processDefinition.getLocations()); exportProcessMeta.setProcessDefinitionConnects(processDefinition.getConnects()); @@ -756,7 +792,7 @@ public class ProcessDefinitionServiceImpl extends BaseService implements */ @Transactional(rollbackFor = RuntimeException.class) public Map importProcessDefinition(User loginUser, MultipartFile file, String currentProjectName) { - Map result = new HashMap<>(); + Map result = new HashMap<>(5); String processMetaJson = FileUtils.file2String(file); List processMetaList = JSONUtils.toList(processMetaJson, ProcessMeta.class); @@ -851,7 +887,7 @@ public class ProcessDefinitionServiceImpl extends BaseService implements processMeta.getProcessDefinitionLocations(), processMeta.getProcessDefinitionConnects()); putMsg(result, Status.SUCCESS); - } catch (JsonProcessingException e) { + } catch (Exception e) { logger.error("import process meta json data: {}", e.getMessage(), e); putMsg(result, Status.IMPORT_PROCESS_DEFINE_ERROR); } @@ -925,7 +961,7 @@ public class ProcessDefinitionServiceImpl extends BaseService implements } //recursive sub-process parameter correction map key for old process id value for new process id - Map subProcessIdMap = new HashMap<>(); + Map subProcessIdMap = new HashMap<>(20); List subProcessList = StreamUtils.asStream(jsonArray.elements()) .filter(elem -> checkTaskHasSubProcess(JSONUtils.parseObject(elem.toString()).path("type").asText())) @@ -1213,7 +1249,7 @@ public class ProcessDefinitionServiceImpl extends BaseService implements */ public Map queryProcessDefinitionAllByProjectId(Integer projectId) { - HashMap result = new HashMap<>(); + HashMap result = new HashMap<>(5); List resourceList = processDefineMapper.queryAllDefinitionList(projectId); result.put(Constants.DATA_LIST, resourceList); @@ -1423,7 +1459,7 @@ public class ProcessDefinitionServiceImpl extends BaseService implements Integer processId, Project targetProject) throws JsonProcessingException { - Map result = new HashMap<>(); + Map result = new HashMap<>(5); ProcessDefinition processDefinition = processDefineMapper.selectById(processId); if (processDefinition == null) { @@ -1442,6 +1478,41 @@ public class ProcessDefinitionServiceImpl extends BaseService implements } } + /** + * copy process definition + * + * @param loginUser login user + * @param projectName project name + * @param processId process definition id + * @return copy result code + */ + public Map copyProcessDefinition(User loginUser, String projectName, Integer processId) { + + Map result = new HashMap<>(5); + Project project = projectMapper.queryByName(projectName); + + Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); + Status resultStatus = (Status) checkResult.get(Constants.STATUS); + if (resultStatus != Status.SUCCESS) { + return checkResult; + } + + ProcessDefinition processDefinition = processDefineMapper.selectById(processId); + if (processDefinition == null) { + putMsg(result, Status.PROCESS_DEFINE_NOT_EXIST, processId); + return result; + } else { + return createProcessDefinition( + loginUser, + projectName, + processDefinition.getName() + "_copy_" + System.currentTimeMillis(), + processDefinition.getProcessDefinitionJson(), + processDefinition.getDescription(), + processDefinition.getLocations(), + processDefinition.getConnects()); + } + } + /** * batch copy process definition * diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java index 395da6027f..fd47342c12 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java @@ -375,6 +375,28 @@ public class ProjectServiceImpl extends BaseService implements ProjectService { return result; } + /** + * query authorized and user create project list by user + * + * @param loginUser login user + * @return + */ + public Map queryProjectCreatedAndAuthorizedByUser(User loginUser) { + Map result = new HashMap<>(); + + List projects = null; + if (loginUser.getUserType() == UserType.ADMIN_USER) { + projects = projectMapper.selectList(null); + } else { + projects = projectMapper.queryProjectCreatedAndAuthorizedByUserId(loginUser.getId()); + } + + result.put(Constants.DATA_LIST, projects); + putMsg(result, Status.SUCCESS); + + return result; + } + /** * check whether have read permission * diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java index 3a267bcc8c..ecb9bae98c 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java @@ -21,6 +21,7 @@ import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.service.BaseService; import org.apache.dolphinscheduler.api.service.TenantService; import org.apache.dolphinscheduler.api.utils.PageInfo; +import org.apache.dolphinscheduler.api.utils.RegexUtils; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.CollectionUtils; @@ -73,18 +74,16 @@ public class TenantServiceImpl extends BaseService implements TenantService { /** * create tenant * - * @param loginUser login user + * @param loginUser login user * @param tenantCode tenant code - * @param tenantName tenant name - * @param queueId queue id - * @param desc description + * @param queueId queue id + * @param desc description * @return create result code * @throws Exception exception */ @Transactional(rollbackFor = Exception.class) public Map createTenant(User loginUser, String tenantCode, - String tenantName, int queueId, String desc) throws Exception { @@ -94,6 +93,11 @@ public class TenantServiceImpl extends BaseService implements TenantService { return result; } + if (RegexUtils.isNumeric(tenantCode)) { + putMsg(result, Status.CHECK_TENANT_CODE_ERROR); + return result; + } + if (checkTenantExists(tenantCode)) { putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, tenantCode); return result; @@ -107,7 +111,6 @@ public class TenantServiceImpl extends BaseService implements TenantService { return result; } tenant.setTenantCode(tenantCode); - tenant.setTenantName(tenantName); tenant.setQueueId(queueId); tenant.setDescription(desc); tenant.setCreateTime(now); @@ -131,8 +134,8 @@ public class TenantServiceImpl extends BaseService implements TenantService { * * @param loginUser login user * @param searchVal search value - * @param pageNo page number - * @param pageSize page size + * @param pageNo page number + * @param pageSize page size * @return tenant list page */ public Map queryTenantList(User loginUser, String searchVal, Integer pageNo, Integer pageSize) { @@ -157,16 +160,15 @@ public class TenantServiceImpl extends BaseService implements TenantService { /** * updateProcessInstance tenant * - * @param loginUser login user - * @param id tennat id + * @param loginUser login user + * @param id tennat id * @param tenantCode tennat code - * @param tenantName tennat name - * @param queueId queue id - * @param desc description + * @param queueId queue id + * @param desc description * @return update result code * @throws Exception exception */ - public Map updateTenant(User loginUser, int id, String tenantCode, String tenantName, int queueId, + public Map updateTenant(User loginUser, int id, String tenantCode, int queueId, String desc) throws Exception { Map result = new HashMap<>(5); @@ -209,10 +211,6 @@ public class TenantServiceImpl extends BaseService implements TenantService { tenant.setTenantCode(tenantCode); } - if (StringUtils.isNotEmpty(tenantName)) { - tenant.setTenantName(tenantName); - } - if (queueId != 0) { tenant.setQueueId(queueId); } @@ -229,7 +227,7 @@ public class TenantServiceImpl extends BaseService implements TenantService { * delete tenant * * @param loginUser login user - * @param id tenant id + * @param id tenant id * @return delete result code * @throws Exception exception */ @@ -285,6 +283,26 @@ public class TenantServiceImpl extends BaseService implements TenantService { return processInstanceMapper.queryByTenantIdAndStatus(tenant.getId(), Constants.NOT_TERMINATED_STATES); } + /** + * query tenant list + * + * @param tenantCode tenant code + * @return tenant list + */ + public Map queryTenantList(String tenantCode) { + + Map result = new HashMap<>(5); + + List resourceList = tenantMapper.queryByTenantCode(tenantCode); + if (CollectionUtils.isNotEmpty(resourceList)) { + result.put(Constants.DATA_LIST, resourceList); + putMsg(result, Status.SUCCESS); + } else { + putMsg(result, Status.TENANT_NOT_EXIST); + } + return result; + } + /** * query tenant list * @@ -311,7 +329,7 @@ public class TenantServiceImpl extends BaseService implements TenantService { public Result verifyTenantCode(String tenantCode) { Result result = new Result(); if (checkTenantExists(tenantCode)) { - putMsg(result, Status.TENANT_NAME_EXIST, tenantCode); + putMsg(result, Status.TENANT_CODE_EXIST, tenantCode); } else { putMsg(result, Status.SUCCESS); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/RegexUtils.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/RegexUtils.java new file mode 100644 index 0000000000..9ff7fac463 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/RegexUtils.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.utils; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * This is Regex expression utils. + */ +public class RegexUtils { + + /** + * check number regex expression + */ + private static final String CHECK_NUMBER = "^-?\\d+(\\.\\d+)?$"; + + private RegexUtils() { + } + + /** + * check if the input is number + * + * @param str input + * @return + */ + public static boolean isNumeric(String str) { + Pattern pattern = Pattern.compile(CHECK_NUMBER); + Matcher isNum = pattern.matcher(str); + return isNum.matches(); + } +} diff --git a/dolphinscheduler-api/src/main/resources/i18n/messages.properties b/dolphinscheduler-api/src/main/resources/i18n/messages.properties index d1da3e94a8..233d3b7edb 100644 --- a/dolphinscheduler-api/src/main/resources/i18n/messages.properties +++ b/dolphinscheduler-api/src/main/resources/i18n/messages.properties @@ -117,7 +117,6 @@ VERIFY_QUEUE_NOTES=verify queue TENANT_TAG=tenant related operation CREATE_TENANT_NOTES=create tenant TENANT_CODE=tenant code -TENANT_NAME=tenant name QUEUE_NAME=queue name PASSWORD=password DATA_SOURCE_OTHER=jdbc connection params, format:{"key1":"value1",...} @@ -254,6 +253,7 @@ QUERY_ALERT_GROUP_LIST_PAGING_NOTES=query alert group list paging EXPORT_PROCESS_DEFINITION_BY_ID_NOTES=export process definition by id BATCH_EXPORT_PROCESS_DEFINITION_BY_IDS_NOTES= batch export process definition by ids QUERY_USER_CREATED_PROJECT_NOTES= query user created project +QUERY_AUTHORIZED_AND_USER_CREATED_PROJECT_NOTES= query authorized and user created project COPY_PROCESS_DEFINITION_NOTES= copy process definition notes MOVE_PROCESS_DEFINITION_NOTES= move process definition notes TARGET_PROJECT_ID= target project id diff --git a/dolphinscheduler-api/src/main/resources/i18n/messages_en_US.properties b/dolphinscheduler-api/src/main/resources/i18n/messages_en_US.properties index 267f93b14d..a32ffb026d 100644 --- a/dolphinscheduler-api/src/main/resources/i18n/messages_en_US.properties +++ b/dolphinscheduler-api/src/main/resources/i18n/messages_en_US.properties @@ -117,7 +117,6 @@ VERIFY_QUEUE_NOTES=verify queue TENANT_TAG=tenant related operation CREATE_TENANT_NOTES=create tenant TENANT_CODE=tenant code -TENANT_NAME=tenant name QUEUE_NAME=queue name PASSWORD=password DATA_SOURCE_OTHER=jdbc connection params, format:{"key1":"value1",...} @@ -254,6 +253,7 @@ QUERY_ALERT_GROUP_LIST_PAGING_NOTES=query alert group list paging EXPORT_PROCESS_DEFINITION_BY_ID_NOTES=export process definition by id BATCH_EXPORT_PROCESS_DEFINITION_BY_IDS_NOTES= batch export process definition by ids QUERY_USER_CREATED_PROJECT_NOTES= query user created project +QUERY_AUTHORIZED_AND_USER_CREATED_PROJECT_NOTES= query authorized and user created project COPY_PROCESS_DEFINITION_NOTES= copy process definition notes MOVE_PROCESS_DEFINITION_NOTES= move process definition notes TARGET_PROJECT_ID= target project id diff --git a/dolphinscheduler-api/src/main/resources/i18n/messages_zh_CN.properties b/dolphinscheduler-api/src/main/resources/i18n/messages_zh_CN.properties index 16262e6bbc..24e51ad522 100644 --- a/dolphinscheduler-api/src/main/resources/i18n/messages_zh_CN.properties +++ b/dolphinscheduler-api/src/main/resources/i18n/messages_zh_CN.properties @@ -116,7 +116,6 @@ VERIFY_QUEUE_NOTES=验证队列 TENANT_TAG=租户相关操作 CREATE_TENANT_NOTES=创建租户 TENANT_CODE=租户编码 -TENANT_NAME=租户名称 QUEUE_NAME=队列名 PASSWORD=密码 DATA_SOURCE_OTHER=jdbc连接参数，格式为:{"key1":"value1",...} @@ -252,6 +251,7 @@ QUERY_ALERT_GROUP_LIST_PAGING_NOTES=分页查询告警组列表 EXPORT_PROCESS_DEFINITION_BY_ID_NOTES=通过工作流ID导出工作流定义 BATCH_EXPORT_PROCESS_DEFINITION_BY_IDS_NOTES=批量导出工作流定义 QUERY_USER_CREATED_PROJECT_NOTES= 查询用户创建的项目 +QUERY_AUTHORIZED_AND_USER_CREATED_PROJECT_NOTES= 查询授权和用户创建的项目 COPY_PROCESS_DEFINITION_NOTES= 复制工作流定义 MOVE_PROCESS_DEFINITION_NOTES= 移动工作流定义 TARGET_PROJECT_ID= 目标项目ID diff --git a/dolphinscheduler-api/src/main/resources/logback-api.xml b/dolphinscheduler-api/src/main/resources/logback-api.xml index 2df90d8392..e5cb37afac 100644 --- a/dolphinscheduler-api/src/main/resources/logback-api.xml +++ b/dolphinscheduler-api/src/main/resources/logback-api.xml @@ -55,7 +55,6 @@ - diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionControllerTest.java index f2a54a1a88..692d28b56e 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionControllerTest.java @@ -117,14 +117,14 @@ public class ProcessDefinitionControllerTest { public void testVerifyProcessDefinitionName() throws Exception { Map result = new HashMap<>(); - putMsg(result, Status.PROCESS_INSTANCE_EXIST); + putMsg(result, Status.VERIFY_PROCESS_DEFINITION_NAME_UNIQUE_ERROR); String projectName = "test"; String name = "dag_test"; Mockito.when(processDefinitionService.verifyProcessDefinitionName(user, projectName, name)).thenReturn(result); Result response = processDefinitionController.verifyProcessDefinitionName(user, projectName, name); - Assert.assertEquals(Status.PROCESS_INSTANCE_EXIST.getCode(), response.getCode().intValue()); + Assert.assertEquals(Status.VERIFY_PROCESS_DEFINITION_NAME_UNIQUE_ERROR.getCode(), response.getCode().intValue()); } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskInstanceControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskInstanceControllerTest.java index 368981c0d3..b97c7c192f 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskInstanceControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskInstanceControllerTest.java @@ -14,52 +14,60 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.api.controller; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.when; + import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.service.TaskInstanceService; +import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.utils.*; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.ExecutionStatus; +import org.apache.dolphinscheduler.dao.entity.TaskInstance; + +import java.util.HashMap; +import java.util.Map; + import org.junit.Assert; import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.http.MediaType; -import org.springframework.test.web.servlet.MvcResult; -import org.springframework.util.LinkedMultiValueMap; -import org.springframework.util.MultiValueMap; - -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; /** * task instance controller test */ -public class TaskInstanceControllerTest extends AbstractControllerTest{ - private static Logger logger = LoggerFactory.getLogger(TaskInstanceControllerTest.class); +@RunWith(MockitoJUnitRunner.Silent.class) +public class TaskInstanceControllerTest { + + @InjectMocks + private TaskInstanceController taskInstanceController; + + @Mock + private TaskInstanceService taskInstanceService; @Test - public void testQueryTaskListPaging() throws Exception { + public void testQueryTaskListPaging() { - MultiValueMap paramsMap = new LinkedMultiValueMap<>(); - //paramsMap.add("processInstanceId","1380"); - paramsMap.add("searchVal",""); - paramsMap.add("taskName",""); - //paramsMap.add("stateType",""); - paramsMap.add("startDate","2019-02-26 19:48:00"); - paramsMap.add("endDate","2019-02-26 19:48:22"); - paramsMap.add("pageNo","1"); - paramsMap.add("pageSize","20"); + Map result = new HashMap<>(); + Integer pageNo = 1; + Integer pageSize = 20; + PageInfo pageInfo = new PageInfo(pageNo, pageSize); + result.put(Constants.DATA_LIST, pageInfo); + result.put(Constants.STATUS, Status.SUCCESS); - MvcResult mvcResult = mockMvc.perform(get("/projects/{projectName}/task-instance/list-paging","cxc_1113") - .header(SESSION_ID, sessionId) - .params(paramsMap)) - .andExpect(status().isOk()) - .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8)) - .andReturn(); + when(taskInstanceService.queryTaskListPaging(any(), eq(""), eq(1), eq(""), eq(""), eq(""),any(), any(), + eq(""), Mockito.any(), eq("192.168.xx.xx"), any(), any())).thenReturn(result); + Result taskResult = taskInstanceController.queryTaskListPaging(null, "", 1, "", "", + "", "", ExecutionStatus.SUCCESS,"192.168.xx.xx", "2020-01-01 00:00:00", "2020-01-02 00:00:00",pageNo, pageSize); + Assert.assertEquals(Integer.valueOf(Status.SUCCESS.getCode()), taskResult.getCode()); - Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); - Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); - logger.info(mvcResult.getResponse().getContentAsString()); } + } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TenantControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TenantControllerTest.java index 7cf622ab9f..010b7ba36c 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TenantControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TenantControllerTest.java @@ -43,7 +43,6 @@ public class TenantControllerTest extends AbstractControllerTest{ public void testCreateTenant() throws Exception { MultiValueMap paramsMap = new LinkedMultiValueMap<>(); paramsMap.add("tenantCode","tenantCode"); - paramsMap.add("tenantName","tenantName"); paramsMap.add("queueId","1"); paramsMap.add("description","tenant description"); @@ -84,7 +83,6 @@ public class TenantControllerTest extends AbstractControllerTest{ MultiValueMap paramsMap = new LinkedMultiValueMap<>(); paramsMap.add("id","9"); paramsMap.add("tenantCode","cxc_te"); - paramsMap.add("tenantName","tenant_update_2"); paramsMap.add("queueId","1"); paramsMap.add("description","tenant description"); @@ -96,7 +94,7 @@ public class TenantControllerTest extends AbstractControllerTest{ .andReturn(); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); - Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); + Assert.assertEquals(Status.TENANT_NOT_EXIST.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); } @@ -133,7 +131,7 @@ public class TenantControllerTest extends AbstractControllerTest{ .andReturn(); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); - Assert.assertEquals(Status.TENANT_NAME_EXIST.getCode(), result.getCode().intValue()); + Assert.assertEquals(Status.TENANT_CODE_EXIST.getCode(), result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); } @@ -165,7 +163,7 @@ public class TenantControllerTest extends AbstractControllerTest{ .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8)) .andReturn(); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); - Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); + Assert.assertEquals(Status.TENANT_NOT_EXIST.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); } } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AccessTokenServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AccessTokenServiceTest.java index f5543487ea..e10d7185f9 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AccessTokenServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AccessTokenServiceTest.java @@ -81,7 +81,7 @@ public class AccessTokenServiceTest { public void testCreateToken() { when(accessTokenMapper.insert(any(AccessToken.class))).thenReturn(2); - Map result = accessTokenService.createToken(1, getDate(), "AccessTokenServiceTest"); + Map result = accessTokenService.createToken(getLoginUser(), 1, getDate(), "AccessTokenServiceTest"); logger.info(result.toString()); Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); } @@ -89,7 +89,7 @@ public class AccessTokenServiceTest { @Test public void testGenerateToken() { - Map result = accessTokenService.generateToken(Integer.MAX_VALUE, getDate()); + Map result = accessTokenService.generateToken(getLoginUser(), Integer.MAX_VALUE,getDate()); logger.info(result.toString()); Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); String token = (String) result.get(Constants.DATA_LIST); @@ -121,16 +121,24 @@ public class AccessTokenServiceTest { public void testUpdateToken() { when(accessTokenMapper.selectById(1)).thenReturn(getEntity()); - Map result = accessTokenService.updateToken(1, Integer.MAX_VALUE, getDate(), "token"); + Map result = accessTokenService.updateToken(getLoginUser(), 1,Integer.MAX_VALUE,getDate(),"token"); logger.info(result.toString()); Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); // not exist - result = accessTokenService.updateToken(2, Integer.MAX_VALUE, getDate(), "token"); + result = accessTokenService.updateToken(getLoginUser(), 2,Integer.MAX_VALUE,getDate(),"token"); logger.info(result.toString()); Assert.assertEquals(Status.ACCESS_TOKEN_NOT_EXIST, result.get(Constants.STATUS)); } + + private User getLoginUser(){ + User loginUser = new User(); + loginUser.setId(1); + loginUser.setUserType(UserType.ADMIN_USER); + return loginUser; + } + /** * create entity */ diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataAnalysisServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataAnalysisServiceTest.java index 3d8ae91287..e4f4a4e18c 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataAnalysisServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataAnalysisServiceTest.java @@ -16,12 +16,19 @@ */ package org.apache.dolphinscheduler.api.service; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; + +import org.apache.dolphinscheduler.api.dto.CommandStateCount; +import org.apache.dolphinscheduler.api.dto.TaskStateCount; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.service.impl.DataAnalysisServiceImpl; import org.apache.dolphinscheduler.api.service.impl.ProjectServiceImpl; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; +import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.dao.entity.CommandCount; import org.apache.dolphinscheduler.dao.entity.ExecuteStatusCount; @@ -36,6 +43,7 @@ import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; import org.apache.dolphinscheduler.service.process.ProcessService; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -123,6 +131,74 @@ public class DataAnalysisServiceTest { result = dataAnalysisService.countTaskStateByProject(user, 1, startDate, endDate); Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + // when date in illegal format then return error message + String startDate2 = "illegalDateString"; + String endDate2 = "illegalDateString"; + result = dataAnalysisService.countTaskStateByProject(user, 0, startDate2, endDate2); + Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result.get(Constants.STATUS)); + + // when one of date in illegal format then return error message + String startDate3 = "2020-08-28 14:13:40"; + String endDate3 = "illegalDateString"; + result = dataAnalysisService.countTaskStateByProject(user, 0, startDate3, endDate3); + Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result.get(Constants.STATUS)); + + // when one of date in illegal format then return error message + String startDate4 = "illegalDateString"; + String endDate4 = "2020-08-28 14:13:40"; + result = dataAnalysisService.countTaskStateByProject(user, 0, startDate4, endDate4); + Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result.get(Constants.STATUS)); + + // when counting general user's task status then return user's task status count + user.setUserType(UserType.GENERAL_USER); + Mockito.when(processService.getProjectIdListHavePerm(anyInt())) + .thenReturn(Collections.singletonList(123)); + ExecuteStatusCount executeStatusCount = new ExecuteStatusCount(); + executeStatusCount.setExecutionStatus(ExecutionStatus.RUNNING_EXECUTION); + executeStatusCount.setCount(10); + Mockito.when(taskInstanceMapper.countTaskInstanceStateByUser(any(), any(), any())) + .thenReturn(Collections.singletonList(executeStatusCount)); + result = dataAnalysisService.countTaskStateByProject(user, 0, startDate, null); + assertThat(result.get(Constants.DATA_LIST)).extracting("taskCountDtos").first().asList() + .hasSameSizeAs(ExecutionStatus.values()); + assertThat(result.get(Constants.DATA_LIST)).extracting("totalCount").first().isEqualTo(10); + TaskStateCount taskStateCount = new TaskStateCount(ExecutionStatus.RUNNING_EXECUTION, 10); + assertThat(result.get(Constants.DATA_LIST)).extracting("taskCountDtos").first().asList().containsOnlyOnce(taskStateCount); + + // when general user doesn't have any task then return all count are 0 + user.setUserType(UserType.GENERAL_USER); + Mockito.when(processService.getProjectIdListHavePerm(anyInt())) + .thenReturn(new ArrayList<>()); + Mockito.when(taskInstanceMapper.countTaskInstanceStateByUser(any(), any(), any())) + .thenReturn(Collections.emptyList()); + result = dataAnalysisService.countTaskStateByProject(user, 0, null, null); + assertThat(result.get(Constants.DATA_LIST)).extracting("totalCount").first().isEqualTo(0); + assertThat(result.get(Constants.DATA_LIST)).extracting("taskCountDtos").first().asList() + .hasSameSizeAs(ExecutionStatus.values()); + assertThat(result.get(Constants.DATA_LIST)).extracting("taskCountDtos").first().asList() + .extracting("count").allMatch(count -> count.equals(0)); + + // when general user doesn't have any task then return all count are 0 + user.setUserType(UserType.GENERAL_USER); + Mockito.when(processService.getProjectIdListHavePerm(anyInt())) + .thenReturn(new ArrayList<>()); + Mockito.when(taskInstanceMapper.countTaskInstanceStateByUser(any(), any(), any())) + .thenReturn(Collections.emptyList()); + result = dataAnalysisService.countTaskStateByProject(user, 0, null, null); + assertThat(result.get(Constants.DATA_LIST)).extracting("totalCount").first().isEqualTo(0); + assertThat(result.get(Constants.DATA_LIST)).extracting("taskCountDtos").first().asList() + .hasSameSizeAs(ExecutionStatus.values()); + assertThat(result.get(Constants.DATA_LIST)).extracting("taskCountDtos").first().asList() + .extracting("count").allMatch(count -> count.equals(0)); + + // when instanceStateCounter return null, then return nothing + user.setUserType(UserType.GENERAL_USER); + Mockito.when(processService.getProjectIdListHavePerm(anyInt())) + .thenReturn(new ArrayList<>()); + Mockito.when(taskInstanceMapper.countTaskInstanceStateByUser(any(), any(), any())) + .thenReturn(null); + result = dataAnalysisService.countTaskStateByProject(user, 0, null, null); + assertThat(result).isEmpty(); } @Test @@ -169,6 +245,67 @@ public class DataAnalysisServiceTest { result = dataAnalysisService.countCommandState(user, 1, startDate, endDate); Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + // when project check fail then return nothing + Map result1 = dataAnalysisService.countCommandState(user, 2, null, null); + Assert.assertTrue(result1.isEmpty()); + + // when all date in illegal format then return error message + String startDate2 = "illegalDateString"; + String endDate2 = "illegalDateString"; + Map result2 = dataAnalysisService.countCommandState(user, 0, startDate2, endDate2); + Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result2.get(Constants.STATUS)); + + // when one of date in illegal format then return error message + String startDate3 = "2020-08-22 09:23:10"; + String endDate3 = "illegalDateString"; + Map result3 = dataAnalysisService.countCommandState(user, 0, startDate3, endDate3); + Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result3.get(Constants.STATUS)); + + // when one of date in illegal format then return error message + String startDate4 = "illegalDateString"; + String endDate4 = "2020-08-22 09:23:10"; + Map result4 = dataAnalysisService.countCommandState(user, 0, startDate4, endDate4); + Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result4.get(Constants.STATUS)); + + // when no command found then return all count are 0 + Mockito.when(commandMapper.countCommandState(anyInt(), any(), any(), any())).thenReturn(Collections.emptyList()); + Mockito.when(errorCommandMapper.countCommandState(any(), any(), any())).thenReturn(Collections.emptyList()); + Map result5 = dataAnalysisService.countCommandState(user, 0, startDate, null); + assertThat(result5).containsEntry(Constants.STATUS, Status.SUCCESS); + assertThat(result5.get(Constants.DATA_LIST)).asList().extracting("errorCount").allMatch(count -> count.equals(0)); + assertThat(result5.get(Constants.DATA_LIST)).asList().extracting("normalCount").allMatch(count -> count.equals(0)); + + // when command found then return combination result + CommandCount normalCommandCount = new CommandCount(); + normalCommandCount.setCommandType(CommandType.START_PROCESS); + normalCommandCount.setCount(10); + CommandCount errorCommandCount = new CommandCount(); + errorCommandCount.setCommandType(CommandType.START_PROCESS); + errorCommandCount.setCount(5); + Mockito.when(commandMapper.countCommandState(anyInt(), any(), any(), any())).thenReturn(Collections.singletonList(normalCommandCount)); + Mockito.when(errorCommandMapper.countCommandState(any(), any(), any())).thenReturn(Collections.singletonList(errorCommandCount)); + + Map result6 = dataAnalysisService.countCommandState(user, 0, null, null); + + assertThat(result6).containsEntry(Constants.STATUS, Status.SUCCESS); + CommandStateCount commandStateCount = new CommandStateCount(); + commandStateCount.setCommandState(CommandType.START_PROCESS); + commandStateCount.setNormalCount(10); + commandStateCount.setErrorCount(5); + assertThat(result6.get(Constants.DATA_LIST)).asList().containsOnlyOnce(commandStateCount); + } + + @Test + public void testCountQueueState() { + // when project check fail then return nothing + Map result1 = dataAnalysisService.countQueueState(user, 2); + Assert.assertTrue(result1.isEmpty()); + + // when project check success when return all count are 0 + Map result2 = dataAnalysisService.countQueueState(user, 1); + assertThat(result2.get(Constants.DATA_LIST)).extracting("taskQueue", "taskKill") + .isNotEmpty() + .allMatch(count -> count.equals(0)); } /** diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java index 33032f54e4..55d7b09173 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java @@ -17,6 +17,8 @@ package org.apache.dolphinscheduler.api.service; +import static org.assertj.core.api.Assertions.assertThat; + import org.apache.dolphinscheduler.api.dto.ProcessMeta; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.service.impl.ProcessDefinitionServiceImpl; @@ -28,6 +30,9 @@ import org.apache.dolphinscheduler.common.enums.Priority; import org.apache.dolphinscheduler.common.enums.ReleaseState; import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.enums.WarningType; +import org.apache.dolphinscheduler.common.model.TaskNode; +import org.apache.dolphinscheduler.common.process.ResourceInfo; +import org.apache.dolphinscheduler.common.task.shell.ShellParameters; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; @@ -51,8 +56,11 @@ import org.apache.http.entity.ContentType; import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.lang.reflect.Method; import java.text.MessageFormat; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -66,6 +74,7 @@ import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; import org.springframework.mock.web.MockMultipartFile; +import org.springframework.util.ReflectionUtils; import org.springframework.web.multipart.MultipartFile; import com.baomidou.mybatisplus.core.metadata.IPage; @@ -609,16 +618,16 @@ public class ProcessDefinitionServiceTest { //project check auth success, process not exist putMsg(result, Status.SUCCESS, projectName); - Mockito.when(processDefineMapper.queryByDefineName(project.getId(), "test_pdf")).thenReturn(null); + Mockito.when(processDefineMapper.verifyByDefineName(project.getId(), "test_pdf")).thenReturn(null); Map processNotExistRes = processDefinitionService.verifyProcessDefinitionName(loginUser, "project_test1", "test_pdf"); Assert.assertEquals(Status.SUCCESS, processNotExistRes.get(Constants.STATUS)); //process exist - Mockito.when(processDefineMapper.queryByDefineName(project.getId(), "test_pdf")).thenReturn(getProcessDefinition()); + Mockito.when(processDefineMapper.verifyByDefineName(project.getId(), "test_pdf")).thenReturn(getProcessDefinition()); Map processExistRes = processDefinitionService.verifyProcessDefinitionName(loginUser, "project_test1", "test_pdf"); - Assert.assertEquals(Status.PROCESS_INSTANCE_EXIST, processExistRes.get(Constants.STATUS)); + Assert.assertEquals(Status.VERIFY_PROCESS_DEFINITION_NAME_UNIQUE_ERROR, processExistRes.get(Constants.STATUS)); } @Test @@ -984,6 +993,81 @@ public class ProcessDefinitionServiceTest { loginUser, projectName, "1", null); } + @Test + public void testGetResourceIds() throws Exception { + // set up + Method testMethod = ReflectionUtils.findMethod(ProcessDefinitionServiceImpl.class, "getResourceIds", ProcessData.class); + assertThat(testMethod).isNotNull(); + testMethod.setAccessible(true); + + // when processData has empty task, then return empty string + ProcessData input1 = new ProcessData(); + input1.setTasks(Collections.emptyList()); + String output1 = (String) testMethod.invoke(processDefinitionService, input1); + assertThat(output1).isEmpty(); + + // when task is null, then return empty string + ProcessData input2 = new ProcessData(); + input2.setTasks(null); + String output2 = (String) testMethod.invoke(processDefinitionService, input2); + assertThat(output2).isEmpty(); + + // when task type is incorrect mapping, then return empty string + ProcessData input3 = new ProcessData(); + TaskNode taskNode3 = new TaskNode(); + taskNode3.setType("notExistType"); + input3.setTasks(Collections.singletonList(taskNode3)); + String output3 = (String) testMethod.invoke(processDefinitionService, input3); + assertThat(output3).isEmpty(); + + // when task parameter list is null, then return empty string + ProcessData input4 = new ProcessData(); + TaskNode taskNode4 = new TaskNode(); + taskNode4.setType("SHELL"); + taskNode4.setParams(null); + input4.setTasks(Collections.singletonList(taskNode4)); + String output4 = (String) testMethod.invoke(processDefinitionService, input4); + assertThat(output4).isEmpty(); + + // when resource id list is 0 1, then return 0,1 + ProcessData input5 = new ProcessData(); + TaskNode taskNode5 = new TaskNode(); + taskNode5.setType("SHELL"); + ShellParameters shellParameters5 = new ShellParameters(); + ResourceInfo resourceInfo5A = new ResourceInfo(); + resourceInfo5A.setId(1); + ResourceInfo resourceInfo5B = new ResourceInfo(); + resourceInfo5B.setId(2); + shellParameters5.setResourceList(Arrays.asList(resourceInfo5A, resourceInfo5B)); + taskNode5.setParams(JSONUtils.toJsonString(shellParameters5)); + input5.setTasks(Collections.singletonList(taskNode5)); + String output5 = (String) testMethod.invoke(processDefinitionService, input5); + assertThat(output5.split(",")).hasSize(2) + .containsExactlyInAnyOrder("1", "2"); + + // when resource id list is 0 1 1 2, then return 0,1,2 + ProcessData input6 = new ProcessData(); + TaskNode taskNode6 = new TaskNode(); + taskNode6.setType("SHELL"); + ShellParameters shellParameters6 = new ShellParameters(); + ResourceInfo resourceInfo6A = new ResourceInfo(); + resourceInfo6A.setId(3); + ResourceInfo resourceInfo6B = new ResourceInfo(); + resourceInfo6B.setId(1); + ResourceInfo resourceInfo6C = new ResourceInfo(); + resourceInfo6C.setId(1); + ResourceInfo resourceInfo6D = new ResourceInfo(); + resourceInfo6D.setId(2); + shellParameters6.setResourceList(Arrays.asList(resourceInfo6A, resourceInfo6B, resourceInfo6C, resourceInfo6D)); + taskNode6.setParams(JSONUtils.toJsonString(shellParameters6)); + input6.setTasks(Collections.singletonList(taskNode6)); + + String output6 = (String) testMethod.invoke(processDefinitionService, input6); + + assertThat(output6.split(",")).hasSize(3) + .containsExactlyInAnyOrder("3", "1", "2"); + } + /** * get mock datasource * diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectServiceTest.java index 85b23b3fcb..cc39925aa5 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectServiceTest.java @@ -295,6 +295,28 @@ public class ProjectServiceTest { } + @Test + public void testQueryProjectCreatedAndAuthorizedByUser() { + + Map result = null; + User loginUser = getLoginUser(); + + // not admin user + Mockito.when(projectMapper.queryProjectCreatedAndAuthorizedByUserId(1)).thenReturn(getList()); + result = projectService.queryProjectCreatedAndAuthorizedByUser(loginUser); + List notAdminUserResult = (List) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(notAdminUserResult)); + + //admin user + loginUser.setUserType(UserType.ADMIN_USER); + Mockito.when(projectMapper.selectList(null)).thenReturn(getList()); + result = projectService.queryProjectCreatedAndAuthorizedByUser(loginUser); + List projects = (List) result.get(Constants.DATA_LIST); + + Assert.assertTrue(CollectionUtils.isNotEmpty(projects)); + + } + @Test public void testQueryAllProjectList() { @@ -340,13 +362,11 @@ public class ProjectServiceTest { * create admin user */ private User getLoginUser() { - User loginUser = new User(); loginUser.setUserType(UserType.GENERAL_USER); loginUser.setUserName(userName); loginUser.setId(1); return loginUser; - } /** diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java index 407f6b587f..d430d3a755 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java @@ -14,12 +14,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.api.service; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import org.apache.dolphinscheduler.api.enums.Status; -import org.apache.dolphinscheduler.api.exceptions.ServiceException; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; @@ -33,7 +31,18 @@ import org.apache.dolphinscheduler.dao.entity.Resource; import org.apache.dolphinscheduler.dao.entity.Tenant; import org.apache.dolphinscheduler.dao.entity.UdfFunc; import org.apache.dolphinscheduler.dao.entity.User; -import org.apache.dolphinscheduler.dao.mapper.*; +import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; +import org.apache.dolphinscheduler.dao.mapper.ResourceMapper; +import org.apache.dolphinscheduler.dao.mapper.ResourceUserMapper; +import org.apache.dolphinscheduler.dao.mapper.TenantMapper; +import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper; +import org.apache.dolphinscheduler.dao.mapper.UserMapper; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -49,14 +58,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.mock.web.MockMultipartFile; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; @RunWith(PowerMockRunner.class) @PowerMockIgnore({"sun.security.*", "javax.net.*"}) -@PrepareForTest({HadoopUtils.class,PropertyUtils.class, FileUtils.class,org.apache.dolphinscheduler.api.utils.FileUtils.class}) +@PrepareForTest({HadoopUtils.class, PropertyUtils.class, FileUtils.class, org.apache.dolphinscheduler.api.utils.FileUtils.class}) public class ResourcesServiceTest { private static final Logger logger = LoggerFactory.getLogger(ResourcesServiceTest.class); @@ -94,100 +101,103 @@ public class ResourcesServiceTest { } @Test - public void testCreateResource(){ + public void testCreateResource() { PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false); User user = new User(); //HDFS_NOT_STARTUP - Result result = resourcesService.createResource(user,"ResourcesServiceTest","ResourcesServiceTest",ResourceType.FILE,null,-1,"/"); + Result result = resourcesService.createResource(user, "ResourcesServiceTest", "ResourcesServiceTest", ResourceType.FILE, null, -1, "/"); logger.info(result.toString()); - Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(),result.getMsg()); + Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(), result.getMsg()); //RESOURCE_FILE_IS_EMPTY - MockMultipartFile mockMultipartFile = new MockMultipartFile("test.pdf",new String().getBytes()); + MockMultipartFile mockMultipartFile = new MockMultipartFile("test.pdf", "".getBytes()); PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); - result = resourcesService.createResource(user,"ResourcesServiceTest","ResourcesServiceTest",ResourceType.FILE,mockMultipartFile,-1,"/"); + result = resourcesService.createResource(user, "ResourcesServiceTest", "ResourcesServiceTest", ResourceType.FILE, mockMultipartFile, -1, "/"); logger.info(result.toString()); - Assert.assertEquals(Status.RESOURCE_FILE_IS_EMPTY.getMsg(),result.getMsg()); + Assert.assertEquals(Status.RESOURCE_FILE_IS_EMPTY.getMsg(), result.getMsg()); //RESOURCE_SUFFIX_FORBID_CHANGE - mockMultipartFile = new MockMultipartFile("test.pdf","test.pdf","pdf",new String("test").getBytes()); + mockMultipartFile = new MockMultipartFile("test.pdf", "test.pdf", "pdf", "test".getBytes()); PowerMockito.when(FileUtils.suffix("test.pdf")).thenReturn("pdf"); PowerMockito.when(FileUtils.suffix("ResourcesServiceTest.jar")).thenReturn("jar"); - result = resourcesService.createResource(user,"ResourcesServiceTest.jar","ResourcesServiceTest",ResourceType.FILE,mockMultipartFile,-1,"/"); + result = resourcesService.createResource(user, "ResourcesServiceTest.jar", "ResourcesServiceTest", ResourceType.FILE, mockMultipartFile, -1, "/"); logger.info(result.toString()); - Assert.assertEquals(Status.RESOURCE_SUFFIX_FORBID_CHANGE.getMsg(),result.getMsg()); + Assert.assertEquals(Status.RESOURCE_SUFFIX_FORBID_CHANGE.getMsg(), result.getMsg()); //UDF_RESOURCE_SUFFIX_NOT_JAR - mockMultipartFile = new MockMultipartFile("ResourcesServiceTest.pdf","ResourcesServiceTest.pdf","pdf",new String("test").getBytes()); + mockMultipartFile = new MockMultipartFile("ResourcesServiceTest.pdf", "ResourcesServiceTest.pdf", "pdf", "test".getBytes()); PowerMockito.when(FileUtils.suffix("ResourcesServiceTest.pdf")).thenReturn("pdf"); - result = resourcesService.createResource(user,"ResourcesServiceTest.pdf","ResourcesServiceTest",ResourceType.UDF,mockMultipartFile,-1,"/"); + result = resourcesService.createResource(user, "ResourcesServiceTest.pdf", "ResourcesServiceTest", ResourceType.UDF, mockMultipartFile, -1, "/"); logger.info(result.toString()); - Assert.assertEquals(Status.UDF_RESOURCE_SUFFIX_NOT_JAR.getMsg(),result.getMsg()); - + Assert.assertEquals(Status.UDF_RESOURCE_SUFFIX_NOT_JAR.getMsg(), result.getMsg()); } @Test - public void testCreateDirecotry(){ + public void testCreateDirecotry() { PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false); User user = new User(); //HDFS_NOT_STARTUP - Result result = resourcesService.createDirectory(user,"directoryTest","directory test",ResourceType.FILE,-1,"/"); + Result result = resourcesService.createDirectory(user, "directoryTest", "directory test", ResourceType.FILE, -1, "/"); logger.info(result.toString()); - Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(),result.getMsg()); + Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(), result.getMsg()); //PARENT_RESOURCE_NOT_EXIST + user.setId(1); + user.setTenantId(1); + Mockito.when(userMapper.selectById(1)).thenReturn(getUser()); + Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); Mockito.when(resourcesMapper.selectById(Mockito.anyInt())).thenReturn(null); - result = resourcesService.createDirectory(user,"directoryTest","directory test",ResourceType.FILE,1,"/"); + result = resourcesService.createDirectory(user, "directoryTest", "directory test", ResourceType.FILE, 1, "/"); logger.info(result.toString()); - Assert.assertEquals(Status.PARENT_RESOURCE_NOT_EXIST.getMsg(),result.getMsg()); + Assert.assertEquals(Status.PARENT_RESOURCE_NOT_EXIST.getMsg(), result.getMsg()); //RESOURCE_EXIST PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); Mockito.when(resourcesMapper.queryResourceList("/directoryTest", 0, 0)).thenReturn(getResourceList()); - result = resourcesService.createDirectory(user,"directoryTest","directory test",ResourceType.FILE,-1,"/"); + result = resourcesService.createDirectory(user, "directoryTest", "directory test", ResourceType.FILE, -1, "/"); logger.info(result.toString()); - Assert.assertEquals(Status.RESOURCE_EXIST.getMsg(),result.getMsg()); + Assert.assertEquals(Status.RESOURCE_EXIST.getMsg(), result.getMsg()); } @Test - public void testUpdateResource(){ + public void testUpdateResource() { PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false); User user = new User(); //HDFS_NOT_STARTUP - Result result = resourcesService.updateResource(user,1,"ResourcesServiceTest","ResourcesServiceTest",ResourceType.FILE); + Result result = resourcesService.updateResource(user, 1, "ResourcesServiceTest", "ResourcesServiceTest", ResourceType.FILE, null); logger.info(result.toString()); - Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(),result.getMsg()); + Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(), result.getMsg()); //RESOURCE_NOT_EXIST Mockito.when(resourcesMapper.selectById(1)).thenReturn(getResource()); PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); - result = resourcesService.updateResource(user,0,"ResourcesServiceTest","ResourcesServiceTest",ResourceType.FILE); + result = resourcesService.updateResource(user, 0, "ResourcesServiceTest", "ResourcesServiceTest", ResourceType.FILE, null); logger.info(result.toString()); - Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(),result.getMsg()); + Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg()); //USER_NO_OPERATION_PERM - result = resourcesService.updateResource(user,1,"ResourcesServiceTest","ResourcesServiceTest",ResourceType.FILE); + result = resourcesService.updateResource(user, 1, "ResourcesServiceTest", "ResourcesServiceTest", ResourceType.FILE, null); logger.info(result.toString()); - Assert.assertEquals(Status.USER_NO_OPERATION_PERM.getMsg(),result.getMsg()); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM.getMsg(), result.getMsg()); //RESOURCE_NOT_EXIST user.setId(1); Mockito.when(userMapper.selectById(1)).thenReturn(getUser()); Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); - PowerMockito.when(HadoopUtils.getHdfsFileName(Mockito.any(), Mockito.any(),Mockito.anyString())).thenReturn("test1"); + PowerMockito.when(HadoopUtils.getHdfsFileName(Mockito.any(), Mockito.any(), Mockito.anyString())).thenReturn("test1"); try { Mockito.when(HadoopUtils.getInstance().exists(Mockito.any())).thenReturn(false); } catch (IOException e) { - logger.error(e.getMessage(),e); + logger.error(e.getMessage(), e); } - result = resourcesService.updateResource(user, 1, "ResourcesServiceTest1.jar", "ResourcesServiceTest", ResourceType.UDF); - Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(),result.getMsg()); + result = resourcesService.updateResource(user, 1, "ResourcesServiceTest1.jar", "ResourcesServiceTest", ResourceType.UDF, null); + Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg()); //SUCCESS user.setId(1); @@ -196,57 +206,56 @@ public class ResourcesServiceTest { try { Mockito.when(HadoopUtils.getInstance().exists(Mockito.any())).thenReturn(true); } catch (IOException e) { - logger.error(e.getMessage(),e); + logger.error(e.getMessage(), e); } - result = resourcesService.updateResource(user,1,"ResourcesServiceTest.jar","ResourcesServiceTest",ResourceType.FILE); + result = resourcesService.updateResource(user, 1, "ResourcesServiceTest.jar", "ResourcesServiceTest", ResourceType.FILE, null); logger.info(result.toString()); - Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg()); + Assert.assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); //RESOURCE_EXIST Mockito.when(resourcesMapper.queryResourceList("/ResourcesServiceTest1.jar", 0, 0)).thenReturn(getResourceList()); - result = resourcesService.updateResource(user,1,"ResourcesServiceTest1.jar","ResourcesServiceTest",ResourceType.FILE); + result = resourcesService.updateResource(user, 1, "ResourcesServiceTest1.jar", "ResourcesServiceTest", ResourceType.FILE, null); logger.info(result.toString()); - Assert.assertEquals(Status.RESOURCE_EXIST.getMsg(),result.getMsg()); + Assert.assertEquals(Status.RESOURCE_EXIST.getMsg(), result.getMsg()); //USER_NOT_EXIST Mockito.when(userMapper.selectById(Mockito.anyInt())).thenReturn(null); - result = resourcesService.updateResource(user,1,"ResourcesServiceTest1.jar","ResourcesServiceTest",ResourceType.UDF); + result = resourcesService.updateResource(user, 1, "ResourcesServiceTest1.jar", "ResourcesServiceTest", ResourceType.UDF, null); logger.info(result.toString()); Assert.assertTrue(Status.USER_NOT_EXIST.getCode() == result.getCode()); //TENANT_NOT_EXIST Mockito.when(userMapper.selectById(1)).thenReturn(getUser()); Mockito.when(tenantMapper.queryById(Mockito.anyInt())).thenReturn(null); - result = resourcesService.updateResource(user,1,"ResourcesServiceTest1.jar","ResourcesServiceTest",ResourceType.UDF); + result = resourcesService.updateResource(user, 1, "ResourcesServiceTest1.jar", "ResourcesServiceTest", ResourceType.UDF, null); logger.info(result.toString()); - Assert.assertEquals(Status.TENANT_NOT_EXIST.getMsg(),result.getMsg()); - + Assert.assertEquals(Status.TENANT_NOT_EXIST.getMsg(), result.getMsg()); //SUCCESS Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); PowerMockito.when(HadoopUtils.getHdfsResourceFileName(Mockito.any(), Mockito.any())).thenReturn("test"); try { - PowerMockito.when(HadoopUtils.getInstance().copy(Mockito.anyString(),Mockito.anyString(),true,true)).thenReturn(true); + PowerMockito.when(HadoopUtils.getInstance().copy(Mockito.anyString(), Mockito.anyString(), true, true)).thenReturn(true); } catch (Exception e) { - logger.error(e.getMessage(),e); + logger.error(e.getMessage(), e); } - result = resourcesService.updateResource(user,1,"ResourcesServiceTest1.jar","ResourcesServiceTest1.jar",ResourceType.UDF); + result = resourcesService.updateResource(user, 1, "ResourcesServiceTest1.jar", "ResourcesServiceTest1.jar", ResourceType.UDF, null); logger.info(result.toString()); - Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg()); + Assert.assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); } @Test - public void testQueryResourceListPaging(){ + public void testQueryResourceListPaging() { User loginUser = new User(); loginUser.setUserType(UserType.ADMIN_USER); - IPage resourcePage = new Page<>(1,10); + IPage resourcePage = new Page<>(1, 10); resourcePage.setTotal(1); resourcePage.setRecords(getResourceList()); Mockito.when(resourcesMapper.queryResourcePaging(Mockito.any(Page.class), - Mockito.eq(0),Mockito.eq(-1), Mockito.eq(0), Mockito.eq("test"))).thenReturn(resourcePage); - Map result = resourcesService.queryResourceListPaging(loginUser,-1,ResourceType.FILE,"test",1,10); + Mockito.eq(0), Mockito.eq(-1), Mockito.eq(0), Mockito.eq("test"))).thenReturn(resourcePage); + Map result = resourcesService.queryResourceListPaging(loginUser, -1, ResourceType.FILE, "test", 1, 10); logger.info(result.toString()); Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); PageInfo pageInfo = (PageInfo) result.get(Constants.DATA_LIST); @@ -255,11 +264,11 @@ public class ResourcesServiceTest { } @Test - public void testQueryResourceList(){ + public void testQueryResourceList() { User loginUser = new User(); loginUser.setId(0); loginUser.setUserType(UserType.ADMIN_USER); - Mockito.when(resourcesMapper.queryResourceListAuthored(0, 0,0)).thenReturn(getResourceList()); + Mockito.when(resourcesMapper.queryResourceListAuthored(0, 0, 0)).thenReturn(getResourceList()); Map result = resourcesService.queryResourceList(loginUser, ResourceType.FILE); logger.info(result.toString()); Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); @@ -268,7 +277,7 @@ public class ResourcesServiceTest { } @Test - public void testDelete(){ + public void testDelete() { User loginUser = new User(); loginUser.setId(0); @@ -277,19 +286,19 @@ public class ResourcesServiceTest { try { // HDFS_NOT_STARTUP - Result result = resourcesService.delete(loginUser,1); + Result result = resourcesService.delete(loginUser, 1); logger.info(result.toString()); Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(), result.getMsg()); //RESOURCE_NOT_EXIST PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); Mockito.when(resourcesMapper.selectById(1)).thenReturn(getResource()); - result = resourcesService.delete(loginUser,2); + result = resourcesService.delete(loginUser, 2); logger.info(result.toString()); Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg()); // USER_NO_OPERATION_PERM - result = resourcesService.delete(loginUser,2); + result = resourcesService.delete(loginUser, 2); logger.info(result.toString()); Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg()); @@ -297,30 +306,30 @@ public class ResourcesServiceTest { loginUser.setUserType(UserType.ADMIN_USER); loginUser.setTenantId(2); Mockito.when(userMapper.selectById(Mockito.anyInt())).thenReturn(loginUser); - result = resourcesService.delete(loginUser,1); + result = resourcesService.delete(loginUser, 1); logger.info(result.toString()); Assert.assertEquals(Status.TENANT_NOT_EXIST.getMsg(), result.getMsg()); //SUCCESS loginUser.setTenantId(1); Mockito.when(hadoopUtils.delete(Mockito.anyString(), Mockito.anyBoolean())).thenReturn(true); - result = resourcesService.delete(loginUser,1); + result = resourcesService.delete(loginUser, 1); logger.info(result.toString()); Assert.assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); } catch (Exception e) { - logger.error("delete error",e); + logger.error("delete error", e); Assert.assertTrue(false); } } @Test - public void testVerifyResourceName(){ + public void testVerifyResourceName() { User user = new User(); user.setId(1); Mockito.when(resourcesMapper.queryResourceList("/ResourcesServiceTest.jar", 0, 0)).thenReturn(getResourceList()); - Result result = resourcesService.verifyResourceName("/ResourcesServiceTest.jar",ResourceType.FILE,user); + Result result = resourcesService.verifyResourceName("/ResourcesServiceTest.jar", ResourceType.FILE, user); logger.info(result.toString()); Assert.assertEquals(Status.RESOURCE_EXIST.getMsg(), result.getMsg()); @@ -330,94 +339,89 @@ public class ResourcesServiceTest { try { Mockito.when(hadoopUtils.exists(unExistFullName)).thenReturn(false); } catch (IOException e) { - logger.error("hadoop error",e); + logger.error("hadoop error", e); } - result = resourcesService.verifyResourceName("/test.jar",ResourceType.FILE,user); + result = resourcesService.verifyResourceName("/test.jar", ResourceType.FILE, user); logger.info(result.toString()); Assert.assertEquals(Status.TENANT_NOT_EXIST.getMsg(), result.getMsg()); - //RESOURCE_FILE_EXIST user.setTenantId(1); try { Mockito.when(hadoopUtils.exists("test")).thenReturn(true); } catch (IOException e) { - logger.error("hadoop error",e); + logger.error("hadoop error", e); } PowerMockito.when(HadoopUtils.getHdfsResourceFileName("123", "test1")).thenReturn("test"); - result = resourcesService.verifyResourceName("/ResourcesServiceTest.jar",ResourceType.FILE,user); + result = resourcesService.verifyResourceName("/ResourcesServiceTest.jar", ResourceType.FILE, user); logger.info(result.toString()); - Assert.assertTrue(Status.RESOURCE_EXIST.getCode()==result.getCode()); + Assert.assertTrue(Status.RESOURCE_EXIST.getCode() == result.getCode()); //SUCCESS - result = resourcesService.verifyResourceName("test2",ResourceType.FILE,user); + result = resourcesService.verifyResourceName("test2", ResourceType.FILE, user); logger.info(result.toString()); Assert.assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); } @Test - public void testReadResource(){ + public void testReadResource() { PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false); //HDFS_NOT_STARTUP - Result result = resourcesService.readResource(1,1,10); + Result result = resourcesService.readResource(1, 1, 10); logger.info(result.toString()); - Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(),result.getMsg()); + Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(), result.getMsg()); //RESOURCE_NOT_EXIST Mockito.when(resourcesMapper.selectById(1)).thenReturn(getResource()); PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); - result = resourcesService.readResource(2,1,10); + result = resourcesService.readResource(2, 1, 10); logger.info(result.toString()); - Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(),result.getMsg()); - + Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg()); //RESOURCE_SUFFIX_NOT_SUPPORT_VIEW PowerMockito.when(FileUtils.getResourceViewSuffixs()).thenReturn("class"); PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); - result = resourcesService.readResource(1,1,10); + result = resourcesService.readResource(1, 1, 10); logger.info(result.toString()); - Assert.assertEquals(Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW.getMsg(),result.getMsg()); + Assert.assertEquals(Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW.getMsg(), result.getMsg()); //USER_NOT_EXIST PowerMockito.when(FileUtils.getResourceViewSuffixs()).thenReturn("jar"); PowerMockito.when(FileUtils.suffix("ResourcesServiceTest.jar")).thenReturn("jar"); - result = resourcesService.readResource(1,1,10); + result = resourcesService.readResource(1, 1, 10); logger.info(result.toString()); - Assert.assertTrue(Status.USER_NOT_EXIST.getCode()==result.getCode()); - + Assert.assertTrue(Status.USER_NOT_EXIST.getCode() == result.getCode()); //TENANT_NOT_EXIST Mockito.when(userMapper.selectById(1)).thenReturn(getUser()); - result = resourcesService.readResource(1,1,10); + result = resourcesService.readResource(1, 1, 10); logger.info(result.toString()); - Assert.assertEquals(Status.TENANT_NOT_EXIST.getMsg(),result.getMsg()); - + Assert.assertEquals(Status.TENANT_NOT_EXIST.getMsg(), result.getMsg()); //RESOURCE_FILE_NOT_EXIST Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); try { Mockito.when(hadoopUtils.exists(Mockito.anyString())).thenReturn(false); } catch (IOException e) { - logger.error("hadoop error",e); + logger.error("hadoop error", e); } - result = resourcesService.readResource(1,1,10); + result = resourcesService.readResource(1, 1, 10); logger.info(result.toString()); - Assert.assertTrue(Status.RESOURCE_FILE_NOT_EXIST.getCode()==result.getCode()); + Assert.assertTrue(Status.RESOURCE_FILE_NOT_EXIST.getCode() == result.getCode()); //SUCCESS try { Mockito.when(hadoopUtils.exists(null)).thenReturn(true); - Mockito.when(hadoopUtils.catFile(null,1,10)).thenReturn(getContent()); + Mockito.when(hadoopUtils.catFile(null, 1, 10)).thenReturn(getContent()); } catch (IOException e) { - logger.error("hadoop error",e); + logger.error("hadoop error", e); } - result = resourcesService.readResource(1,1,10); + result = resourcesService.readResource(1, 1, 10); logger.info(result.toString()); - Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg()); - + Assert.assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); } @@ -429,23 +433,23 @@ public class ResourcesServiceTest { PowerMockito.when(HadoopUtils.getHdfsUdfDir("udfDir")).thenReturn("udfDir"); User user = getUser(); //HDFS_NOT_STARTUP - Result result = resourcesService.onlineCreateResource(user,ResourceType.FILE,"test","jar","desc","content",-1,"/"); + Result result = resourcesService.onlineCreateResource(user, ResourceType.FILE, "test", "jar", "desc", "content", -1, "/"); logger.info(result.toString()); - Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(),result.getMsg()); + Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(), result.getMsg()); //RESOURCE_SUFFIX_NOT_SUPPORT_VIEW PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); PowerMockito.when(FileUtils.getResourceViewSuffixs()).thenReturn("class"); - result = resourcesService.onlineCreateResource(user,ResourceType.FILE,"test","jar","desc","content",-1,"/"); + result = resourcesService.onlineCreateResource(user, ResourceType.FILE, "test", "jar", "desc", "content", -1, "/"); logger.info(result.toString()); - Assert.assertEquals(Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW.getMsg(),result.getMsg()); + Assert.assertEquals(Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW.getMsg(), result.getMsg()); //RuntimeException try { PowerMockito.when(FileUtils.getResourceViewSuffixs()).thenReturn("jar"); Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); - result = resourcesService.onlineCreateResource(user, ResourceType.FILE, "test", "jar", "desc", "content",-1,"/"); - }catch (RuntimeException ex){ + result = resourcesService.onlineCreateResource(user, ResourceType.FILE, "test", "jar", "desc", "content", -1, "/"); + } catch (RuntimeException ex) { logger.info(result.toString()); Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), ex.getMessage()); } @@ -453,50 +457,48 @@ public class ResourcesServiceTest { //SUCCESS Mockito.when(FileUtils.getUploadFilename(Mockito.anyString(), Mockito.anyString())).thenReturn("test"); PowerMockito.when(FileUtils.writeContent2File(Mockito.anyString(), Mockito.anyString())).thenReturn(true); - result = resourcesService.onlineCreateResource(user,ResourceType.FILE,"test","jar","desc","content",-1,"/"); + result = resourcesService.onlineCreateResource(user, ResourceType.FILE, "test", "jar", "desc", "content", -1, "/"); logger.info(result.toString()); - Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg()); - + Assert.assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); } @Test - public void testUpdateResourceContent(){ + public void testUpdateResourceContent() { User loginUser = new User(); loginUser.setId(0); PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false); // HDFS_NOT_STARTUP - Result result = resourcesService.updateResourceContent(1,"content"); + Result result = resourcesService.updateResourceContent(1, "content"); logger.info(result.toString()); Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(), result.getMsg()); //RESOURCE_NOT_EXIST PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); Mockito.when(resourcesMapper.selectById(1)).thenReturn(getResource()); - result = resourcesService.updateResourceContent(2,"content"); + result = resourcesService.updateResourceContent(2, "content"); logger.info(result.toString()); Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg()); //RESOURCE_SUFFIX_NOT_SUPPORT_VIEW PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); PowerMockito.when(FileUtils.getResourceViewSuffixs()).thenReturn("class"); - result = resourcesService.updateResourceContent(1,"content"); + result = resourcesService.updateResourceContent(1, "content"); logger.info(result.toString()); - Assert.assertEquals(Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW.getMsg(),result.getMsg()); + Assert.assertEquals(Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW.getMsg(), result.getMsg()); //USER_NOT_EXIST PowerMockito.when(FileUtils.getResourceViewSuffixs()).thenReturn("jar"); PowerMockito.when(FileUtils.suffix("ResourcesServiceTest.jar")).thenReturn("jar"); - result = resourcesService.updateResourceContent(1,"content"); + result = resourcesService.updateResourceContent(1, "content"); logger.info(result.toString()); Assert.assertTrue(Status.USER_NOT_EXIST.getCode() == result.getCode()); - //TENANT_NOT_EXIST Mockito.when(userMapper.selectById(1)).thenReturn(getUser()); - result = resourcesService.updateResourceContent(1,"content"); + result = resourcesService.updateResourceContent(1, "content"); logger.info(result.toString()); Assert.assertTrue(Status.TENANT_NOT_EXIST.getCode() == result.getCode()); @@ -504,13 +506,13 @@ public class ResourcesServiceTest { Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); Mockito.when(FileUtils.getUploadFilename(Mockito.anyString(), Mockito.anyString())).thenReturn("test"); PowerMockito.when(FileUtils.writeContent2File(Mockito.anyString(), Mockito.anyString())).thenReturn(true); - result = resourcesService.updateResourceContent(1,"content"); + result = resourcesService.updateResourceContent(1, "content"); logger.info(result.toString()); Assert.assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); } @Test - public void testDownloadResource(){ + public void testDownloadResource() { PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); @@ -526,102 +528,117 @@ public class ResourcesServiceTest { resource = resourcesService.downloadResource(1); Assert.assertNotNull(resource); } catch (Exception e) { - logger.error("DownloadResource error",e); + logger.error("DownloadResource error", e); Assert.assertTrue(false); } } @Test - public void testUnauthorizedFile(){ + public void testUnauthorizedFile() { User user = getUser(); //USER_NO_OPERATION_PERM - Map result = resourcesService.unauthorizedFile(user,1); + Map result = resourcesService.unauthorizedFile(user, 1); logger.info(result.toString()); - Assert.assertEquals(Status.USER_NO_OPERATION_PERM,result.get(Constants.STATUS)); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS)); //SUCCESS user.setUserType(UserType.ADMIN_USER); Mockito.when(resourcesMapper.queryResourceExceptUserId(1)).thenReturn(getResourceList()); - result = resourcesService.unauthorizedFile(user,1); + result = resourcesService.unauthorizedFile(user, 1); logger.info(result.toString()); - Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); List resources = (List) result.get(Constants.DATA_LIST); Assert.assertTrue(CollectionUtils.isNotEmpty(resources)); } @Test - public void testUnauthorizedUDFFunction(){ + public void testUnauthorizedUDFFunction() { User user = getUser(); //USER_NO_OPERATION_PERM - Map result = resourcesService.unauthorizedUDFFunction(user,1); + Map result = resourcesService.unauthorizedUDFFunction(user, 1); logger.info(result.toString()); - Assert.assertEquals(Status.USER_NO_OPERATION_PERM,result.get(Constants.STATUS)); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS)); //SUCCESS user.setUserType(UserType.ADMIN_USER); Mockito.when(udfFunctionMapper.queryUdfFuncExceptUserId(1)).thenReturn(getUdfFuncList()); - result = resourcesService.unauthorizedUDFFunction(user,1); + result = resourcesService.unauthorizedUDFFunction(user, 1); logger.info(result.toString()); - Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); List udfFuncs = (List) result.get(Constants.DATA_LIST); Assert.assertTrue(CollectionUtils.isNotEmpty(udfFuncs)); } - @Test - public void testAuthorizedUDFFunction(){ + public void testAuthorizedUDFFunction() { User user = getUser(); //USER_NO_OPERATION_PERM - Map result = resourcesService.authorizedUDFFunction(user,1); + Map result = resourcesService.authorizedUDFFunction(user, 1); logger.info(result.toString()); - Assert.assertEquals(Status.USER_NO_OPERATION_PERM,result.get(Constants.STATUS)); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS)); //SUCCESS user.setUserType(UserType.ADMIN_USER); Mockito.when(udfFunctionMapper.queryAuthedUdfFunc(1)).thenReturn(getUdfFuncList()); - result = resourcesService.authorizedUDFFunction(user,1); + result = resourcesService.authorizedUDFFunction(user, 1); logger.info(result.toString()); - Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); List udfFuncs = (List) result.get(Constants.DATA_LIST); Assert.assertTrue(CollectionUtils.isNotEmpty(udfFuncs)); } @Test - public void testAuthorizedFile(){ + public void testAuthorizedFile() { User user = getUser(); //USER_NO_OPERATION_PERM - Map result = resourcesService.authorizedFile(user,1); + Map result = resourcesService.authorizedFile(user, 1); logger.info(result.toString()); - Assert.assertEquals(Status.USER_NO_OPERATION_PERM,result.get(Constants.STATUS)); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS)); //SUCCESS user.setUserType(UserType.ADMIN_USER); Mockito.when(resourcesMapper.queryAuthorizedResourceList(1)).thenReturn(getResourceList()); - result = resourcesService.authorizedFile(user,1); + result = resourcesService.authorizedFile(user, 1); logger.info(result.toString()); - Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); List resources = (List) result.get(Constants.DATA_LIST); Assert.assertTrue(CollectionUtils.isNotEmpty(resources)); } + @Test + public void testCatFile() { + + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false); + + //SUCCESS + try { + Mockito.when(hadoopUtils.exists(null)).thenReturn(true); + Mockito.when(hadoopUtils.catFile(null, 1, 10)).thenReturn(getContent()); + + List list = hadoopUtils.catFile(null, 1, 10); + Assert.assertNotNull(list); + + } catch (IOException e) { + logger.error("hadoop error", e); + } + } - private List getResourceList(){ + private List getResourceList() { - List resources = new ArrayList<>(); + List resources = new ArrayList<>(); resources.add(getResource()); return resources; } - private Tenant getTenant() { Tenant tenant = new Tenant(); tenant.setTenantCode("123"); return tenant; } - private Resource getResource(){ + private Resource getResource() { Resource resource = new Resource(); resource.setPid(-1); @@ -633,7 +650,7 @@ public class ResourcesServiceTest { return resource; } - private Resource getUdfResource(){ + private Resource getUdfResource() { Resource resource = new Resource(); resource.setUserId(1); @@ -644,27 +661,28 @@ public class ResourcesServiceTest { return resource; } - private UdfFunc getUdfFunc(){ + private UdfFunc getUdfFunc() { UdfFunc udfFunc = new UdfFunc(); udfFunc.setId(1); return udfFunc; } - private List getUdfFuncList(){ + private List getUdfFuncList() { List udfFuncs = new ArrayList<>(); udfFuncs.add(getUdfFunc()); return udfFuncs; } - private User getUser(){ + private User getUser() { User user = new User(); user.setId(1); user.setTenantId(1); return user; } - private List getContent(){ + + private List getContent() { List contentList = new ArrayList<>(); contentList.add("test"); return contentList; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskInstanceServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskInstanceServiceTest.java index 16547b3fd7..199b34cc1b 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskInstanceServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskInstanceServiceTest.java @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.api.service; import static org.mockito.ArgumentMatchers.any; @@ -88,7 +89,7 @@ public class TaskInstanceServiceTest { //project auth fail when(projectMapper.queryByName(projectName)).thenReturn(null); when(projectService.checkProjectAndAuth(loginUser, null, projectName)).thenReturn(result); - Map proejctAuthFailRes = taskInstanceService.queryTaskListPaging(loginUser, "project_test1", 0, "", + Map proejctAuthFailRes = taskInstanceService.queryTaskListPaging(loginUser, "project_test1", 0, "", "", "test_user", "2019-02-26 19:48:00", "2019-02-26 19:48:22", "", null, "", 1, 20); Assert.assertEquals(Status.PROJECT_NOT_FOUNT, proejctAuthFailRes.get(Constants.STATUS)); @@ -107,43 +108,43 @@ public class TaskInstanceServiceTest { when(projectService.checkProjectAndAuth(loginUser, project, projectName)).thenReturn(result); when(usersService.queryUser(loginUser.getId())).thenReturn(loginUser); when(usersService.getUserIdByName(loginUser.getUserName())).thenReturn(loginUser.getId()); - when(taskInstanceMapper.queryTaskInstanceListPaging(Mockito.any(Page.class), eq(project.getId()), eq(1), eq(""), eq(""), + when(taskInstanceMapper.queryTaskInstanceListPaging(Mockito.any(Page.class), eq(project.getId()), eq(1), eq(""), eq(""), eq(""), eq(0), Mockito.any(), eq("192.168.xx.xx"), eq(start), eq(end))).thenReturn(pageReturn); when(usersService.queryUser(processInstance.getExecutorId())).thenReturn(loginUser); when(processService.findProcessInstanceDetailById(taskInstance.getProcessInstanceId())).thenReturn(processInstance); - Map successRes = taskInstanceService.queryTaskListPaging(loginUser, projectName, 1, "", + Map successRes = taskInstanceService.queryTaskListPaging(loginUser, projectName, 1, "", "", "test_user", "2020-01-01 00:00:00", "2020-01-02 00:00:00", "", ExecutionStatus.SUCCESS, "192.168.xx.xx", 1, 20); Assert.assertEquals(Status.SUCCESS, successRes.get(Constants.STATUS)); //executor name empty - when(taskInstanceMapper.queryTaskInstanceListPaging(Mockito.any(Page.class), eq(project.getId()), eq(1), eq(""), eq(""), + when(taskInstanceMapper.queryTaskInstanceListPaging(Mockito.any(Page.class), eq(project.getId()), eq(1), eq(""), eq(""), eq(""), eq(0), Mockito.any(), eq("192.168.xx.xx"), eq(start), eq(end))).thenReturn(pageReturn); - Map executorEmptyRes = taskInstanceService.queryTaskListPaging(loginUser, projectName, 1, "", + Map executorEmptyRes = taskInstanceService.queryTaskListPaging(loginUser, projectName, 1, "", "", "", "2020-01-01 00:00:00", "2020-01-02 00:00:00", "", ExecutionStatus.SUCCESS, "192.168.xx.xx", 1, 20); Assert.assertEquals(Status.SUCCESS, executorEmptyRes.get(Constants.STATUS)); //executor null when(usersService.queryUser(loginUser.getId())).thenReturn(null); when(usersService.getUserIdByName(loginUser.getUserName())).thenReturn(-1); - Map executorNullRes = taskInstanceService.queryTaskListPaging(loginUser, projectName, 1, "", + Map executorNullRes = taskInstanceService.queryTaskListPaging(loginUser, projectName, 1, "", "", "test_user", "2020-01-01 00:00:00", "2020-01-02 00:00:00", "", ExecutionStatus.SUCCESS, "192.168.xx.xx", 1, 20); Assert.assertEquals(Status.SUCCESS, executorNullRes.get(Constants.STATUS)); //start/end date null - when(taskInstanceMapper.queryTaskInstanceListPaging(Mockito.any(Page.class), eq(project.getId()), eq(1), eq(""), eq(""), + when(taskInstanceMapper.queryTaskInstanceListPaging(Mockito.any(Page.class), eq(project.getId()), eq(1), eq(""), eq(""), eq(""), eq(0), Mockito.any(), eq("192.168.xx.xx"), any(), any())).thenReturn(pageReturn); - Map executorNullDateRes = taskInstanceService.queryTaskListPaging(loginUser, projectName, 1, "", + Map executorNullDateRes = taskInstanceService.queryTaskListPaging(loginUser, projectName, 1, "", "", "", null, null, "", ExecutionStatus.SUCCESS, "192.168.xx.xx", 1, 20); Assert.assertEquals(Status.SUCCESS, executorNullDateRes.get(Constants.STATUS)); //start date error format - when(taskInstanceMapper.queryTaskInstanceListPaging(Mockito.any(Page.class), eq(project.getId()), eq(1), eq(""), eq(""), + when(taskInstanceMapper.queryTaskInstanceListPaging(Mockito.any(Page.class), eq(project.getId()), eq(1), eq(""), eq(""), eq(""), eq(0), Mockito.any(), eq("192.168.xx.xx"), any(), any())).thenReturn(pageReturn); - Map executorErrorStartDateRes = taskInstanceService.queryTaskListPaging(loginUser, projectName, 1, "", + Map executorErrorStartDateRes = taskInstanceService.queryTaskListPaging(loginUser, projectName, 1, "", "", "", "error date", null, "", ExecutionStatus.SUCCESS, "192.168.xx.xx", 1, 20); Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, executorErrorStartDateRes.get(Constants.STATUS)); - Map executorErrorEndDateRes = taskInstanceService.queryTaskListPaging(loginUser, projectName, 1, "", + Map executorErrorEndDateRes = taskInstanceService.queryTaskListPaging(loginUser, projectName, 1, "", "", "", null, "error date", "", ExecutionStatus.SUCCESS, "192.168.xx.xx", 1, 20); Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, executorErrorEndDateRes.get(Constants.STATUS)); } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java index 5dcf59cf74..86b2079f26 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java @@ -73,8 +73,6 @@ public class TenantServiceTest { private static final String tenantCode = "TenantServiceTest"; - private static final String tenantName = "TenantServiceTest"; - @Test public void testCreateTenant() { @@ -83,17 +81,17 @@ public class TenantServiceTest { try { //check tenantCode Map result = - tenantService.createTenant(getLoginUser(), "%!1111", tenantName, 1, "TenantServiceTest"); + tenantService.createTenant(getLoginUser(), "%!1111", 1, "TenantServiceTest"); logger.info(result.toString()); Assert.assertEquals(Status.VERIFY_TENANT_CODE_ERROR, result.get(Constants.STATUS)); //check exist - result = tenantService.createTenant(loginUser, tenantCode, tenantName, 1, "TenantServiceTest"); + result = tenantService.createTenant(loginUser, tenantCode, 1, "TenantServiceTest"); logger.info(result.toString()); Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result.get(Constants.STATUS)); // success - result = tenantService.createTenant(loginUser, "test", "test", 1, "TenantServiceTest"); + result = tenantService.createTenant(loginUser, "test", 1, "TenantServiceTest"); logger.info(result.toString()); Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); @@ -126,11 +124,11 @@ public class TenantServiceTest { try { // id not exist Map result = - tenantService.updateTenant(getLoginUser(), 912222, tenantCode, tenantName, 1, "desc"); + tenantService.updateTenant(getLoginUser(), 912222, tenantCode, 1, "desc"); logger.info(result.toString()); // success Assert.assertEquals(Status.TENANT_NOT_EXIST, result.get(Constants.STATUS)); - result = tenantService.updateTenant(getLoginUser(), 1, tenantCode, "TenantServiceTest001", 1, "desc"); + result = tenantService.updateTenant(getLoginUser(), 1, tenantCode, 1, "desc"); logger.info(result.toString()); Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); } catch (Exception e) { @@ -236,7 +234,6 @@ public class TenantServiceTest { Tenant tenant = new Tenant(); tenant.setId(id); tenant.setTenantCode(tenantCode); - tenant.setTenantName(tenantName); return tenant; } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java index 9ec24bbb50..884e9b6b36 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java @@ -149,9 +149,11 @@ public class UdfFuncServiceTest { } @Test - public void testQueryResourceList(){ - Mockito.when(udfFuncMapper.getUdfFuncByType(1, 1)).thenReturn(getList()); - Map result = udfFuncService.queryResourceList(getLoginUser(),1); + public void testQueryUdfFuncList(){ + User user = getLoginUser(); + user.setUserType(UserType.GENERAL_USER); + Mockito.when(udfFuncMapper.getUdfFuncByType(user.getId(), UdfType.HIVE.ordinal())).thenReturn(getList()); + Map result = udfFuncService.queryUdfFuncList(user,UdfType.HIVE.ordinal()); logger.info(result.toString()); Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); List udfFuncList = (List) result.get(Constants.DATA_LIST); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java index c4d3d6e126..d86c2cc93e 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java @@ -225,13 +225,13 @@ public class UsersServiceTest { String userPassword = "userTest0001"; try { //user not exist - Map result = usersService.updateUser(0,userName,userPassword,"3443@qq.com",1,"13457864543","queue", 1); + Map result = usersService.updateUser(getLoginUser(), 0,userName,userPassword,"3443@qq.com",1,"13457864543","queue", 1); Assert.assertEquals(Status.USER_NOT_EXIST, result.get(Constants.STATUS)); logger.info(result.toString()); //success when(userMapper.selectById(1)).thenReturn(getUser()); - result = usersService.updateUser(1,userName,userPassword,"32222s@qq.com",1,"13457864543","queue", 1); + result = usersService.updateUser(getLoginUser(), 1,userName,userPassword,"32222s@qq.com",1,"13457864543","queue", 1); logger.info(result.toString()); Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); } catch (Exception e) { @@ -357,6 +357,12 @@ public class UsersServiceTest { } + private User getLoginUser(){ + User loginUser = new User(); + loginUser.setId(1); + loginUser.setUserType(UserType.ADMIN_USER); + return loginUser; + } @Test public void getUserInfo(){ diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/JsonSerializerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/RegexUtilsTest.java similarity index 50% rename from dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/JsonSerializerTest.java rename to dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/RegexUtilsTest.java index cb92db7f25..5b62d51b07 100644 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/JsonSerializerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/RegexUtilsTest.java @@ -15,44 +15,25 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote; +package org.apache.dolphinscheduler.api.utils; - -import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import org.junit.Assert; import org.junit.Test; -public class JsonSerializerTest { +/** + * RegexUtils test case + */ +public class RegexUtilsTest { @Test - public void testSerialize(){ - TestObj testObj = new TestObj(); - testObj.setAge(12); - byte[] serializeByte = JsonSerializer.serialize(testObj); - - // - TestObj deserialize = JsonSerializer.deserialize(serializeByte, TestObj.class); - - Assert.assertEquals(testObj.getAge(), deserialize.getAge()); + public void testIsNumeric() { + String num1 = "123467854678"; + boolean numeric = RegexUtils.isNumeric(num1); + Assert.assertTrue(numeric); + + String num2 = "0.0.01"; + boolean numeric2 = RegexUtils.isNumeric(num2); + Assert.assertFalse(numeric2); } - static class TestObj { - - private int age; - - public int getAge() { - return age; - } - - public void setAge(int age) { - this.age = age; - } - - @Override - public String toString() { - return "TestObj{" + - "age=" + age + - '}'; - } - } -} +} \ No newline at end of file diff --git a/dolphinscheduler-common/pom.xml b/dolphinscheduler-common/pom.xml index 2ade59550f..52f2361581 100644 --- a/dolphinscheduler-common/pom.xml +++ b/dolphinscheduler-common/pom.xml @@ -21,7 +21,7 @@ org.apache.dolphinscheduler dolphinscheduler - 1.3.2-SNAPSHOT + 1.3.4-SNAPSHOT dolphinscheduler-common dolphinscheduler-common diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java index 85066cc55a..4a696d2e37 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java @@ -221,6 +221,11 @@ public final class Constants { */ public static final String COLON = ":"; + /** + * SPACE " " + */ + public static final String SPACE = " "; + /** * SINGLE_SLASH / */ @@ -231,6 +236,15 @@ public final class Constants { */ public static final String DOUBLE_SLASH = "//"; + /** + * SINGLE_QUOTES "'" + */ + public static final String SINGLE_QUOTES = "'"; + /** + * DOUBLE_QUOTES "\"" + */ + public static final String DOUBLE_QUOTES = "\""; + /** * SEMICOLON ; */ @@ -523,10 +537,9 @@ public final class Constants { public static final int HEARTBEAT_FOR_ZOOKEEPER_INFO_LENGTH = 10; + /** - * hadoop params constant - */ - /** + * hadoop params * jar */ public static final String JAR = "jar"; @@ -833,15 +846,15 @@ public final class Constants { public static final String FLINK_MAIN_CLASS = "-c"; - public static final int[] NOT_TERMINATED_STATES = new int[]{ - ExecutionStatus.SUBMITTED_SUCCESS.ordinal(), - ExecutionStatus.RUNNING_EXECUTION.ordinal(), - ExecutionStatus.DELAY_EXECUTION.ordinal(), - ExecutionStatus.READY_PAUSE.ordinal(), - ExecutionStatus.READY_STOP.ordinal(), - ExecutionStatus.NEED_FAULT_TOLERANCE.ordinal(), - ExecutionStatus.WAITTING_THREAD.ordinal(), - ExecutionStatus.WAITTING_DEPEND.ordinal() + public static final int[] NOT_TERMINATED_STATES = new int[] { + ExecutionStatus.SUBMITTED_SUCCESS.ordinal(), + ExecutionStatus.RUNNING_EXECUTION.ordinal(), + ExecutionStatus.DELAY_EXECUTION.ordinal(), + ExecutionStatus.READY_PAUSE.ordinal(), + ExecutionStatus.READY_STOP.ordinal(), + ExecutionStatus.NEED_FAULT_TOLERANCE.ordinal(), + ExecutionStatus.WAITTING_THREAD.ordinal(), + ExecutionStatus.WAITTING_DEPEND.ordinal() }; /** @@ -1009,4 +1022,14 @@ public final class Constants { * Network IP gets priority, default inner outer */ public static final String NETWORK_PRIORITY_STRATEGY = "dolphin.scheduler.network.priority.strategy"; + + /** + * exec shell scripts + */ + public static final String SH = "sh"; + + /** + * pstree, get pud and sub pid + */ + public static final String PSTREE = "pstree"; } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/QueryType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Event.java similarity index 74% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/QueryType.java rename to dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Event.java index 13820b4bab..9cec2766f1 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/QueryType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Event.java @@ -14,20 +14,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.common.enums; - -public enum QueryType { - FORM, - SQL; +package org.apache.dolphinscheduler.common.enums; - public static QueryType getEnum(int value){ - for (QueryType e:QueryType.values()) { - if(e.ordinal() == value) { - return e; - } - } - //For values out of enum scope - return null; - } +public enum Event { + ACK, + RESULT; } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/SqoopQueryType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/SqoopQueryType.java new file mode 100644 index 0000000000..4d279f5d8e --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/SqoopQueryType.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.enums; + +public enum SqoopQueryType { + + FORM(0, "SQOOP_QUERY_FORM"), + SQL(1, "SQOOP_QUERY_SQL"); + + private final Integer code; + + private final String desc; + + SqoopQueryType(Integer code, String desc) { + this.code = code; + this.desc = desc; + } + + public Integer getCode() { + return code; + } + + public String getDesc() { + return desc; + } +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/process/ResourceInfo.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/process/ResourceInfo.java index a7fc0839eb..287f7267bc 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/process/ResourceInfo.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/process/ResourceInfo.java @@ -42,6 +42,4 @@ public class ResourceInfo { public void setRes(String res) { this.res = res; } - - } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/conditions/ConditionsParameters.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/conditions/ConditionsParameters.java index 7f0f2c8079..094620aee5 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/conditions/ConditionsParameters.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/conditions/ConditionsParameters.java @@ -21,6 +21,7 @@ import org.apache.dolphinscheduler.common.model.DependentTaskModel; import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.AbstractParameters; +import java.util.ArrayList; import java.util.List; public class ConditionsParameters extends AbstractParameters { @@ -43,7 +44,7 @@ public class ConditionsParameters extends AbstractParameters { @Override public List getResourceFilesList() { - return null; + return new ArrayList<>(); } public List getDependTaskList() { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/datax/DataxParameters.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/datax/DataxParameters.java index f54e107995..c1f5f1d81c 100755 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/datax/DataxParameters.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/datax/DataxParameters.java @@ -14,15 +14,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.common.task.datax; -import java.util.ArrayList; -import java.util.List; +package org.apache.dolphinscheduler.common.task.datax; -import org.apache.commons.lang.StringUtils; import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.AbstractParameters; +import org.apache.dolphinscheduler.common.utils.StringUtils; + +import java.util.ArrayList; +import java.util.List; /** * DataX parameter @@ -89,6 +90,16 @@ public class DataxParameters extends AbstractParameters { */ private int jobSpeedRecord; + /** + * Xms memory + */ + private int xms; + + /** + * Xmx memory + */ + private int xmx; + public int getCustomConfig() { return customConfig; } @@ -185,6 +196,22 @@ public class DataxParameters extends AbstractParameters { this.jobSpeedRecord = jobSpeedRecord; } + public int getXms() { + return xms; + } + + public void setXms(int xms) { + this.xms = xms; + } + + public int getXmx() { + return xmx; + } + + public void setXmx(int xmx) { + this.xmx = xmx; + } + @Override public boolean checkParameters() { if (customConfig == Flag.NO.ordinal()) { @@ -204,19 +231,21 @@ public class DataxParameters extends AbstractParameters { @Override public String toString() { - return "DataxParameters{" + - "customConfig=" + customConfig + - ", json='" + json + '\'' + - ", dsType='" + dsType + '\'' + - ", dataSource=" + dataSource + - ", dtType='" + dtType + '\'' + - ", dataTarget=" + dataTarget + - ", sql='" + sql + '\'' + - ", targetTable='" + targetTable + '\'' + - ", preStatements=" + preStatements + - ", postStatements=" + postStatements + - ", jobSpeedByte=" + jobSpeedByte + - ", jobSpeedRecord=" + jobSpeedRecord + - '}'; + return "DataxParameters{" + + "customConfig=" + customConfig + + ", json='" + json + '\'' + + ", dsType='" + dsType + '\'' + + ", dataSource=" + dataSource + + ", dtType='" + dtType + '\'' + + ", dataTarget=" + dataTarget + + ", sql='" + sql + '\'' + + ", targetTable='" + targetTable + '\'' + + ", preStatements=" + preStatements + + ", postStatements=" + postStatements + + ", jobSpeedByte=" + jobSpeedByte + + ", jobSpeedRecord=" + jobSpeedRecord + + ", xms=" + xms + + ", xmx=" + xmx + + '}'; } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/flink/FlinkParameters.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/flink/FlinkParameters.java index 231dd33146..1b1f0a6c5d 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/flink/FlinkParameters.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/flink/FlinkParameters.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.common.task.flink; import org.apache.dolphinscheduler.common.enums.ProgramType; import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.AbstractParameters; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; import java.util.ArrayList; import java.util.List; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/spark/SparkParameters.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/spark/SparkParameters.java index 4e58201bf3..32a2a6b05d 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/spark/SparkParameters.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/spark/SparkParameters.java @@ -216,7 +216,7 @@ public class SparkParameters extends AbstractParameters { @Override public boolean checkParameters() { - return mainJar != null && programType != null && sparkVersion != null; + return mainJar != null && programType != null; } @Override diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java index e3e33566e9..f0e3ec8214 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java @@ -30,13 +30,14 @@ import java.io.BufferedWriter; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileOutputStream; -import java.io.FileWriter; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.io.OutputStreamWriter; import java.io.StringReader; import java.io.UnsupportedEncodingException; import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.nio.charset.UnsupportedCharsetException; import java.util.Optional; @@ -217,7 +218,7 @@ public class FileUtils { return false; } bufferedReader = new BufferedReader(new StringReader(content)); - bufferedWriter = new BufferedWriter(new FileWriter(distFile)); + bufferedWriter = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(distFile), StandardCharsets.UTF_8)); char[] buf = new char[1024]; int len; while ((len = bufferedReader.read(buf)) != -1) { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java index fe9ac932c1..a0c2bf4e58 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java @@ -39,6 +39,7 @@ import java.io.Closeable; import java.io.File; import java.io.IOException; import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.security.PrivilegedExceptionAction; import java.util.Collections; @@ -263,7 +264,7 @@ public class HadoopUtils implements Closeable { } try (FSDataInputStream in = fs.open(new Path(hdfsFilePath))) { - BufferedReader br = new BufferedReader(new InputStreamReader(in)); + BufferedReader br = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8)); Stream stream = br.lines().skip(skipLineNums).limit(limit); return stream.collect(Collectors.toList()); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java index 56ef74d6ee..fc11a2add2 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java @@ -17,6 +17,8 @@ package org.apache.dolphinscheduler.common.utils; +import static java.nio.charset.StandardCharsets.UTF_8; + import static com.fasterxml.jackson.databind.DeserializationFeature.ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT; import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES; import static com.fasterxml.jackson.databind.DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL; @@ -126,6 +128,22 @@ public class JSONUtils { return null; } + /** + * deserialize + * + * @param src byte array + * @param clazz class + * @param deserialize type + * @return deserialize type + */ + public static T parseObject(byte[] src, Class clazz) { + if (src == null) { + return null; + } + String json = new String(src, UTF_8); + return parseObject(json, clazz); + } + /** * json to list * @@ -253,6 +271,27 @@ public class JSONUtils { } } + /** + * serialize to json byte + * + * @param obj object + * @param object type + * @return byte array + */ + public static byte[] toJsonByteArray(T obj) { + if (obj == null) { + return null; + } + String json = ""; + try { + json = toJsonString(obj); + } catch (Exception e) { + logger.error("json serialize exception.", e); + } + + return json.getBytes(UTF_8); + } + public static ObjectNode parseObject(String text) { try { return (ObjectNode) objectMapper.readTree(text); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java index 4f4f12766b..bedfa85f57 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java @@ -45,6 +45,10 @@ public class StringUtils { } public static String replaceNRTtoUnderline(String src) { - return src.replaceAll("[\n|\r|\t]", "_"); + if (isBlank(src)) { + return src; + } else { + return src.replaceAll("[\n|\r|\t]", "_"); + } } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/VarPoolUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/VarPoolUtils.java new file mode 100644 index 0000000000..89a8605a99 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/VarPoolUtils.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.utils; + +import org.apache.dolphinscheduler.common.model.TaskNode; + +import java.text.ParseException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; + +public class VarPoolUtils { + + private static final String LOCALPARAMS = "localParams"; + + private static final String PROP = "prop"; + + private static final String VALUE = "value"; + + /** + * setTaskNodeLocalParams + * @param taskNode taskNode + * @param propToValue propToValue + */ + public static void setTaskNodeLocalParams(TaskNode taskNode, Map propToValue) { + String taskParamsJson = taskNode.getParams(); + Map taskParams = JSONUtils.parseObject(taskParamsJson, HashMap.class); + + Object localParamsObject = taskParams.get(LOCALPARAMS); + if (null != localParamsObject && null != propToValue && propToValue.size() > 0) { + ArrayList localParams = (ArrayList)localParamsObject; + for (int i = 0; i < localParams.size(); i++) { + Map map = (Map)localParams.get(i); + String prop = map.get(PROP); + if (StringUtils.isNotEmpty(prop) && propToValue.containsKey(prop)) { + map.put(VALUE,(String)propToValue.get(prop)); + } + } + taskParams.put(LOCALPARAMS,localParams); + } + taskNode.setParams(JSONUtils.toJsonString(taskParams)); + } + + /** + * convertVarPoolToMap + * @param propToValue propToValue + * @param varPool varPool + * @throws ParseException ParseException + */ + public static void convertVarPoolToMap(Map propToValue, String varPool) throws ParseException { + if (varPool == null || propToValue == null) { + return; + } + String[] splits = varPool.split("\\$VarPool\\$"); + for (String kv : splits) { + String[] kvs = kv.split(","); + if (kvs.length == 2) { + propToValue.put(kvs[0], kvs[1]); + } else { + throw new ParseException(kv, 2); + } + } + } + + /** + * convertPythonScriptPlaceholders + * @param rawScript rawScript + * @return String + * @throws StringIndexOutOfBoundsException StringIndexOutOfBoundsException + */ + public static String convertPythonScriptPlaceholders(String rawScript) throws StringIndexOutOfBoundsException { + int len = "${setShareVar(${".length(); + int scriptStart = 0; + while ((scriptStart = rawScript.indexOf("${setShareVar(${", scriptStart)) != -1) { + int start = -1; + int end = rawScript.indexOf('}', scriptStart + len); + String prop = rawScript.substring(scriptStart + len, end); + + start = rawScript.indexOf(',', end); + end = rawScript.indexOf(')', start); + + String value = rawScript.substring(start + 1, end); + + start = rawScript.indexOf('}', start) + 1; + end = rawScript.length(); + + String replaceScript = String.format("print(\"${{setValue({},{})}}\".format(\"%s\",%s))", prop, value); + + rawScript = rawScript.substring(0, scriptStart) + replaceScript + rawScript.substring(start, end); + + scriptStart += replaceScript.length(); + } + return rawScript; + } +} \ No newline at end of file diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/DataxParametersTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/DataxParametersTest.java new file mode 100644 index 0000000000..d6e2f69882 --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/DataxParametersTest.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.task; + +import org.apache.dolphinscheduler.common.task.datax.DataxParameters; + +import org.junit.Assert; +import org.junit.Test; + +public class DataxParametersTest { + + /** + * jvm parameters + */ + public static final String JVM_EVN = " --jvm=\"-Xms%sG -Xmx%sG\" "; + + @Test + public void testLoadJvmEnv() { + + DataxParameters dataxParameters = new DataxParameters(); + dataxParameters.setXms(0); + dataxParameters.setXmx(-100); + + String actual = loadJvmEnvTest(dataxParameters); + + String except = " --jvm=\"-Xms1G -Xmx1G\" "; + Assert.assertEquals(except,actual); + + dataxParameters.setXms(13); + dataxParameters.setXmx(14); + actual = loadJvmEnvTest(dataxParameters); + except = " --jvm=\"-Xms13G -Xmx14G\" "; + Assert.assertEquals(except,actual); + + } + + @Test + public void testToString() { + + DataxParameters dataxParameters = new DataxParameters(); + dataxParameters.setCustomConfig(0); + dataxParameters.setXms(0); + dataxParameters.setXmx(-100); + dataxParameters.setDataSource(1); + dataxParameters.setDataTarget(1); + dataxParameters.setDsType("MYSQL"); + dataxParameters.setDtType("MYSQL"); + dataxParameters.setJobSpeedByte(1); + dataxParameters.setJobSpeedRecord(1); + dataxParameters.setJson("json"); + + String expected = "DataxParameters" + + "{" + + "customConfig=0, " + + "json='json', " + + "dsType='MYSQL', " + + "dataSource=1, " + + "dtType='MYSQL', " + + "dataTarget=1, " + + "sql='null', " + + "targetTable='null', " + + "preStatements=null, " + + "postStatements=null, " + + "jobSpeedByte=1, " + + "jobSpeedRecord=1, " + + "xms=0, " + + "xmx=-100" + + "}"; + + Assert.assertEquals(expected,dataxParameters.toString()); + } + + public String loadJvmEnvTest(DataxParameters dataXParameters) { + int xms = dataXParameters.getXms() < 1 ? 1 : dataXParameters.getXms(); + int xmx = dataXParameters.getXmx() < 1 ? 1 : dataXParameters.getXmx(); + return String.format(JVM_EVN, xms, xmx); + } +} diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/threadutils/ThreadUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/threadutils/ThreadUtilsTest.java deleted file mode 100644 index 2c76f40c0b..0000000000 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/threadutils/ThreadUtilsTest.java +++ /dev/null @@ -1,190 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.dolphinscheduler.common.threadutils; - -import org.apache.dolphinscheduler.common.thread.Stopper; -import org.apache.dolphinscheduler.common.thread.ThreadPoolExecutors; -import org.apache.dolphinscheduler.common.thread.ThreadUtils; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Calendar; -import java.util.concurrent.*; - -import static org.junit.Assert.*; - - -public class ThreadUtilsTest { - private static final Logger logger = LoggerFactory.getLogger(ThreadUtilsTest.class); - /** - * create a naming thread - */ - @Test - public void testNewDaemonFixedThreadExecutor() { - // create core size and max size are all 3 - ExecutorService testExec = ThreadUtils.newDaemonFixedThreadExecutor("test-exec-thread",10); - - for (int i = 0; i < 19; i++) { - final int index = i; - testExec.submit(() -> { - System.out.println("do some work index " + index); - }); - } - assertFalse(testExec.isShutdown()); - testExec.shutdownNow(); - assertTrue(testExec.isShutdown()); - - } - - /** - * test schedulerThreadExecutor as for print time in scheduler - * default check thread is 1 - */ - @Test - public void testNewDaemonScheduleThreadExecutor() { - - ScheduledExecutorService scheduleService = ThreadUtils.newDaemonThreadScheduledExecutor("scheduler-thread", 1); - Calendar start = Calendar.getInstance(); - Calendar globalTimer = Calendar.getInstance(); - globalTimer.set(2019, Calendar.DECEMBER, 1, 0, 0, 0); - // current - Calendar end = Calendar.getInstance(); - end.set(2019, Calendar.DECEMBER, 1, 0, 0, 3); - Runnable schedulerTask = new Runnable() { - @Override - public void run() { - start.set(2019, Calendar.DECEMBER, 1, 0, 0, 0); - int index = 0; - // send heart beat work - while (start.getTime().getTime() <= end.getTime().getTime()) { - System.out.println("worker here"); - System.out.println(index ++); - start.add(Calendar.SECOND, 1); - globalTimer.add(Calendar.SECOND, 1); - } - System.out.println("time is " + System.currentTimeMillis()); - } - }; - scheduleService.scheduleAtFixedRate(schedulerTask, 2, 10, TimeUnit.SECONDS); - assertFalse(scheduleService.isShutdown()); - try { - Thread.sleep(60000); - } catch (InterruptedException e) { - e.printStackTrace(); - } - scheduleService.shutdownNow(); - assertTrue(scheduleService.isShutdown()); - } - - /** - * test stopper is working normal - */ - @Test - public void testStopper() { - assertTrue(Stopper.isRunning()); - Stopper.stop(); - assertTrue(Stopper.isStopped()); - } - - /** - * test threadPoolExecutors with 3 workers and current each 5 tasks - * @throws InterruptedException - */ - @Test - public void testThreadInfo() throws InterruptedException { - ThreadPoolExecutors workers = ThreadPoolExecutors.getInstance("worker", 3); - for (int i = 0; i < 5; ++i ) { - int index = i; - workers.execute(() -> { - for (int j = 0; j < 10; ++j) { - try { - Thread.sleep(1000); - System.out.printf("worker %d is doing the task", index); - System.out.println(); - workers.printStatus(); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - }); - workers.submit(() -> { - for (int j = 0; j < 10; ++j) { - try { - Thread.sleep(1000); - System.out.printf("worker_2 %d is doing the task", index); - System.out.println(); - workers.printStatus(); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - }); - } - Thread.sleep(50001); - workers.shutdown(); - } - - /** - * test a single daemon thread pool - */ - @Test - public void testNewDaemonSingleThreadExecutor() { - ExecutorService threadTest = ThreadUtils.newDaemonSingleThreadExecutor("thread_test"); - threadTest.execute(() -> { - for (int i = 0; i < 100; ++i) { - System.out.println("daemon working "); - } - - }); - assertFalse(threadTest.isShutdown()); - threadTest.shutdownNow(); - assertTrue(threadTest.isShutdown()); - } - - @Test - public void testNewDaemonCachedThreadPool() { - - ThreadPoolExecutor threadPoolExecutor = ThreadUtils.newDaemonCachedThreadPool("threadTest-"); - Thread thread1 = threadPoolExecutor.getThreadFactory().newThread(() -> { - for (int i = 0; i < 10; ++i) { - System.out.println("this task is with index " + i ); - } - }); - assertTrue(thread1.getName().startsWith("threadTest-")); - assertFalse(threadPoolExecutor.isShutdown()); - threadPoolExecutor.shutdown(); - assertTrue(threadPoolExecutor.isShutdown()); - } - - @Test - public void testNewDaemonCachedThreadPoolWithThreadNumber() { - ThreadPoolExecutor threadPoolExecutor = ThreadUtils.newDaemonCachedThreadPool("threadTest--", 3, 10); - for (int i = 0; i < 10; ++ i) { - threadPoolExecutor.getThreadFactory().newThread(() -> { - assertEquals(3, threadPoolExecutor.getActiveCount()); - System.out.println("this task is first work to do"); - }); - } - assertFalse(threadPoolExecutor.isShutdown()); - threadPoolExecutor.shutdown(); - assertTrue(threadPoolExecutor.isShutdown()); - } - - - -} diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java index b2a255b2e2..f87628c950 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java @@ -14,16 +14,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.common.utils; +import static org.apache.dolphinscheduler.common.Constants.YYYYMMDDHHMMSS; + import org.apache.dolphinscheduler.common.Constants; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; + import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; -import static org.apache.dolphinscheduler.common.Constants.YYYYMMDDHHMMSS; @RunWith(PowerMockRunner.class) @PrepareForTest(DateUtils.class) @@ -81,4 +88,15 @@ public class FileUtilsTest { } } + @Test + public void testWriteContent2File() throws FileNotFoundException { + // file exists, fmt is invalid + String filePath = "test/testFile.txt"; + String content = "正正正faffdasfasdfas"; + FileUtils.writeContent2File(content, filePath); + + String fileContent = FileUtils.readFile2Str(new FileInputStream(new File(filePath))); + Assert.assertEquals(content, fileContent); + } + } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java index e273496f56..af12d5a625 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java @@ -14,13 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.common.utils; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.SerializationFeature; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.JsonNodeFactory; -import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.dolphinscheduler.common.enums.DataType; +import org.apache.dolphinscheduler.common.enums.Direct; +import org.apache.dolphinscheduler.common.model.TaskNode; +import org.apache.dolphinscheduler.common.process.Property; import java.util.ArrayList; import java.util.HashMap; @@ -28,13 +28,15 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import org.apache.dolphinscheduler.common.enums.DataType; -import org.apache.dolphinscheduler.common.enums.Direct; -import org.apache.dolphinscheduler.common.model.TaskNode; -import org.apache.dolphinscheduler.common.process.Property; import org.junit.Assert; import org.junit.Test; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; + public class JSONUtilsTest { @Test @@ -108,9 +110,8 @@ public class JSONUtilsTest { Assert.assertEquals(Direct.IN, direct); } - @Test - public void String2MapTest() { + public void string2MapTest() { String str = list2String(); List maps = JSONUtils.toList(str, @@ -145,6 +146,18 @@ public class JSONUtilsTest { Assert.assertNull(JSONUtils.parseObject("foo", String.class)); } + @Test + public void testJsonByteArray() { + String str = "foo"; + byte[] serializeByte = JSONUtils.toJsonByteArray(str); + String deserialize = JSONUtils.parseObject(serializeByte, String.class); + Assert.assertEquals(str, deserialize); + str = null; + serializeByte = JSONUtils.toJsonByteArray(str); + deserialize = JSONUtils.parseObject(serializeByte, String.class); + Assert.assertNull(deserialize); + } + @Test public void testToList() { Assert.assertEquals(new ArrayList(), diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringUtilsTest.java index eca22def30..f67e89e7e2 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringUtilsTest.java @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.common.utils; import org.junit.Assert; @@ -61,4 +62,19 @@ public class StringUtilsTest { b = StringUtils.isNotBlank("test"); Assert.assertTrue(b); } + + @Test + public void testreplaceNRTtoUnderline() { + String result1 = StringUtils.replaceNRTtoUnderline("abc\n"); + Assert.assertEquals("abc_", result1); + + String result2 = StringUtils.replaceNRTtoUnderline("abc\r"); + Assert.assertEquals("abc_", result2); + + String result3 = StringUtils.replaceNRTtoUnderline("abc\t"); + Assert.assertEquals("abc_", result3); + + String result4 = StringUtils.replaceNRTtoUnderline(null); + Assert.assertNull(result4); + } } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/VarPoolUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/VarPoolUtilsTest.java new file mode 100644 index 0000000000..6713b221bc --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/VarPoolUtilsTest.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.utils; + +import org.apache.dolphinscheduler.common.model.TaskNode; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class VarPoolUtilsTest { + + private static final Logger logger = LoggerFactory.getLogger(VarPoolUtilsTest.class); + + @Test + public void testConvertVarPoolToMap() throws Exception { + String varPool = "p1,66$VarPool$p2,69$VarPool$"; + ConcurrentHashMap propToValue = new ConcurrentHashMap(); + VarPoolUtils.convertVarPoolToMap(propToValue, varPool); + Assert.assertEquals((String)propToValue.get("p1"), "66"); + Assert.assertEquals((String)propToValue.get("p2"), "69"); + logger.info(propToValue.toString()); + } + + @Test + public void testConvertPythonScriptPlaceholders() throws Exception { + String rawScript = "print(${p1});\n${setShareVar(${p1},3)};\n${setShareVar(${p2},4)};"; + rawScript = VarPoolUtils.convertPythonScriptPlaceholders(rawScript); + Assert.assertEquals(rawScript, "print(${p1});\n" + + "print(\"${{setValue({},{})}}\".format(\"p1\",3));\n" + + "print(\"${{setValue({},{})}}\".format(\"p2\",4));"); + logger.info(rawScript); + } + + @Test + public void testSetTaskNodeLocalParams() throws Exception { + String taskJson = "{\"id\":\"tasks-66199\",\"name\":\"file-shell\",\"desc\":null,\"type\":\"SHELL\"," + + "\"runFlag\":\"NORMAL\",\"loc\":null,\"maxRetryTimes\":0,\"retryInterval\":1,\"" + + "params\":{\"rawScript\":\"sh n-1/n-1-1/run.sh\",\"" + + "localParams\":[{\"prop\":\"k1\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"v1\"},{\"prop\":\"k2\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"v2\"}," + + "{\"prop\":\"k3\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"v3\"}],\"" + + "resourceList\":[{\"id\":\"dolphinschedule-code\",\"res\":\"n-1/n-1-1/dolphinscheduler-api-server.log\"}," + + "{\"id\":\"mr-code\",\"res\":\"n-1/n-1-1/hadoop-mapreduce-examples-2.7.4.jar\"}," + + "{\"id\":\"run\",\"res\":\"n-1/n-1-1/run.sh\"}]},\"preTasks\":[],\"extras\":null,\"depList\":[],\"" + + "dependence\":{},\"conditionResult\":{\"successNode\":[\"\"],\"failedNode\":[\"\"]},\"taskInstancePriority\":\"MEDIUM\",\"" + + "workerGroup\":\"default\",\"workerGroupId\":null,\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false},\"delayTime\":0}"; + String changeTaskJson = "{\"id\":\"tasks-66199\",\"name\":\"file-shell\",\"desc\":null,\"type\":\"SHELL\"," + + "\"runFlag\":\"NORMAL\",\"loc\":null,\"maxRetryTimes\":0,\"retryInterval\":1,\"" + + "params\":{\"rawScript\":\"sh n-1/n-1-1/run.sh\",\"" + + "localParams\":[{\"prop\":\"k1\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"k1-value-change\"}," + + "{\"prop\":\"k2\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"k2-value-change\"}," + + "{\"prop\":\"k3\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"v3\"}],\"" + + "resourceList\":[{\"id\":\"dolphinschedule-code\",\"res\":\"n-1/n-1-1/dolphinscheduler-api-server.log\"}," + + "{\"id\":\"mr-code\",\"res\":\"n-1/n-1-1/hadoop-mapreduce-examples-2.7.4.jar\"}," + + "{\"id\":\"run\",\"res\":\"n-1/n-1-1/run.sh\"}]},\"preTasks\":[],\"extras\":null,\"depList\":[],\"" + + "dependence\":{},\"conditionResult\":{\"successNode\":[\"\"],\"failedNode\":[\"\"]},\"taskInstancePriority\":\"MEDIUM\",\"" + + "workerGroup\":\"default\",\"workerGroupId\":null,\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false},\"delayTime\":0}"; + Map propToValue = new HashMap(); + propToValue.put("k1","k1-value-change"); + propToValue.put("k2","k2-value-change"); + + TaskNode taskNode = JSONUtils.parseObject(taskJson,TaskNode.class); + + VarPoolUtils.setTaskNodeLocalParams(taskNode,propToValue); + + Assert.assertEquals(changeTaskJson,JSONUtils.toJsonString(taskNode)); + + } + +} diff --git a/dolphinscheduler-dao/pom.xml b/dolphinscheduler-dao/pom.xml index c474f6d992..6bdaf416d9 100644 --- a/dolphinscheduler-dao/pom.xml +++ b/dolphinscheduler-dao/pom.xml @@ -16,12 +16,13 @@ ~ limitations under the License. --> - + 4.0.0 org.apache.dolphinscheduler dolphinscheduler - 1.3.2-SNAPSHOT + 1.3.4-SNAPSHOT dolphinscheduler-dao ${project.artifactId} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Command.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Command.java index 7d52dc93f3..cba0151828 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Command.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Command.java @@ -131,6 +131,7 @@ public class Command { WarningType warningType, int warningGroupId, Date scheduleTime, + String workerGroup, Priority processInstancePriority) { this.commandType = commandType; this.executorId = executorId; @@ -143,6 +144,7 @@ public class Command { this.failureStrategy = failureStrategy; this.startTime = new Date(); this.updateTime = new Date(); + this.workerGroup = workerGroup; this.processInstancePriority = processInstancePriority; } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java index 3d1a756d25..e3a3f11386 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java @@ -224,6 +224,11 @@ public class ProcessInstance { */ private int tenantId; + /** + * varPool string + */ + private String varPool; + /** * receivers for api */ @@ -256,6 +261,14 @@ public class ProcessInstance { DateUtils.getCurrentTimeStamp(); } + public String getVarPool() { + return varPool; + } + + public void setVarPool(String varPool) { + this.varPool = varPool; + } + public ProcessDefinition getProcessDefinition() { return processDefinition; } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java index 9688200b2c..b13ca87e38 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java @@ -211,6 +211,11 @@ public class TaskInstance implements Serializable { */ private int executorId; + /** + * varPool string + */ + private String varPool; + /** * executor name */ @@ -232,7 +237,14 @@ public class TaskInstance implements Serializable { this.executePath = executePath; } + public String getVarPool() { + return varPool; + } + public void setVarPool(String varPool) { + this.varPool = varPool; + } + public ProcessInstance getProcessInstance() { return processInstance; } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Tenant.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Tenant.java index 1b2eea403d..87674fe995 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Tenant.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Tenant.java @@ -42,11 +42,6 @@ public class Tenant { */ private String tenantCode; - /** - * tenant name - */ - private String tenantName; - /** * description */ @@ -97,14 +92,6 @@ public class Tenant { this.tenantCode = tenantCode; } - public String getTenantName() { - return tenantName; - } - - public void setTenantName(String tenantName) { - this.tenantName = tenantName; - } - public int getQueueId() { return queueId; } @@ -150,7 +137,6 @@ public class Tenant { return "Tenant{" + "id=" + id + ", tenantCode='" + tenantCode + '\'' + - ", tenantName='" + tenantName + '\'' + ", queueId=" + queueId + ", queueName='" + queueName + '\'' + ", queue='" + queue + '\'' + diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/User.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/User.java index 89da171caf..2110fe2021 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/User.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/User.java @@ -79,12 +79,6 @@ public class User { @TableField(exist = false) private String tenantCode; - /** - * tenant name - */ - @TableField(exist = false) - private String tenantName; - /** * queue name */ @@ -203,14 +197,6 @@ public class User { this.alertGroup = alertGroup; } - public String getTenantName() { - return tenantName; - } - - public void setTenantName(String tenantName) { - this.tenantName = tenantName; - } - public String getTenantCode() { return tenantCode; } @@ -272,7 +258,6 @@ public class User { ", tenantId=" + tenantId + ", state=" + state + ", tenantCode='" + tenantCode + '\'' + - ", tenantName='" + tenantName + '\'' + ", queueName='" + queueName + '\'' + ", alertGroup='" + alertGroup + '\'' + ", queue='" + queue + '\'' + diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.java index 86e3172f23..621ec43c27 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.java @@ -34,6 +34,15 @@ import com.baomidou.mybatisplus.core.metadata.IPage; */ public interface ProcessDefinitionMapper extends BaseMapper { + /** + * verify process definition by name + * + * @param projectId projectId + * @param name name + * @return process definition + */ + ProcessDefinition verifyByDefineName(@Param("projectId") int projectId, + @Param("processDefinitionName") String name); /** * query process definition by name diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.java index 8048fda812..a9ebbf000c 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.java @@ -75,6 +75,7 @@ public interface ProcessInstanceMapper extends BaseMapper { * @param projectId projectId * @param processDefinitionId processDefinitionId * @param searchVal searchVal + * @param executorId executorId * @param statusArray statusArray * @param host host * @param startTime startTime diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.java index 424cb8e869..b24fd2ede4 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.java @@ -74,4 +74,11 @@ public interface ProjectMapper extends BaseMapper { */ List queryProjectExceptUserId(@Param("userId") int userId); + /** + * query project list by userId + * @param userId + * @return + */ + List queryProjectCreatedAndAuthorizedByUserId(@Param("userId") int userId); + } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.java index ac23b25c9c..b0e9ca7338 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.java @@ -14,25 +14,27 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.dao.mapper; -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import com.baomidou.mybatisplus.core.metadata.IPage; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.dao.entity.ExecuteStatusCount; import org.apache.dolphinscheduler.dao.entity.TaskInstance; + import org.apache.ibatis.annotations.Param; import java.util.Date; import java.util.List; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.baomidou.mybatisplus.core.metadata.IPage; + /** * task instance mapper interface */ public interface TaskInstanceMapper extends BaseMapper { - List queryTaskByProcessIdAndState(@Param("processInstanceId") Integer processInstanceId, @Param("state") Integer state); @@ -61,6 +63,7 @@ public interface TaskInstanceMapper extends BaseMapper { IPage queryTaskInstanceListPaging(IPage page, @Param("projectId") int projectId, @Param("processInstanceId") Integer processInstanceId, + @Param("processInstanceName") String processInstanceName, @Param("searchVal") String searchVal, @Param("taskName") String taskName, @Param("executorId") int executorId, diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java index a2ce6b29b8..b7351f4b49 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java @@ -100,5 +100,12 @@ public interface UdfFuncMapper extends BaseMapper { */ List listAuthorizedUdfByResourceId(@Param("userId") int userId,@Param("resourceIds") int[] resourceIds); + /** + * batch update udf func + * @param udfFuncList udf list + * @return update num + */ + int batchUpdateUdfFunc(@Param("udfFuncList") List udfFuncList); + } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/DolphinSchedulerManager.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/DolphinSchedulerManager.java index 8d1d862640..b2daae28cb 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/DolphinSchedulerManager.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/DolphinSchedulerManager.java @@ -117,6 +117,8 @@ public class DolphinSchedulerManager { upgradeDao.upgradeDolphinScheduler(schemaDir); if ("1.3.0".equals(schemaVersion)) { upgradeDao.upgradeDolphinSchedulerWorkerGroup(); + } else if ("1.3.2".equals(schemaVersion)) { + upgradeDao.upgradeDolphinSchedulerResourceList(); } version = schemaVersion; } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ResourceDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ResourceDao.java new file mode 100644 index 0000000000..c9ff149306 --- /dev/null +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ResourceDao.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.upgrade; + +import org.apache.dolphinscheduler.common.utils.ConnectionUtils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.util.HashMap; +import java.util.Map; + +/** + * resource dao + */ +public class ResourceDao { + public static final Logger logger = LoggerFactory.getLogger(ProcessDefinitionDao.class); + + /** + * list all resources + * + * @param conn connection + * @return map that key is full_name and value is id + */ + Map listAllResources(Connection conn) { + Map resourceMap = new HashMap<>(); + + String sql = String.format("SELECT id,full_name FROM t_ds_resources"); + ResultSet rs = null; + PreparedStatement pstmt = null; + try { + pstmt = conn.prepareStatement(sql); + rs = pstmt.executeQuery(); + + while (rs.next()) { + Integer id = rs.getInt(1); + String fullName = rs.getString(2); + resourceMap.put(fullName, id); + } + + } catch (Exception e) { + logger.error(e.getMessage(), e); + throw new RuntimeException("sql: " + sql, e); + } finally { + ConnectionUtils.releaseResource(rs, pstmt, conn); + } + + return resourceMap; + } + +} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java index 29c625337d..f0ffc52f68 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java @@ -16,10 +16,10 @@ */ package org.apache.dolphinscheduler.dao.upgrade; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.dao.AbstractBaseDao; import org.apache.dolphinscheduler.dao.datasource.ConnectionFactory; @@ -34,7 +34,9 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.text.MessageFormat; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.stream.Collectors; public abstract class UpgradeDao extends AbstractBaseDao { @@ -88,7 +90,7 @@ public abstract class UpgradeDao extends AbstractBaseDao { /** * init schema */ - public void initSchema(){ + public void initSchema() { DbType dbType = getDbType(); String initSqlPath = ""; if (dbType != null) { @@ -111,6 +113,7 @@ public abstract class UpgradeDao extends AbstractBaseDao { /** * init scheam + * * @param initSqlPath initSqlPath */ public void initSchema(String initSqlPath) { @@ -126,6 +129,7 @@ public abstract class UpgradeDao extends AbstractBaseDao { /** * run DML + * * @param initSqlPath initSqlPath */ private void runInitDML(String initSqlPath) { @@ -148,20 +152,20 @@ public abstract class UpgradeDao extends AbstractBaseDao { try { conn.rollback(); } catch (SQLException e1) { - logger.error(e1.getMessage(),e1); + logger.error(e1.getMessage(), e1); } - logger.error(e.getMessage(),e); - throw new RuntimeException(e.getMessage(),e); + logger.error(e.getMessage(), e); + throw new RuntimeException(e.getMessage(), e); } catch (Exception e) { try { if (null != conn) { conn.rollback(); } } catch (SQLException e1) { - logger.error(e1.getMessage(),e1); + logger.error(e1.getMessage(), e1); } - logger.error(e.getMessage(),e); - throw new RuntimeException(e.getMessage(),e); + logger.error(e.getMessage(), e); + throw new RuntimeException(e.getMessage(), e); } finally { ConnectionUtils.releaseResource(conn); @@ -171,6 +175,7 @@ public abstract class UpgradeDao extends AbstractBaseDao { /** * run DDL + * * @param initSqlPath initSqlPath */ private void runInitDDL(String initSqlPath) { @@ -189,12 +194,12 @@ public abstract class UpgradeDao extends AbstractBaseDao { } catch (IOException e) { - logger.error(e.getMessage(),e); - throw new RuntimeException(e.getMessage(),e); + logger.error(e.getMessage(), e); + throw new RuntimeException(e.getMessage(), e); } catch (Exception e) { - logger.error(e.getMessage(),e); - throw new RuntimeException(e.getMessage(),e); + logger.error(e.getMessage(), e); + throw new RuntimeException(e.getMessage(), e); } finally { ConnectionUtils.releaseResource(conn); @@ -204,6 +209,7 @@ public abstract class UpgradeDao extends AbstractBaseDao { /** * determines whether a table exists + * * @param tableName tableName * @return if table exist return true，else return false */ @@ -211,20 +217,22 @@ public abstract class UpgradeDao extends AbstractBaseDao { /** * determines whether a field exists in the specified table - * @param tableName tableName + * + * @param tableName tableName * @param columnName columnName - * @return if column name exist return true，else return false + * @return if column name exist return true，else return false */ - public abstract boolean isExistsColumn(String tableName,String columnName); + public abstract boolean isExistsColumn(String tableName, String columnName); /** * get current version + * * @param versionName versionName * @return version */ public String getCurrentVersion(String versionName) { - String sql = String.format("select version from %s",versionName); + String sql = String.format("select version from %s", versionName); Connection conn = null; ResultSet rs = null; PreparedStatement pstmt = null; @@ -241,7 +249,7 @@ public abstract class UpgradeDao extends AbstractBaseDao { return version; } catch (SQLException e) { - logger.error(e.getMessage(),e); + logger.error(e.getMessage(), e); throw new RuntimeException("sql: " + sql, e); } finally { ConnectionUtils.releaseResource(rs, pstmt, conn); @@ -251,6 +259,7 @@ public abstract class UpgradeDao extends AbstractBaseDao { /** * upgrade DolphinScheduler + * * @param schemaDir schema dir */ public void upgradeDolphinScheduler(String schemaDir) { @@ -268,52 +277,121 @@ public abstract class UpgradeDao extends AbstractBaseDao { public void upgradeDolphinSchedulerWorkerGroup() { updateProcessDefinitionJsonWorkerGroup(); } + + /** + * upgrade DolphinScheduler resource list + * ds-1.3.2 modify the resource list for process definition json + */ + public void upgradeDolphinSchedulerResourceList() { + updateProcessDefinitionJsonResourceList(); + } + /** * updateProcessDefinitionJsonWorkerGroup */ - protected void updateProcessDefinitionJsonWorkerGroup(){ + protected void updateProcessDefinitionJsonWorkerGroup() { WorkerGroupDao workerGroupDao = new WorkerGroupDao(); ProcessDefinitionDao processDefinitionDao = new ProcessDefinitionDao(); - Map replaceProcessDefinitionMap = new HashMap<>(); + Map replaceProcessDefinitionMap = new HashMap<>(); try { Map oldWorkerGroupMap = workerGroupDao.queryAllOldWorkerGroup(dataSource.getConnection()); - Map processDefinitionJsonMap = processDefinitionDao.queryAllProcessDefinition(dataSource.getConnection()); + Map processDefinitionJsonMap = processDefinitionDao.queryAllProcessDefinition(dataSource.getConnection()); - for (Map.Entry entry : processDefinitionJsonMap.entrySet()){ + for (Map.Entry entry : processDefinitionJsonMap.entrySet()) { ObjectNode jsonObject = JSONUtils.parseObject(entry.getValue()); ArrayNode tasks = JSONUtils.parseArray(jsonObject.get("tasks").toString()); - for (int i = 0 ;i < tasks.size() ; i++){ + for (int i = 0; i < tasks.size(); i++) { ObjectNode task = (ObjectNode) tasks.path(i); ObjectNode workerGroupNode = (ObjectNode) task.path("workerGroupId"); Integer workerGroupId = -1; - if(workerGroupNode != null && workerGroupNode.canConvertToInt()){ + if (workerGroupNode != null && workerGroupNode.canConvertToInt()) { workerGroupId = workerGroupNode.asInt(-1); } if (workerGroupId == -1) { task.put("workerGroup", "default"); - }else { + } else { task.put("workerGroup", oldWorkerGroupMap.get(workerGroupId)); } } jsonObject.remove("task"); - jsonObject.put("tasks",tasks); + jsonObject.put("tasks", tasks); - replaceProcessDefinitionMap.put(entry.getKey(),jsonObject.toString()); + replaceProcessDefinitionMap.put(entry.getKey(), jsonObject.toString()); } - if (replaceProcessDefinitionMap.size() > 0){ - processDefinitionDao.updateProcessDefinitionJson(dataSource.getConnection(),replaceProcessDefinitionMap); + if (replaceProcessDefinitionMap.size() > 0) { + processDefinitionDao.updateProcessDefinitionJson(dataSource.getConnection(), replaceProcessDefinitionMap); } - }catch (Exception e){ - logger.error("update process definition json workergroup error",e); + } catch (Exception e) { + logger.error("update process definition json workergroup error", e); + } + } + + /** + * updateProcessDefinitionJsonResourceList + */ + protected void updateProcessDefinitionJsonResourceList() { + ResourceDao resourceDao = new ResourceDao(); + ProcessDefinitionDao processDefinitionDao = new ProcessDefinitionDao(); + Map replaceProcessDefinitionMap = new HashMap<>(); + try { + Map resourcesMap = resourceDao.listAllResources(dataSource.getConnection()); + Map processDefinitionJsonMap = processDefinitionDao.queryAllProcessDefinition(dataSource.getConnection()); + + for (Map.Entry entry : processDefinitionJsonMap.entrySet()) { + ObjectNode jsonObject = JSONUtils.parseObject(entry.getValue()); + ArrayNode tasks = JSONUtils.parseArray(jsonObject.get("tasks").toString()); + + for (int i = 0; i < tasks.size(); i++) { + ObjectNode task = (ObjectNode) tasks.get(i); + ObjectNode param = (ObjectNode) task.get("params"); + if (param != null) { + + List resourceList = JSONUtils.toList(param.get("resourceList").toString(), ResourceInfo.class); + ResourceInfo mainJar = JSONUtils.parseObject(param.get("mainJar").toString(), ResourceInfo.class); + if (mainJar != null && mainJar.getId() == 0) { + String fullName = mainJar.getRes().startsWith("/") ? mainJar.getRes() : String.format("/%s", mainJar.getRes()); + if (resourcesMap.containsKey(fullName)) { + mainJar.setId(resourcesMap.get(fullName)); + param.put("mainJar", JSONUtils.parseObject(JSONUtils.toJsonString(mainJar))); + } + } + + if (CollectionUtils.isNotEmpty(resourceList)) { + List newResourceList = resourceList.stream().map(resInfo -> { + String fullName = resInfo.getRes().startsWith("/") ? resInfo.getRes() : String.format("/%s", resInfo.getRes()); + if (resInfo.getId() == 0 && resourcesMap.containsKey(fullName)) { + resInfo.setId(resourcesMap.get(fullName)); + } + return resInfo; + }).collect(Collectors.toList()); + param.put("resourceList", JSONUtils.parseObject(JSONUtils.toJsonString(newResourceList))); + } + } + task.put("params", param); + + } + + jsonObject.remove("tasks"); + + jsonObject.put("tasks", tasks); + + replaceProcessDefinitionMap.put(entry.getKey(), jsonObject.toString()); + } + if (replaceProcessDefinitionMap.size() > 0) { + processDefinitionDao.updateProcessDefinitionJson(dataSource.getConnection(), replaceProcessDefinitionMap); + } + } catch (Exception e) { + logger.error("update process definition json resource list error", e); } } /** * upgradeDolphinScheduler DML + * * @param schemaDir schemaDir */ private void upgradeDolphinSchedulerDML(String schemaDir) { @@ -321,8 +399,8 @@ public abstract class UpgradeDao extends AbstractBaseDao { if (StringUtils.isEmpty(rootDir)) { throw new RuntimeException("Environment variable user.dir not found"); } - String sqlFilePath = MessageFormat.format("{0}/sql/upgrade/{1}/{2}/dolphinscheduler_dml.sql",rootDir,schemaDir,getDbType().name().toLowerCase()); - logger.info("sqlSQLFilePath"+sqlFilePath); + String sqlFilePath = MessageFormat.format("{0}/sql/upgrade/{1}/{2}/dolphinscheduler_dml.sql", rootDir, schemaDir, getDbType().name().toLowerCase()); + logger.info("sqlSQLFilePath" + sqlFilePath); Connection conn = null; PreparedStatement pstmt = null; try { @@ -334,13 +412,13 @@ public abstract class UpgradeDao extends AbstractBaseDao { scriptRunner.runScript(sqlReader); if (isExistsTable(T_VERSION_NAME)) { // Change version in the version table to the new version - String upgradeSQL = String.format("update %s set version = ?",T_VERSION_NAME); + String upgradeSQL = String.format("update %s set version = ?", T_VERSION_NAME); pstmt = conn.prepareStatement(upgradeSQL); pstmt.setString(1, schemaVersion); pstmt.executeUpdate(); - }else if (isExistsTable(T_NEW_VERSION_NAME)) { + } else if (isExistsTable(T_NEW_VERSION_NAME)) { // Change version in the version table to the new version - String upgradeSQL = String.format("update %s set version = ?",T_NEW_VERSION_NAME); + String upgradeSQL = String.format("update %s set version = ?", T_NEW_VERSION_NAME); pstmt = conn.prepareStatement(upgradeSQL); pstmt.setString(1, schemaVersion); pstmt.executeUpdate(); @@ -350,38 +428,38 @@ public abstract class UpgradeDao extends AbstractBaseDao { try { conn.rollback(); } catch (SQLException e1) { - logger.error(e1.getMessage(),e1); + logger.error(e1.getMessage(), e1); } - logger.error(e.getMessage(),e); + logger.error(e.getMessage(), e); throw new RuntimeException("sql file not found ", e); } catch (IOException e) { try { conn.rollback(); } catch (SQLException e1) { - logger.error(e1.getMessage(),e1); + logger.error(e1.getMessage(), e1); } - logger.error(e.getMessage(),e); - throw new RuntimeException(e.getMessage(),e); + logger.error(e.getMessage(), e); + throw new RuntimeException(e.getMessage(), e); } catch (SQLException e) { try { if (null != conn) { conn.rollback(); } } catch (SQLException e1) { - logger.error(e1.getMessage(),e1); + logger.error(e1.getMessage(), e1); } - logger.error(e.getMessage(),e); - throw new RuntimeException(e.getMessage(),e); + logger.error(e.getMessage(), e); + throw new RuntimeException(e.getMessage(), e); } catch (Exception e) { try { if (null != conn) { conn.rollback(); } } catch (SQLException e1) { - logger.error(e1.getMessage(),e1); + logger.error(e1.getMessage(), e1); } - logger.error(e.getMessage(),e); - throw new RuntimeException(e.getMessage(),e); + logger.error(e.getMessage(), e); + throw new RuntimeException(e.getMessage(), e); } finally { ConnectionUtils.releaseResource(pstmt, conn); } @@ -390,13 +468,14 @@ public abstract class UpgradeDao extends AbstractBaseDao { /** * upgradeDolphinScheduler DDL + * * @param schemaDir schemaDir */ private void upgradeDolphinSchedulerDDL(String schemaDir) { if (StringUtils.isEmpty(rootDir)) { throw new RuntimeException("Environment variable user.dir not found"); } - String sqlFilePath = MessageFormat.format("{0}/sql/upgrade/{1}/{2}/dolphinscheduler_ddl.sql",rootDir,schemaDir,getDbType().name().toLowerCase()); + String sqlFilePath = MessageFormat.format("{0}/sql/upgrade/{1}/{2}/dolphinscheduler_ddl.sql", rootDir, schemaDir, getDbType().name().toLowerCase()); Connection conn = null; PreparedStatement pstmt = null; try { @@ -411,20 +490,20 @@ public abstract class UpgradeDao extends AbstractBaseDao { } catch (FileNotFoundException e) { - logger.error(e.getMessage(),e); + logger.error(e.getMessage(), e); throw new RuntimeException("sql file not found ", e); } catch (IOException e) { - logger.error(e.getMessage(),e); - throw new RuntimeException(e.getMessage(),e); + logger.error(e.getMessage(), e); + throw new RuntimeException(e.getMessage(), e); } catch (SQLException e) { - logger.error(e.getMessage(),e); - throw new RuntimeException(e.getMessage(),e); + logger.error(e.getMessage(), e); + throw new RuntimeException(e.getMessage(), e); } catch (Exception e) { - logger.error(e.getMessage(),e); - throw new RuntimeException(e.getMessage(),e); + logger.error(e.getMessage(), e); + throw new RuntimeException(e.getMessage(), e); } finally { ConnectionUtils.releaseResource(pstmt, conn); } @@ -434,15 +513,16 @@ public abstract class UpgradeDao extends AbstractBaseDao { /** * update version + * * @param version version */ public void updateVersion(String version) { // Change version in the version table to the new version String versionName = T_VERSION_NAME; - if(!SchemaUtils.isAGreatVersion("1.2.0" , version)){ + if (!SchemaUtils.isAGreatVersion("1.2.0", version)) { versionName = "t_ds_version"; } - String upgradeSQL = String.format("update %s set version = ?",versionName); + String upgradeSQL = String.format("update %s set version = ?", versionName); PreparedStatement pstmt = null; Connection conn = null; try { @@ -452,7 +532,7 @@ public abstract class UpgradeDao extends AbstractBaseDao { pstmt.executeUpdate(); } catch (SQLException e) { - logger.error(e.getMessage(),e); + logger.error(e.getMessage(), e); throw new RuntimeException("sql: " + upgradeSQL, e); } finally { ConnectionUtils.releaseResource(pstmt, conn); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/DagHelper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/DagHelper.java index d3b829cb4f..9a772322e8 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/DagHelper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/DagHelper.java @@ -22,11 +22,13 @@ import org.apache.dolphinscheduler.common.graph.DAG; import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.model.TaskNodeRelation; import org.apache.dolphinscheduler.common.process.ProcessDag; +import org.apache.dolphinscheduler.common.task.conditions.ConditionsParameters; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.dao.entity.ProcessData; import org.apache.dolphinscheduler.dao.entity.TaskInstance; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,6 +47,7 @@ public class DagHelper { /** * generate flow node relation list by task node list; * Edges that are not in the task Node List will not be added to the result + * * @param taskNodeList taskNodeList * @return task node relation list */ @@ -66,10 +69,11 @@ public class DagHelper { /** * generate task nodes needed by dag - * @param taskNodeList taskNodeList - * @param startNodeNameList startNodeNameList + * + * @param taskNodeList taskNodeList + * @param startNodeNameList startNodeNameList * @param recoveryNodeNameList recoveryNodeNameList - * @param taskDependType taskDependType + * @param taskDependType taskDependType * @return task node list */ public static List generateFlowNodeListByStartNode(List taskNodeList, List startNodeNameList, @@ -77,8 +81,8 @@ public class DagHelper { List destFlowNodeList = new ArrayList<>(); List startNodeList = startNodeNameList; - if(taskDependType != TaskDependType.TASK_POST - && CollectionUtils.isEmpty(startNodeList)){ + if (taskDependType != TaskDependType.TASK_POST + && CollectionUtils.isEmpty(startNodeList)) { logger.error("start node list is empty! cannot continue run the process "); return destFlowNodeList; } @@ -126,7 +130,8 @@ public class DagHelper { /** * find all the nodes that depended on the start node - * @param startNode startNode + * + * @param startNode startNode * @param taskNodeList taskNodeList * @return task node list */ @@ -150,9 +155,10 @@ public class DagHelper { /** * find all nodes that start nodes depend on. - * @param startNode startNode + * + * @param startNode startNode * @param recoveryNodeNameList recoveryNodeNameList - * @param taskNodeList taskNodeList + * @param taskNodeList taskNodeList * @return task node list */ private static List getFlowNodeListPre(TaskNode startNode, List recoveryNodeNameList, List taskNodeList, List visitedNodeNameList) { @@ -165,9 +171,6 @@ public class DagHelper { resultList.add(startNode); } if (CollectionUtils.isEmpty(depList)) { - if (null != startNode) { - visitedNodeNameList.add(startNode.getName()); - } return resultList; } for (String depNodeName : depList) { @@ -187,10 +190,11 @@ public class DagHelper { /** * generate dag by start nodes and recovery nodes + * * @param processDefinitionJson processDefinitionJson - * @param startNodeNameList startNodeNameList - * @param recoveryNodeNameList recoveryNodeNameList - * @param depNodeType depNodeType + * @param startNodeNameList startNodeNameList + * @param recoveryNodeNameList recoveryNodeNameList + * @param depNodeType depNodeType * @return process dag * @throws Exception if error throws Exception */ @@ -217,10 +221,11 @@ public class DagHelper { /** * parse the forbidden task nodes in process definition. + * * @param processDefinitionJson processDefinitionJson * @return task node map */ - public static Map getForbiddenTaskNodeMaps(String processDefinitionJson){ + public static Map getForbiddenTaskNodeMaps(String processDefinitionJson) { Map forbidTaskNodeMap = new ConcurrentHashMap<>(); ProcessData processData = JSONUtils.parseObject(processDefinitionJson, ProcessData.class); @@ -228,8 +233,8 @@ public class DagHelper { if (null != processData) { taskNodeList = processData.getTasks(); } - for(TaskNode node : taskNodeList){ - if(node.isForbidden()){ + for (TaskNode node : taskNodeList) { + if (node.isForbidden()) { forbidTaskNodeMap.putIfAbsent(node.getName(), node); } } @@ -239,8 +244,9 @@ public class DagHelper { /** * find node by node name + * * @param nodeDetails nodeDetails - * @param nodeName nodeName + * @param nodeName nodeName * @return task node */ public static TaskNode findNodeByName(List nodeDetails, String nodeName) { @@ -252,71 +258,89 @@ public class DagHelper { return null; } - /** - * get start vertex in one dag - * it would find the post node if the start vertex is forbidden running - * @param parentNodeName previous node - * @param dag dag + * the task can be submit when all the depends nodes are forbidden or complete + * + * @param taskNode taskNode + * @param dag dag * @param completeTaskList completeTaskList - * @return start Vertex list + * @return can submit */ - public static Collection getStartVertex(String parentNodeName, DAG dag, - Map completeTaskList){ - - if(completeTaskList == null){ - completeTaskList = new HashMap<>(); + public static boolean allDependsForbiddenOrEnd(TaskNode taskNode, + DAG dag, + Map skipTaskNodeList, + Map completeTaskList) { + List dependList = taskNode.getDepList(); + if (dependList == null) { + return true; } - Collection startVertexs = null; - if(StringUtils.isNotEmpty(parentNodeName)){ - startVertexs = dag.getSubsequentNodes(parentNodeName); - }else{ - startVertexs = dag.getBeginNode(); + for (String dependNodeName : dependList) { + TaskNode dependNode = dag.getNode(dependNodeName); + if (dependNode == null || completeTaskList.containsKey(dependNodeName) + || dependNode.isForbidden() + || skipTaskNodeList.containsKey(dependNodeName)) { + continue; + } else { + return false; + } } + return true; + } - List tmpStartVertexs = new ArrayList<>(); - if(startVertexs!= null){ - tmpStartVertexs.addAll(startVertexs); + /** + * parse the successor nodes of previous node. + * this function parse the condition node to find the right branch. + * also check all the depends nodes forbidden or complete + * + * @param preNodeName + * @return successor nodes + */ + public static Set parsePostNodes(String preNodeName, + Map skipTaskNodeList, + DAG dag, + Map completeTaskList) { + Set postNodeList = new HashSet<>(); + Collection startVertexes = new ArrayList<>(); + if (preNodeName == null) { + startVertexes = dag.getBeginNode(); + } else if (dag.getNode(preNodeName).isConditionsTask()) { + List conditionTaskList = parseConditionTask(preNodeName, skipTaskNodeList, dag, completeTaskList); + startVertexes.addAll(conditionTaskList); + } else { + startVertexes = dag.getSubsequentNodes(preNodeName); } - - for(String start : startVertexs){ - TaskNode startNode = dag.getNode(start); - if(!startNode.isForbidden() && !completeTaskList.containsKey(start)){ - // the start can be submit if not forbidden and not in complete tasks + for (String subsequent : startVertexes) { + TaskNode taskNode = dag.getNode(subsequent); + if (isTaskNodeNeedSkip(taskNode, skipTaskNodeList)) { + setTaskNodeSkip(subsequent, dag, completeTaskList, skipTaskNodeList); continue; } - // then submit the post nodes - Collection postNodes = getStartVertex(start, dag, completeTaskList); - for(String post : postNodes){ - TaskNode postNode = dag.getNode(post); - if(taskNodeCanSubmit(postNode, dag, completeTaskList)){ - tmpStartVertexs.add(post); - } + if (!DagHelper.allDependsForbiddenOrEnd(taskNode, dag, skipTaskNodeList, completeTaskList)) { + continue; } - tmpStartVertexs.remove(start); + if (taskNode.isForbidden() || completeTaskList.containsKey(subsequent)) { + postNodeList.addAll(parsePostNodes(subsequent, skipTaskNodeList, dag, completeTaskList)); + continue; + } + postNodeList.add(subsequent); } - return tmpStartVertexs; + return postNodeList; } /** - * the task can be submit when all the depends nodes are forbidden or complete - * @param taskNode taskNode - * @param dag dag - * @param completeTaskList completeTaskList - * @return can submit + * if all of the task dependence are skipped, skip it too. + * + * @param taskNode + * @return */ - public static boolean taskNodeCanSubmit(TaskNode taskNode, - DAG dag, - Map completeTaskList) { - - List dependList = taskNode.getDepList(); - if(dependList == null){ - return true; + private static boolean isTaskNodeNeedSkip(TaskNode taskNode, + Map skipTaskNodeList + ) { + if (CollectionUtils.isEmpty(taskNode.getDepList())) { + return false; } - - for(String dependNodeName : dependList){ - TaskNode dependNode = dag.getNode(dependNodeName); - if(!dependNode.isForbidden() && !completeTaskList.containsKey(dependNodeName)){ + for (String depNode : taskNode.getDepList()) { + if (!skipTaskNodeList.containsKey(depNode)) { return false; } } @@ -324,6 +348,67 @@ public class DagHelper { } + /** + * parse condition task find the branch process + * set skip flag for another one. + * + * @param nodeName + * @return + */ + public static List parseConditionTask(String nodeName, + Map skipTaskNodeList, + DAG dag, + Map completeTaskList) { + List conditionTaskList = new ArrayList<>(); + TaskNode taskNode = dag.getNode(nodeName); + if (!taskNode.isConditionsTask()) { + return conditionTaskList; + } + if (!completeTaskList.containsKey(nodeName)) { + return conditionTaskList; + } + TaskInstance taskInstance = completeTaskList.get(nodeName); + ConditionsParameters conditionsParameters = + JSONUtils.parseObject(taskNode.getConditionResult(), ConditionsParameters.class); + List skipNodeList = new ArrayList<>(); + if (taskInstance.getState().typeIsSuccess()) { + conditionTaskList = conditionsParameters.getSuccessNode(); + skipNodeList = conditionsParameters.getFailedNode(); + } else if (taskInstance.getState().typeIsFailure()) { + conditionTaskList = conditionsParameters.getFailedNode(); + skipNodeList = conditionsParameters.getSuccessNode(); + } else { + conditionTaskList.add(nodeName); + } + for (String failedNode : skipNodeList) { + setTaskNodeSkip(failedNode, dag, completeTaskList, skipTaskNodeList); + } + return conditionTaskList; + } + + /** + * set task node and the post nodes skip flag + * + * @param skipNodeName + * @param dag + * @param completeTaskList + * @param skipTaskNodeList + */ + private static void setTaskNodeSkip(String skipNodeName, + DAG dag, + Map completeTaskList, + Map skipTaskNodeList) { + skipTaskNodeList.putIfAbsent(skipNodeName, dag.getNode(skipNodeName)); + Collection postNodeList = dag.getSubsequentNodes(skipNodeName); + for (String post : postNodeList) { + TaskNode postNode = dag.getNode(post); + if (isTaskNodeNeedSkip(postNode, skipTaskNodeList)) { + setTaskNodeSkip(post, dag, completeTaskList, skipTaskNodeList); + } + } + } + + /*** * build dag graph * @param processDag processDag @@ -331,19 +416,19 @@ public class DagHelper { */ public static DAG buildDagGraph(ProcessDag processDag) { - DAG dag = new DAG<>(); + DAG dag = new DAG<>(); //add vertex - if (CollectionUtils.isNotEmpty(processDag.getNodes())){ - for (TaskNode node : processDag.getNodes()){ - dag.addNode(node.getName(),node); + if (CollectionUtils.isNotEmpty(processDag.getNodes())) { + for (TaskNode node : processDag.getNodes()) { + dag.addNode(node.getName(), node); } } //add edge - if (CollectionUtils.isNotEmpty(processDag.getEdges())){ - for (TaskNodeRelation edge : processDag.getEdges()){ - dag.addEdge(edge.getStartNode(),edge.getEndNode()); + if (CollectionUtils.isNotEmpty(processDag.getEdges())) { + for (TaskNodeRelation edge : processDag.getEdges()) { + dag.addEdge(edge.getStartNode(), edge.getEndNode()); } } return dag; @@ -351,6 +436,7 @@ public class DagHelper { /** * get process dag + * * @param taskNodeList task node list * @return Process dag */ @@ -378,21 +464,22 @@ public class DagHelper { /** * is there have conditions after the parent node + * * @param parentNodeName * @return */ public static boolean haveConditionsAfterNode(String parentNodeName, DAG dag - ){ + ) { boolean result = false; Set subsequentNodes = dag.getSubsequentNodes(parentNodeName); - if(CollectionUtils.isEmpty(subsequentNodes)){ + if (CollectionUtils.isEmpty(subsequentNodes)) { return result; } - for(String nodeName : subsequentNodes){ + for (String nodeName : subsequentNodes) { TaskNode taskNode = dag.getNode(nodeName); List preTasksList = JSONUtils.toList(taskNode.getPreTasks(), String.class); - if(preTasksList.contains(parentNodeName) && taskNode.isConditionsTask()){ + if (preTasksList.contains(parentNodeName) && taskNode.isConditionsTask()) { return true; } } @@ -401,19 +488,20 @@ public class DagHelper { /** * is there have conditions after the parent node + * * @param parentNodeName * @return */ public static boolean haveConditionsAfterNode(String parentNodeName, List taskNodes - ){ + ) { boolean result = false; - if(CollectionUtils.isEmpty(taskNodes)){ + if (CollectionUtils.isEmpty(taskNodes)) { return result; } - for(TaskNode taskNode : taskNodes){ + for (TaskNode taskNode : taskNodes) { List preTasksList = JSONUtils.toList(taskNode.getPreTasks(), String.class); - if(preTasksList.contains(parentNodeName) && taskNode.isConditionsTask()){ + if (preTasksList.contains(parentNodeName) && taskNode.isConditionsTask()) { return true; } } diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.xml index 0481f7deab..ff9ab8f63b 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.xml @@ -18,8 +18,14 @@ + select * from t_ds_process_instance - where 1=1 + where process_definition_id=#{processDefinitionId} and state in @@ -175,7 +178,6 @@ - and process_definition_id=#{processDefinitionId} and (schedule_time = ]]> #{startTime} and schedule_time #{endTime} or start_time = ]]> #{startTime} and start_time #{endTime}) @@ -203,4 +205,4 @@ order by id asc - \ No newline at end of file + diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.xml index 5ab0756250..17a57c8a89 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.xml @@ -64,5 +64,9 @@ from t_ds_project where user_id = #{userId} + \ No newline at end of file diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml index 143761bf8c..eb34ffcb3a 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml @@ -98,7 +98,8 @@ diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TenantMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TenantMapper.xml index fc9219ce86..93d4909cb3 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TenantMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TenantMapper.xml @@ -34,7 +34,7 @@ FROM t_ds_tenant t,t_ds_queue q WHERE t.queue_id = q.id - and t.tenant_name like concat('%', #{searchVal}, '%') + and t.tenant_code like concat('%', #{searchVal}, '%') order by t.update_time desc diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.xml index e38d1637d6..18de6db620 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.xml @@ -111,4 +111,17 @@ + + + + update t_ds_udfs + + resource_name=#{udf.resourceName}, + update_time=#{udf.updateTime} + + + id=#{udf.id} + + + \ No newline at end of file diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UserMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UserMapper.xml index f219ad3efe..d1a3a90a73 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UserMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UserMapper.xml @@ -32,7 +32,7 @@