diff --git a/.github/workflows/ci_e2e.yml b/.github/workflows/ci_e2e.yml index fe818d0668..7ec204e145 100644 --- a/.github/workflows/ci_e2e.yml +++ b/.github/workflows/ci_e2e.yml @@ -49,7 +49,8 @@ jobs: - name: Docker Run run: | VERSION=`cat $(pwd)/pom.xml| grep "SNAPSHOT" | awk -F "-SNAPSHOT" '{print $1}' | awk -F ">" '{print $2}'` - docker run -dit -e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test -p 8888:8888 dolphinscheduler:$VERSION all + mkdir -p /tmp/logs + docker run -dit -e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test -v /tmp/logs:/opt/dolphinscheduler/logs -p 8888:8888 dolphinscheduler:$VERSION all - name: Check Server Status run: sh ./dockerfile/hooks/check - name: Prepare e2e env @@ -65,7 +66,10 @@ jobs: - name: Run e2e Test run: cd ./e2e && mvn -B clean test - name: Collect logs - run: | - mkdir -p ${LOG_DIR} - docker logs dolphinscheduler > ${LOG_DIR}/dolphinscheduler.txt - continue-on-error: true + if: failure() + uses: actions/upload-artifact@v1 + with: + name: dslogs + path: /tmp/logs + + diff --git a/.github/workflows/ci_ut.yml b/.github/workflows/ci_ut.yml index 12f7c04ed6..8d75bae163 100644 --- a/.github/workflows/ci_ut.yml +++ b/.github/workflows/ci_ut.yml @@ -15,7 +15,11 @@ # limitations under the License. # -on: ["pull_request", "push"] +on: + pull_request: + push: + branches: + - dev env: DOCKER_DIR: ./docker LOG_DIR: /tmp/dolphinscheduler @@ -48,19 +52,18 @@ jobs: uses: actions/setup-java@v1 with: java-version: 1.8 + - name: Git fetch unshallow + run: | + git fetch --unshallow + git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" + git fetch origin - name: Compile run: | export MAVEN_OPTS='-Dmaven.repo.local=.m2/repository -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC -XX:-UseGCOverheadLimit -Xmx3g' mvn test -B -Dmaven.test.skip=false - name: Upload coverage report to codecov - if: github.event_name == 'pull_request' run: | CODECOV_TOKEN="09c2663f-b091-4258-8a47-c981827eb29a" bash <(curl -s https://codecov.io/bash) - - name: Git fetch unshallow - run: | - git fetch --unshallow - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" - git fetch origin - name: Run SonarCloud Analysis run: > mvn verify --batch-mode diff --git a/.gitignore b/.gitignore index 7cf1d4d7db..edf803fbeb 100644 --- a/.gitignore +++ b/.gitignore @@ -145,6 +145,5 @@ dolphinscheduler-ui/dist/js/home/index.78a5d12.js.map dolphinscheduler-ui/dist/js/login/index.291b8e3.js dolphinscheduler-ui/dist/js/login/index.291b8e3.js.map dolphinscheduler-ui/dist/lib/external/ -dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/taskInstance/index.vue /dolphinscheduler-dao/src/main/resources/dao/data_source.properties diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8ed9aac897..e02ed113c4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,35 +1,53 @@ -* First from the remote repository *https://github.com/apache/incubator-dolphinscheduler.git* fork code to your own repository -* there are three branches in the remote repository currently: - * master normal delivery branch - After the stable version is released, the code for the stable version branch is merged into the master branch. +# Development - * dev daily development branch - The daily development branch, the newly submitted code can pull requests to this branch. +Start by forking the dolphinscheduler GitHub repository, make changes in a branch and then send a pull request. +## Set up your dolphinscheduler GitHub Repository -* Clone your own warehouse to your local +There are three branches in the remote repository currently: + - `master` : normal delivery branch. After the stable version is released, the code for the stable version branch is merged into the master branch. + + - `dev` : daily development branch. The daily development branch, the newly submitted code can pull requests to this branch. + + - `x.x.x-release` : the stable release version. - `git clone https://github.com/apache/incubator-dolphinscheduler.git` +So, you should fork the `dev` branch. -* Add remote repository address, named upstream +After forking the [dolphinscheduler upstream source repository](https://github.com/apache/incubator-dolphinscheduler/fork) to your personal repository, you can set your personal development environment. - `git remote add upstream https://github.com/apache/incubator-dolphinscheduler.git` +```sh +$ cd +$ git clone < your personal forked dolphinscheduler repo> +$ cd incubator-dolphinscheduler +``` -* View repository: +## Set git remote as ``upstream`` - `git remote -v` +Add remote repository address, named upstream -> There will be two repositories at this time: origin (your own warehouse) and upstream (remote repository) +```sh +git remote add upstream https://github.com/apache/incubator-dolphinscheduler.git +``` -* Get/update remote repository code (already the latest code, skip it) +View repository: - `git fetch upstream` +```sh +git remote -v +``` +There will be two repositories at this time: origin (your own warehouse) and upstream (remote repository) -* Synchronize remote repository code to local repository +Get/update remote repository code (already the latest code, skip it). + +```sh +git fetch upstream ``` + +Synchronize remote repository code to local repository + +```sh git checkout origin/dev git merge --no-ff upstream/dev ``` @@ -41,24 +59,46 @@ git checkout -b dev-1.0 upstream/dev-1.0 git push --set-upstream origin dev1.0 ``` -* After modifying the code locally, submit it to your own repository: +## Create your feature branch +Before making code changes, make sure you create a separate branch for them. + +```sh +$ git checkout -b +``` + +## Commit changes +After modifying the code locally, submit it to your own repository: + +```sh + +git commit -m 'information about your feature' +``` + +## Push to the branch + + +Push your locally committed changes to the remote origin (your fork). -`git commit -m 'test commit'` -`git push` +``` +$ git push origin +``` + +## Create a pull request -* Submit changes to the remote repository +After submitting changes to your remote repository, you should click on the new pull request On the following github page. -* On the github page, click on the new pull request.

- - + +

+ + +Select the modified local branch and the branch to merge past to create a pull request. -* Select the modified local branch and the branch to merge past to create a pull request.

- - + +

-* Next, the administrator is responsible for **merging** to complete the pull request +Next, the administrator is responsible for **merging** to complete the pull request. diff --git a/ReleaseNotes.md b/ReleaseNotes.md deleted file mode 100644 index 8d837c465b..0000000000 --- a/ReleaseNotes.md +++ /dev/null @@ -1,55 +0,0 @@ -## 1.2.0 - -### New Feature -1. Support postgre sql -2. Change all Chinese names to English -3. Add flink and http task support -4. Cross project dependencies -5. Modify mybatis to mybatisplus, support multy databases. -6. Add export and import definition feaure -7. Github actions ci compile check -8. Add method and parameters comments -9. Add java doc for common module - - -### Enhancement -1. Add license and notice files -2. Move batchDelete Process Define/Instance Outside for transactional -3. Remove space before and after login user name -4. Dockerfile optimization -5. Change mysql-connector-java scope to test -6. Owners and administrators can delete schedule -7. DB page rename and background color modification  -8. Add postgre performance monitor -9. Resolve style conflict, recipient cannot tab and value verification -10. Checkbox change background color and env to Chinese -11. Change chinese sql to english -12. Change sqlSessionTemplate singleton and reformat code  -13. The value of loadaverage should be two decimal places -14. Delete alert group need delete the relation of user and alert group -15. Remove check resources when delete tenant -16. Check processInstance state before delete worker group  -17. Add check user and definitions function when delete tenant -18. Delete before check to avoid KeeperException$NoNodeException - -### Bug Fixes -1. Fix #1245, make scanCommand transactional -2. Fix ZKWorkerClient not close PathChildrenCache -3. Data type convert error ，email send error bug fix -4. Catch exception transaction method does not take effect to modify -5. Fix the spring transaction not worker bug -6. Task log print worker log bug fix -7. Fix api server debug mode bug -8. The task is abnormal and task is running bug fix -9. Fix bug: tasks queue length error -10. Fix unsuitable error message -11. Fix bug: phone can be empty -12. Fix email error password -13. Fix CheckUtils.checkUserParams method -14. The process cannot be terminated while tasks in the status submit success -15. Fix too many connection in upgrade or create  -16. Fix the bug when worker execute task using queue. and remove checking -17. Resole verify udf name error and delete udf error  -18. Fix bug: task cannot submit when recovery failover -19. Fix bug: the administrator authorizes the project to ordinary users,but ordinary users cannot see the process definition created by the administrator -20. Fix bug: create dolphinscheduler sql failed \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/params.py b/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/params.py index 3780f6c27e..93b3249614 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/params.py +++ b/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/params.py @@ -7,16 +7,16 @@ to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE_2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - """ + import sys from resource_management import * from resource_management.core.logger import Logger diff --git a/charts/README.md b/charts/README.md new file mode 100644 index 0000000000..6f0317b9e2 --- /dev/null +++ b/charts/README.md @@ -0,0 +1,226 @@ +# Dolphin Scheduler + +[Dolphin Scheduler](https://dolphinscheduler.apache.org) is a distributed and easy-to-expand visual DAG workflow scheduling system, dedicated to solving the complex dependencies in data processing, making the scheduling system out of the box for data processing. + +## Introduction +This chart bootstraps a [Dolphin Scheduler](https://dolphinscheduler.apache.org) distributed deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +## Prerequisites + +- Kubernetes 1.10+ +- PV provisioner support in the underlying infrastructure + +## Installing the Chart + +To install the chart with the release name `my-release`: + +```bash +$ git clone https://github.com/apache/incubator-dolphinscheduler.git +$ cd incubator-dolphinscheduler +$ helm install --name dolphinscheduler . +``` +These commands deploy Dolphin Scheduler on the Kubernetes cluster in the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation. + +> **Tip**: List all releases using `helm list` + +## Uninstalling the Chart + +To uninstall/delete the `dolphinscheduler` deployment: + +```bash +$ helm delete --purge dolphinscheduler +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +## Configuration + +The following tables lists the configurable parameters of the Dolphins Scheduler chart and their default values. + +| Parameter | Description | Default | +| --------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------- | +| `timezone` | World time and date for cities in all time zones | `Asia/Shanghai` | +| `image.registry` | Docker image registry for the Dolphins Scheduler | `docker.io` | +| `image.repository` | Docker image repository for the Dolphins Scheduler | `dolphinscheduler` | +| `image.tag` | Docker image version for the Dolphins Scheduler | `1.2.1` | +| `image.imagePullPolicy` | Image pull policy. One of Always, Never, IfNotPresent | `IfNotPresent` | +| `imagePullSecrets` | ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images | `[]` | +| | | | +| `postgresql.enabled` | If not exists external PostgreSQL, by default, the Dolphins Scheduler will use a internal PostgreSQL | `true` | +| `postgresql.postgresqlUsername` | The username for internal PostgreSQL | `root` | +| `postgresql.postgresqlPassword` | The password for internal PostgreSQL | `root` | +| `postgresql.postgresqlDatabase` | The database for internal PostgreSQL | `dolphinscheduler` | +| `postgresql.persistence.enabled` | Set `postgresql.persistence.enabled` to `true` to mount a new volume for internal PostgreSQL | `false` | +| `postgresql.persistence.size` | `PersistentVolumeClaim` Size | `20Gi` | +| `postgresql.persistence.storageClass` | PostgreSQL data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` | +| `externalDatabase.host` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database host will use it. | `localhost` | +| `externalDatabase.port` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database port will use it. | `5432` | +| `externalDatabase.username` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database username will use it. | `root` | +| `externalDatabase.password` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database password will use it. | `root` | +| `externalDatabase.database` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database database will use it. | `dolphinscheduler` | +| | | | +| `zookeeper.enabled` | If not exists external Zookeeper, by default, the Dolphin Scheduler will use a internal Zookeeper | `true` | +| `zookeeper.taskQueue` | Specify task queue for `master` and `worker` | `zookeeper` | +| `zookeeper.persistence.enabled` | Set `zookeeper.persistence.enabled` to `true` to mount a new volume for internal Zookeeper | `false` | +| `zookeeper.persistence.size` | `PersistentVolumeClaim` Size | `20Gi` | +| `zookeeper.persistence.storageClass` | Zookeeper data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` | +| `externalZookeeper.taskQueue` | If exists external Zookeeper, and set `zookeeper.enable` value to false. Specify task queue for `master` and `worker` | `zookeeper` | +| `externalZookeeper.zookeeperQuorum` | If exists external Zookeeper, and set `zookeeper.enable` value to false. Specify Zookeeper quorum | `127.0.0.1:2181` | +| | | | +| `master.podManagementPolicy` | PodManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down | `Parallel` | +| `master.replicas` | Replicas is the desired number of replicas of the given Template | `3` | +| `master.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` | +| `master.tolerations` | If specified, the pod's tolerations | `{}` | +| `master.affinity` | If specified, the pod's scheduling constraints | `{}` | +| `master.configmap.MASTER_EXEC_THREADS` | Master execute thread num | `100` | +| `master.configmap.MASTER_EXEC_TASK_NUM` | Master execute task number in parallel | `20` | +| `master.configmap.MASTER_HEARTBEAT_INTERVAL` | Master heartbeat interval | `10` | +| `master.configmap.MASTER_TASK_COMMIT_RETRYTIMES` | Master commit task retry times | `5` | +| `master.configmap.MASTER_TASK_COMMIT_INTERVAL` | Master commit task interval | `1000` | +| `master.configmap.MASTER_MAX_CPULOAD_AVG` | Only less than cpu avg load, master server can work. default value : the number of cpu cores * 2 | `100` | +| `master.configmap.MASTER_RESERVED_MEMORY` | Only larger than reserved memory, master server can work. default value : physical memory * 1/10, unit is G | `0.1` | +| `master.livenessProbe.enabled` | Turn on and off liveness probe | `true` | +| `master.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `30` | +| `master.livenessProbe.periodSeconds` | How often to perform the probe | `30` | +| `master.livenessProbe.timeoutSeconds` | When the probe times out | `5` | +| `master.livenessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` | +| `master.livenessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` | +| `master.readinessProbe.enabled` | Turn on and off readiness probe | `true` | +| `master.readinessProbe.initialDelaySeconds` | Delay before readiness probe is initiated | `30` | +| `master.readinessProbe.periodSeconds` | How often to perform the probe | `30` | +| `master.readinessProbe.timeoutSeconds` | When the probe times out | `5` | +| `master.readinessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` | +| `master.readinessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` | +| `master.persistentVolumeClaim.enabled` | Set `master.persistentVolumeClaim.enabled` to `true` to mount a new volume for `master` | `false` | +| `master.persistentVolumeClaim.accessModes` | `PersistentVolumeClaim` Access Modes | `[ReadWriteOnce]` | +| `master.persistentVolumeClaim.storageClassName` | `Master` logs data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` | +| `master.persistentVolumeClaim.storage` | `PersistentVolumeClaim` Size | `20Gi` | +| | | | +| `worker.podManagementPolicy` | PodManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down | `Parallel` | +| `worker.replicas` | Replicas is the desired number of replicas of the given Template | `3` | +| `worker.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` | +| `worker.tolerations` | If specified, the pod's tolerations | `{}` | +| `worker.affinity` | If specified, the pod's scheduling constraints | `{}` | +| `worker.configmap.WORKER_EXEC_THREADS` | Worker execute thread num | `100` | +| `worker.configmap.WORKER_HEARTBEAT_INTERVAL` | Worker heartbeat interval | `10` | +| `worker.configmap.WORKER_FETCH_TASK_NUM` | Submit the number of tasks at a time | `3` | +| `worker.configmap.WORKER_MAX_CPULOAD_AVG` | Only less than cpu avg load, worker server can work. default value : the number of cpu cores * 2 | `100` | +| `worker.configmap.WORKER_RESERVED_MEMORY` | Only larger than reserved memory, worker server can work. default value : physical memory * 1/10, unit is G | `0.1` | +| `worker.configmap.DOLPHINSCHEDULER_DATA_BASEDIR_PATH` | User data directory path, self configuration, please make sure the directory exists and have read write permissions | `/tmp/dolphinscheduler` | +| `worker.configmap.DOLPHINSCHEDULER_ENV` | System env path, self configuration, please read `values.yaml` | `[]` | +| `worker.livenessProbe.enabled` | Turn on and off liveness probe | `true` | +| `worker.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `30` | +| `worker.livenessProbe.periodSeconds` | How often to perform the probe | `30` | +| `worker.livenessProbe.timeoutSeconds` | When the probe times out | `5` | +| `worker.livenessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` | +| `worker.livenessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` | +| `worker.readinessProbe.enabled` | Turn on and off readiness probe | `true` | +| `worker.readinessProbe.initialDelaySeconds` | Delay before readiness probe is initiated | `30` | +| `worker.readinessProbe.periodSeconds` | How often to perform the probe | `30` | +| `worker.readinessProbe.timeoutSeconds` | When the probe times out | `5` | +| `worker.readinessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` | +| `worker.readinessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` | +| `worker.persistentVolumeClaim.enabled` | Set `worker.persistentVolumeClaim.enabled` to `true` to enable `persistentVolumeClaim` for `worker` | `false` | +| `worker.persistentVolumeClaim.dataPersistentVolume.enabled` | Set `worker.persistentVolumeClaim.dataPersistentVolume.enabled` to `true` to mount a data volume for `worker` | `false` | +| `worker.persistentVolumeClaim.dataPersistentVolume.accessModes` | `PersistentVolumeClaim` Access Modes | `[ReadWriteOnce]` | +| `worker.persistentVolumeClaim.dataPersistentVolume.storageClassName` | `Worker` data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` | +| `worker.persistentVolumeClaim.dataPersistentVolume.storage` | `PersistentVolumeClaim` Size | `20Gi` | +| `worker.persistentVolumeClaim.logsPersistentVolume.enabled` | Set `worker.persistentVolumeClaim.logsPersistentVolume.enabled` to `true` to mount a logs volume for `worker` | `false` | +| `worker.persistentVolumeClaim.logsPersistentVolume.accessModes` | `PersistentVolumeClaim` Access Modes | `[ReadWriteOnce]` | +| `worker.persistentVolumeClaim.logsPersistentVolume.storageClassName` | `Worker` logs data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` | +| `worker.persistentVolumeClaim.logsPersistentVolume.storage` | `PersistentVolumeClaim` Size | `20Gi` | +| | | | +| `alert.strategy.type` | Type of deployment. Can be "Recreate" or "RollingUpdate" | `RollingUpdate` | +| `alert.strategy.rollingUpdate.maxSurge` | The maximum number of pods that can be scheduled above the desired number of pods | `25%` | +| `alert.strategy.rollingUpdate.maxUnavailable` | The maximum number of pods that can be unavailable during the update | `25%` | +| `alert.replicas` | Replicas is the desired number of replicas of the given Template | `1` | +| `alert.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` | +| `alert.tolerations` | If specified, the pod's tolerations | `{}` | +| `alert.affinity` | If specified, the pod's scheduling constraints | `{}` | +| `alert.configmap.XLS_FILE_PATH` | XLS file path | `/tmp/xls` | +| `alert.configmap.MAIL_SERVER_HOST` | Mail `SERVER HOST ` | `nil` | +| `alert.configmap.MAIL_SERVER_PORT` | Mail `SERVER PORT` | `nil` | +| `alert.configmap.MAIL_SENDER` | Mail `SENDER` | `nil` | +| `alert.configmap.MAIL_USER` | Mail `USER` | `nil` | +| `alert.configmap.MAIL_PASSWD` | Mail `PASSWORD` | `nil` | +| `alert.configmap.MAIL_SMTP_STARTTLS_ENABLE` | Mail `SMTP STARTTLS` enable | `false` | +| `alert.configmap.MAIL_SMTP_SSL_ENABLE` | Mail `SMTP SSL` enable | `false` | +| `alert.configmap.MAIL_SMTP_SSL_TRUST` | Mail `SMTP SSL TRUST` | `nil` | +| `alert.configmap.ENTERPRISE_WECHAT_ENABLE` | `Enterprise Wechat` enable | `false` | +| `alert.configmap.ENTERPRISE_WECHAT_CORP_ID` | `Enterprise Wechat` corp id | `nil` | +| `alert.configmap.ENTERPRISE_WECHAT_SECRET` | `Enterprise Wechat` secret | `nil` | +| `alert.configmap.ENTERPRISE_WECHAT_AGENT_ID` | `Enterprise Wechat` agent id | `nil` | +| `alert.configmap.ENTERPRISE_WECHAT_USERS` | `Enterprise Wechat` users | `nil` | +| `alert.livenessProbe.enabled` | Turn on and off liveness probe | `true` | +| `alert.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `30` | +| `alert.livenessProbe.periodSeconds` | How often to perform the probe | `30` | +| `alert.livenessProbe.timeoutSeconds` | When the probe times out | `5` | +| `alert.livenessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` | +| `alert.livenessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` | +| `alert.readinessProbe.enabled` | Turn on and off readiness probe | `true` | +| `alert.readinessProbe.initialDelaySeconds` | Delay before readiness probe is initiated | `30` | +| `alert.readinessProbe.periodSeconds` | How often to perform the probe | `30` | +| `alert.readinessProbe.timeoutSeconds` | When the probe times out | `5` | +| `alert.readinessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` | +| `alert.readinessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` | +| `alert.persistentVolumeClaim.enabled` | Set `alert.persistentVolumeClaim.enabled` to `true` to mount a new volume for `alert` | `false` | +| `alert.persistentVolumeClaim.accessModes` | `PersistentVolumeClaim` Access Modes | `[ReadWriteOnce]` | +| `alert.persistentVolumeClaim.storageClassName` | `Alert` logs data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` | +| `alert.persistentVolumeClaim.storage` | `PersistentVolumeClaim` Size | `20Gi` | +| | | | +| `api.strategy.type` | Type of deployment. Can be "Recreate" or "RollingUpdate" | `RollingUpdate` | +| `api.strategy.rollingUpdate.maxSurge` | The maximum number of pods that can be scheduled above the desired number of pods | `25%` | +| `api.strategy.rollingUpdate.maxUnavailable` | The maximum number of pods that can be unavailable during the update | `25%` | +| `api.replicas` | Replicas is the desired number of replicas of the given Template | `1` | +| `api.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` | +| `api.tolerations` | If specified, the pod's tolerations | `{}` | +| `api.affinity` | If specified, the pod's scheduling constraints | `{}` | +| `api.livenessProbe.enabled` | Turn on and off liveness probe | `true` | +| `api.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `30` | +| `api.livenessProbe.periodSeconds` | How often to perform the probe | `30` | +| `api.livenessProbe.timeoutSeconds` | When the probe times out | `5` | +| `api.livenessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` | +| `api.livenessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` | +| `api.readinessProbe.enabled` | Turn on and off readiness probe | `true` | +| `api.readinessProbe.initialDelaySeconds` | Delay before readiness probe is initiated | `30` | +| `api.readinessProbe.periodSeconds` | How often to perform the probe | `30` | +| `api.readinessProbe.timeoutSeconds` | When the probe times out | `5` | +| `api.readinessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` | +| `api.readinessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` | +| `api.persistentVolumeClaim.enabled` | Set `api.persistentVolumeClaim.enabled` to `true` to mount a new volume for `api` | `false` | +| `api.persistentVolumeClaim.accessModes` | `PersistentVolumeClaim` Access Modes | `[ReadWriteOnce]` | +| `api.persistentVolumeClaim.storageClassName` | `api` logs data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` | +| `api.persistentVolumeClaim.storage` | `PersistentVolumeClaim` Size | `20Gi` | +| | | | +| `frontend.strategy.type` | Type of deployment. Can be "Recreate" or "RollingUpdate" | `RollingUpdate` | +| `frontend.strategy.rollingUpdate.maxSurge` | The maximum number of pods that can be scheduled above the desired number of pods | `25%` | +| `frontend.strategy.rollingUpdate.maxUnavailable` | The maximum number of pods that can be unavailable during the update | `25%` | +| `frontend.replicas` | Replicas is the desired number of replicas of the given Template | `1` | +| `frontend.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` | +| `frontend.tolerations` | If specified, the pod's tolerations | `{}` | +| `frontend.affinity` | If specified, the pod's scheduling constraints | `{}` | +| `frontend.livenessProbe.enabled` | Turn on and off liveness probe | `true` | +| `frontend.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `30` | +| `frontend.livenessProbe.periodSeconds` | How often to perform the probe | `30` | +| `frontend.livenessProbe.timeoutSeconds` | When the probe times out | `5` | +| `frontend.livenessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` | +| `frontend.livenessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` | +| `frontend.readinessProbe.enabled` | Turn on and off readiness probe | `true` | +| `frontend.readinessProbe.initialDelaySeconds` | Delay before readiness probe is initiated | `30` | +| `frontend.readinessProbe.periodSeconds` | How often to perform the probe | `30` | +| `frontend.readinessProbe.timeoutSeconds` | When the probe times out | `5` | +| `frontend.readinessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` | +| `frontend.readinessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` | +| `frontend.persistentVolumeClaim.enabled` | Set `frontend.persistentVolumeClaim.enabled` to `true` to mount a new volume for `frontend` | `false` | +| `frontend.persistentVolumeClaim.accessModes` | `PersistentVolumeClaim` Access Modes | `[ReadWriteOnce]` | +| `frontend.persistentVolumeClaim.storageClassName` | `frontend` logs data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` | +| `frontend.persistentVolumeClaim.storage` | `PersistentVolumeClaim` Size | `20Gi` | +| | | | +| `ingress.enabled` | Enable ingress | `false` | +| `ingress.host` | Ingress host | `dolphinscheduler.org` | +| `ingress.path` | Ingress path | `/` | +| `ingress.tls.enabled` | Enable ingress tls | `false` | +| `ingress.tls.hosts` | Ingress tls hosts | `dolphinscheduler.org` | +| `ingress.tls.secretName` | Ingress tls secret name | `dolphinscheduler-tls` | + +For more information please refer to the [chart](https://github.com/apache/incubator-dolphinscheduler.git) documentation. diff --git a/charts/dolphinscheduler/Chart.yaml b/charts/dolphinscheduler/Chart.yaml new file mode 100644 index 0000000000..2c40f94d3c --- /dev/null +++ b/charts/dolphinscheduler/Chart.yaml @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: v2 +name: dolphinscheduler +description: Dolphin Scheduler is a distributed and easy-to-expand visual DAG workflow scheduling system, dedicated to solving the complex dependencies in data processing, making the scheduling system out of the box for data processing. +home: https://dolphinscheduler.apache.org +icon: https://dolphinscheduler.apache.org/img/hlogo_colorful.svg +keywords: + - dolphinscheduler + - Scheduler +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. +appVersion: 1.2.1 + +dependencies: + - name: postgresql + version: 8.x.x + repository: https://charts.bitnami.com/bitnami + condition: postgresql.enabled + - name: zookeeper + version: 5.x.x + repository: https://charts.bitnami.com/bitnami + condition: redis.enabled diff --git a/charts/dolphinscheduler/README.md b/charts/dolphinscheduler/README.md new file mode 100644 index 0000000000..6f0317b9e2 --- /dev/null +++ b/charts/dolphinscheduler/README.md @@ -0,0 +1,226 @@ +# Dolphin Scheduler + +[Dolphin Scheduler](https://dolphinscheduler.apache.org) is a distributed and easy-to-expand visual DAG workflow scheduling system, dedicated to solving the complex dependencies in data processing, making the scheduling system out of the box for data processing. + +## Introduction +This chart bootstraps a [Dolphin Scheduler](https://dolphinscheduler.apache.org) distributed deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +## Prerequisites + +- Kubernetes 1.10+ +- PV provisioner support in the underlying infrastructure + +## Installing the Chart + +To install the chart with the release name `my-release`: + +```bash +$ git clone https://github.com/apache/incubator-dolphinscheduler.git +$ cd incubator-dolphinscheduler +$ helm install --name dolphinscheduler . +``` +These commands deploy Dolphin Scheduler on the Kubernetes cluster in the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation. + +> **Tip**: List all releases using `helm list` + +## Uninstalling the Chart + +To uninstall/delete the `dolphinscheduler` deployment: + +```bash +$ helm delete --purge dolphinscheduler +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +## Configuration + +The following tables lists the configurable parameters of the Dolphins Scheduler chart and their default values. + +| Parameter | Description | Default | +| --------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------- | +| `timezone` | World time and date for cities in all time zones | `Asia/Shanghai` | +| `image.registry` | Docker image registry for the Dolphins Scheduler | `docker.io` | +| `image.repository` | Docker image repository for the Dolphins Scheduler | `dolphinscheduler` | +| `image.tag` | Docker image version for the Dolphins Scheduler | `1.2.1` | +| `image.imagePullPolicy` | Image pull policy. One of Always, Never, IfNotPresent | `IfNotPresent` | +| `imagePullSecrets` | ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images | `[]` | +| | | | +| `postgresql.enabled` | If not exists external PostgreSQL, by default, the Dolphins Scheduler will use a internal PostgreSQL | `true` | +| `postgresql.postgresqlUsername` | The username for internal PostgreSQL | `root` | +| `postgresql.postgresqlPassword` | The password for internal PostgreSQL | `root` | +| `postgresql.postgresqlDatabase` | The database for internal PostgreSQL | `dolphinscheduler` | +| `postgresql.persistence.enabled` | Set `postgresql.persistence.enabled` to `true` to mount a new volume for internal PostgreSQL | `false` | +| `postgresql.persistence.size` | `PersistentVolumeClaim` Size | `20Gi` | +| `postgresql.persistence.storageClass` | PostgreSQL data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` | +| `externalDatabase.host` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database host will use it. | `localhost` | +| `externalDatabase.port` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database port will use it. | `5432` | +| `externalDatabase.username` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database username will use it. | `root` | +| `externalDatabase.password` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database password will use it. | `root` | +| `externalDatabase.database` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database database will use it. | `dolphinscheduler` | +| | | | +| `zookeeper.enabled` | If not exists external Zookeeper, by default, the Dolphin Scheduler will use a internal Zookeeper | `true` | +| `zookeeper.taskQueue` | Specify task queue for `master` and `worker` | `zookeeper` | +| `zookeeper.persistence.enabled` | Set `zookeeper.persistence.enabled` to `true` to mount a new volume for internal Zookeeper | `false` | +| `zookeeper.persistence.size` | `PersistentVolumeClaim` Size | `20Gi` | +| `zookeeper.persistence.storageClass` | Zookeeper data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` | +| `externalZookeeper.taskQueue` | If exists external Zookeeper, and set `zookeeper.enable` value to false. Specify task queue for `master` and `worker` | `zookeeper` | +| `externalZookeeper.zookeeperQuorum` | If exists external Zookeeper, and set `zookeeper.enable` value to false. Specify Zookeeper quorum | `127.0.0.1:2181` | +| | | | +| `master.podManagementPolicy` | PodManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down | `Parallel` | +| `master.replicas` | Replicas is the desired number of replicas of the given Template | `3` | +| `master.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` | +| `master.tolerations` | If specified, the pod's tolerations | `{}` | +| `master.affinity` | If specified, the pod's scheduling constraints | `{}` | +| `master.configmap.MASTER_EXEC_THREADS` | Master execute thread num | `100` | +| `master.configmap.MASTER_EXEC_TASK_NUM` | Master execute task number in parallel | `20` | +| `master.configmap.MASTER_HEARTBEAT_INTERVAL` | Master heartbeat interval | `10` | +| `master.configmap.MASTER_TASK_COMMIT_RETRYTIMES` | Master commit task retry times | `5` | +| `master.configmap.MASTER_TASK_COMMIT_INTERVAL` | Master commit task interval | `1000` | +| `master.configmap.MASTER_MAX_CPULOAD_AVG` | Only less than cpu avg load, master server can work. default value : the number of cpu cores * 2 | `100` | +| `master.configmap.MASTER_RESERVED_MEMORY` | Only larger than reserved memory, master server can work. default value : physical memory * 1/10, unit is G | `0.1` | +| `master.livenessProbe.enabled` | Turn on and off liveness probe | `true` | +| `master.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `30` | +| `master.livenessProbe.periodSeconds` | How often to perform the probe | `30` | +| `master.livenessProbe.timeoutSeconds` | When the probe times out | `5` | +| `master.livenessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` | +| `master.livenessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` | +| `master.readinessProbe.enabled` | Turn on and off readiness probe | `true` | +| `master.readinessProbe.initialDelaySeconds` | Delay before readiness probe is initiated | `30` | +| `master.readinessProbe.periodSeconds` | How often to perform the probe | `30` | +| `master.readinessProbe.timeoutSeconds` | When the probe times out | `5` | +| `master.readinessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` | +| `master.readinessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` | +| `master.persistentVolumeClaim.enabled` | Set `master.persistentVolumeClaim.enabled` to `true` to mount a new volume for `master` | `false` | +| `master.persistentVolumeClaim.accessModes` | `PersistentVolumeClaim` Access Modes | `[ReadWriteOnce]` | +| `master.persistentVolumeClaim.storageClassName` | `Master` logs data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` | +| `master.persistentVolumeClaim.storage` | `PersistentVolumeClaim` Size | `20Gi` | +| | | | +| `worker.podManagementPolicy` | PodManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down | `Parallel` | +| `worker.replicas` | Replicas is the desired number of replicas of the given Template | `3` | +| `worker.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` | +| `worker.tolerations` | If specified, the pod's tolerations | `{}` | +| `worker.affinity` | If specified, the pod's scheduling constraints | `{}` | +| `worker.configmap.WORKER_EXEC_THREADS` | Worker execute thread num | `100` | +| `worker.configmap.WORKER_HEARTBEAT_INTERVAL` | Worker heartbeat interval | `10` | +| `worker.configmap.WORKER_FETCH_TASK_NUM` | Submit the number of tasks at a time | `3` | +| `worker.configmap.WORKER_MAX_CPULOAD_AVG` | Only less than cpu avg load, worker server can work. default value : the number of cpu cores * 2 | `100` | +| `worker.configmap.WORKER_RESERVED_MEMORY` | Only larger than reserved memory, worker server can work. default value : physical memory * 1/10, unit is G | `0.1` | +| `worker.configmap.DOLPHINSCHEDULER_DATA_BASEDIR_PATH` | User data directory path, self configuration, please make sure the directory exists and have read write permissions | `/tmp/dolphinscheduler` | +| `worker.configmap.DOLPHINSCHEDULER_ENV` | System env path, self configuration, please read `values.yaml` | `[]` | +| `worker.livenessProbe.enabled` | Turn on and off liveness probe | `true` | +| `worker.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `30` | +| `worker.livenessProbe.periodSeconds` | How often to perform the probe | `30` | +| `worker.livenessProbe.timeoutSeconds` | When the probe times out | `5` | +| `worker.livenessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` | +| `worker.livenessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` | +| `worker.readinessProbe.enabled` | Turn on and off readiness probe | `true` | +| `worker.readinessProbe.initialDelaySeconds` | Delay before readiness probe is initiated | `30` | +| `worker.readinessProbe.periodSeconds` | How often to perform the probe | `30` | +| `worker.readinessProbe.timeoutSeconds` | When the probe times out | `5` | +| `worker.readinessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` | +| `worker.readinessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` | +| `worker.persistentVolumeClaim.enabled` | Set `worker.persistentVolumeClaim.enabled` to `true` to enable `persistentVolumeClaim` for `worker` | `false` | +| `worker.persistentVolumeClaim.dataPersistentVolume.enabled` | Set `worker.persistentVolumeClaim.dataPersistentVolume.enabled` to `true` to mount a data volume for `worker` | `false` | +| `worker.persistentVolumeClaim.dataPersistentVolume.accessModes` | `PersistentVolumeClaim` Access Modes | `[ReadWriteOnce]` | +| `worker.persistentVolumeClaim.dataPersistentVolume.storageClassName` | `Worker` data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` | +| `worker.persistentVolumeClaim.dataPersistentVolume.storage` | `PersistentVolumeClaim` Size | `20Gi` | +| `worker.persistentVolumeClaim.logsPersistentVolume.enabled` | Set `worker.persistentVolumeClaim.logsPersistentVolume.enabled` to `true` to mount a logs volume for `worker` | `false` | +| `worker.persistentVolumeClaim.logsPersistentVolume.accessModes` | `PersistentVolumeClaim` Access Modes | `[ReadWriteOnce]` | +| `worker.persistentVolumeClaim.logsPersistentVolume.storageClassName` | `Worker` logs data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` | +| `worker.persistentVolumeClaim.logsPersistentVolume.storage` | `PersistentVolumeClaim` Size | `20Gi` | +| | | | +| `alert.strategy.type` | Type of deployment. Can be "Recreate" or "RollingUpdate" | `RollingUpdate` | +| `alert.strategy.rollingUpdate.maxSurge` | The maximum number of pods that can be scheduled above the desired number of pods | `25%` | +| `alert.strategy.rollingUpdate.maxUnavailable` | The maximum number of pods that can be unavailable during the update | `25%` | +| `alert.replicas` | Replicas is the desired number of replicas of the given Template | `1` | +| `alert.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` | +| `alert.tolerations` | If specified, the pod's tolerations | `{}` | +| `alert.affinity` | If specified, the pod's scheduling constraints | `{}` | +| `alert.configmap.XLS_FILE_PATH` | XLS file path | `/tmp/xls` | +| `alert.configmap.MAIL_SERVER_HOST` | Mail `SERVER HOST ` | `nil` | +| `alert.configmap.MAIL_SERVER_PORT` | Mail `SERVER PORT` | `nil` | +| `alert.configmap.MAIL_SENDER` | Mail `SENDER` | `nil` | +| `alert.configmap.MAIL_USER` | Mail `USER` | `nil` | +| `alert.configmap.MAIL_PASSWD` | Mail `PASSWORD` | `nil` | +| `alert.configmap.MAIL_SMTP_STARTTLS_ENABLE` | Mail `SMTP STARTTLS` enable | `false` | +| `alert.configmap.MAIL_SMTP_SSL_ENABLE` | Mail `SMTP SSL` enable | `false` | +| `alert.configmap.MAIL_SMTP_SSL_TRUST` | Mail `SMTP SSL TRUST` | `nil` | +| `alert.configmap.ENTERPRISE_WECHAT_ENABLE` | `Enterprise Wechat` enable | `false` | +| `alert.configmap.ENTERPRISE_WECHAT_CORP_ID` | `Enterprise Wechat` corp id | `nil` | +| `alert.configmap.ENTERPRISE_WECHAT_SECRET` | `Enterprise Wechat` secret | `nil` | +| `alert.configmap.ENTERPRISE_WECHAT_AGENT_ID` | `Enterprise Wechat` agent id | `nil` | +| `alert.configmap.ENTERPRISE_WECHAT_USERS` | `Enterprise Wechat` users | `nil` | +| `alert.livenessProbe.enabled` | Turn on and off liveness probe | `true` | +| `alert.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `30` | +| `alert.livenessProbe.periodSeconds` | How often to perform the probe | `30` | +| `alert.livenessProbe.timeoutSeconds` | When the probe times out | `5` | +| `alert.livenessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` | +| `alert.livenessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` | +| `alert.readinessProbe.enabled` | Turn on and off readiness probe | `true` | +| `alert.readinessProbe.initialDelaySeconds` | Delay before readiness probe is initiated | `30` | +| `alert.readinessProbe.periodSeconds` | How often to perform the probe | `30` | +| `alert.readinessProbe.timeoutSeconds` | When the probe times out | `5` | +| `alert.readinessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` | +| `alert.readinessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` | +| `alert.persistentVolumeClaim.enabled` | Set `alert.persistentVolumeClaim.enabled` to `true` to mount a new volume for `alert` | `false` | +| `alert.persistentVolumeClaim.accessModes` | `PersistentVolumeClaim` Access Modes | `[ReadWriteOnce]` | +| `alert.persistentVolumeClaim.storageClassName` | `Alert` logs data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` | +| `alert.persistentVolumeClaim.storage` | `PersistentVolumeClaim` Size | `20Gi` | +| | | | +| `api.strategy.type` | Type of deployment. Can be "Recreate" or "RollingUpdate" | `RollingUpdate` | +| `api.strategy.rollingUpdate.maxSurge` | The maximum number of pods that can be scheduled above the desired number of pods | `25%` | +| `api.strategy.rollingUpdate.maxUnavailable` | The maximum number of pods that can be unavailable during the update | `25%` | +| `api.replicas` | Replicas is the desired number of replicas of the given Template | `1` | +| `api.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` | +| `api.tolerations` | If specified, the pod's tolerations | `{}` | +| `api.affinity` | If specified, the pod's scheduling constraints | `{}` | +| `api.livenessProbe.enabled` | Turn on and off liveness probe | `true` | +| `api.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `30` | +| `api.livenessProbe.periodSeconds` | How often to perform the probe | `30` | +| `api.livenessProbe.timeoutSeconds` | When the probe times out | `5` | +| `api.livenessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` | +| `api.livenessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` | +| `api.readinessProbe.enabled` | Turn on and off readiness probe | `true` | +| `api.readinessProbe.initialDelaySeconds` | Delay before readiness probe is initiated | `30` | +| `api.readinessProbe.periodSeconds` | How often to perform the probe | `30` | +| `api.readinessProbe.timeoutSeconds` | When the probe times out | `5` | +| `api.readinessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` | +| `api.readinessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` | +| `api.persistentVolumeClaim.enabled` | Set `api.persistentVolumeClaim.enabled` to `true` to mount a new volume for `api` | `false` | +| `api.persistentVolumeClaim.accessModes` | `PersistentVolumeClaim` Access Modes | `[ReadWriteOnce]` | +| `api.persistentVolumeClaim.storageClassName` | `api` logs data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` | +| `api.persistentVolumeClaim.storage` | `PersistentVolumeClaim` Size | `20Gi` | +| | | | +| `frontend.strategy.type` | Type of deployment. Can be "Recreate" or "RollingUpdate" | `RollingUpdate` | +| `frontend.strategy.rollingUpdate.maxSurge` | The maximum number of pods that can be scheduled above the desired number of pods | `25%` | +| `frontend.strategy.rollingUpdate.maxUnavailable` | The maximum number of pods that can be unavailable during the update | `25%` | +| `frontend.replicas` | Replicas is the desired number of replicas of the given Template | `1` | +| `frontend.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` | +| `frontend.tolerations` | If specified, the pod's tolerations | `{}` | +| `frontend.affinity` | If specified, the pod's scheduling constraints | `{}` | +| `frontend.livenessProbe.enabled` | Turn on and off liveness probe | `true` | +| `frontend.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `30` | +| `frontend.livenessProbe.periodSeconds` | How often to perform the probe | `30` | +| `frontend.livenessProbe.timeoutSeconds` | When the probe times out | `5` | +| `frontend.livenessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` | +| `frontend.livenessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` | +| `frontend.readinessProbe.enabled` | Turn on and off readiness probe | `true` | +| `frontend.readinessProbe.initialDelaySeconds` | Delay before readiness probe is initiated | `30` | +| `frontend.readinessProbe.periodSeconds` | How often to perform the probe | `30` | +| `frontend.readinessProbe.timeoutSeconds` | When the probe times out | `5` | +| `frontend.readinessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` | +| `frontend.readinessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` | +| `frontend.persistentVolumeClaim.enabled` | Set `frontend.persistentVolumeClaim.enabled` to `true` to mount a new volume for `frontend` | `false` | +| `frontend.persistentVolumeClaim.accessModes` | `PersistentVolumeClaim` Access Modes | `[ReadWriteOnce]` | +| `frontend.persistentVolumeClaim.storageClassName` | `frontend` logs data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` | +| `frontend.persistentVolumeClaim.storage` | `PersistentVolumeClaim` Size | `20Gi` | +| | | | +| `ingress.enabled` | Enable ingress | `false` | +| `ingress.host` | Ingress host | `dolphinscheduler.org` | +| `ingress.path` | Ingress path | `/` | +| `ingress.tls.enabled` | Enable ingress tls | `false` | +| `ingress.tls.hosts` | Ingress tls hosts | `dolphinscheduler.org` | +| `ingress.tls.secretName` | Ingress tls secret name | `dolphinscheduler-tls` | + +For more information please refer to the [chart](https://github.com/apache/incubator-dolphinscheduler.git) documentation. diff --git a/charts/dolphinscheduler/templates/NOTES.txt b/charts/dolphinscheduler/templates/NOTES.txt new file mode 100644 index 0000000000..eb3a9cfc52 --- /dev/null +++ b/charts/dolphinscheduler/templates/NOTES.txt @@ -0,0 +1,44 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +** Please be patient while the chart is being deployed ** + +1. Get the Dolphinscheduler URL by running: + +{{- if .Values.ingress.enabled }} + + export HOSTNAME=$(kubectl get ingress --namespace {{ .Release.Namespace }} {{ template "dolphinscheduler.fullname" . }} -o jsonpath='{.spec.rules[0].host}') + echo "Dolphinscheduler URL: http://$HOSTNAME/" + +{{- else }} + + kubectl port-forward --namespace {{ .Release.Namespace }} svc/{{ template "dolphinscheduler.fullname" . }}-frontend 8888:8888 + +{{- end }} + +2. Get the Dolphinscheduler URL by running: + +{{- if .Values.ingress.enabled }} + + export HOSTNAME=$(kubectl get ingress --namespace {{ .Release.Namespace }} {{ template "dolphinscheduler.fullname" . }} -o jsonpath='{.spec.rules[0].host}') + echo "Dolphinscheduler URL: http://$HOSTNAME/" + +{{- else }} + + kubectl port-forward --namespace {{ .Release.Namespace }} svc/{{ template "dolphinscheduler.fullname" . }}-frontend 8888:8888 + +{{- end }} \ No newline at end of file diff --git a/charts/dolphinscheduler/templates/_helpers.tpl b/charts/dolphinscheduler/templates/_helpers.tpl new file mode 100644 index 0000000000..37fb034128 --- /dev/null +++ b/charts/dolphinscheduler/templates/_helpers.tpl @@ -0,0 +1,149 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "dolphinscheduler.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "dolphinscheduler.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "dolphinscheduler.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "dolphinscheduler.labels" -}} +helm.sh/chart: {{ include "dolphinscheduler.chart" . }} +{{ include "dolphinscheduler.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Selector labels +*/}} +{{- define "dolphinscheduler.selectorLabels" -}} +app.kubernetes.io/name: {{ include "dolphinscheduler.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "dolphinscheduler.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "dolphinscheduler.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Create a default docker image registry. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "dolphinscheduler.image.registry" -}} +{{- $registry := default "docker.io" .Values.image.registry -}} +{{- printf "%s" $registry | trunc 63 | trimSuffix "/" -}} +{{- end -}} + +{{/* +Create a default docker image repository. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "dolphinscheduler.image.repository" -}} +{{- printf "%s/%s:%s" (include "dolphinscheduler.image.registry" .) .Values.image.repository .Values.image.tag -}} +{{- end -}} + +{{/* +Create a default fully qualified postgresql name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "dolphinscheduler.postgresql.fullname" -}} +{{- $name := default "postgresql" .Values.postgresql.nameOverride -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified zookkeeper name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "dolphinscheduler.zookeeper.fullname" -}} +{{- $name := default "zookeeper" .Values.zookeeper.nameOverride -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified zookkeeper quorum. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "dolphinscheduler.zookeeper.quorum" -}} +{{- $port := default "2181" (.Values.zookeeper.service.port | toString) -}} +{{- printf "%s:%s" (include "dolphinscheduler.zookeeper.fullname" .) $port | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default dolphinscheduler worker base dir. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "dolphinscheduler.worker.base.dir" -}} +{{- $name := default "/tmp/dolphinscheduler" .Values.worker.configmap.DOLPHINSCHEDULER_DATA_BASEDIR_PATH -}} +{{- printf "%s" $name | trunc 63 | trimSuffix "/" -}} +{{- end -}} + +{{/* +Create a default dolphinscheduler worker data download dir. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "dolphinscheduler.worker.data.download.dir" -}} +{{- printf "%s%s" (include "dolphinscheduler.worker.base.dir" .) "/download" -}} +{{- end -}} + +{{/* +Create a default dolphinscheduler worker process exec dir. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "dolphinscheduler.worker.process.exec.dir" -}} +{{- printf "%s%s" (include "dolphinscheduler.worker.base.dir" .) "/exec" -}} +{{- end -}} \ No newline at end of file diff --git a/charts/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml b/charts/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml new file mode 100644 index 0000000000..76daad8568 --- /dev/null +++ b/charts/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml @@ -0,0 +1,41 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +{{- if .Values.alert.configmap }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "dolphinscheduler.fullname" . }}-alert + labels: + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-alert + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +data: + XLS_FILE_PATH: {{ .Values.alert.configmap.XLS_FILE_PATH | quote }} + MAIL_SERVER_HOST: {{ .Values.alert.configmap.MAIL_SERVER_HOST | quote }} + MAIL_SERVER_PORT: {{ .Values.alert.configmap.MAIL_SERVER_PORT | quote }} + MAIL_SENDER: {{ .Values.alert.configmap.MAIL_SENDER | quote }} + MAIL_USER: {{ .Values.alert.configmap.MAIL_USER | quote }} + MAIL_PASSWD: {{ .Values.alert.configmap.MAIL_PASSWD | quote }} + MAIL_SMTP_STARTTLS_ENABLE: {{ .Values.alert.configmap.MAIL_SMTP_STARTTLS_ENABLE | quote }} + MAIL_SMTP_SSL_ENABLE: {{ .Values.alert.configmap.MAIL_SMTP_SSL_ENABLE | quote }} + MAIL_SMTP_SSL_TRUST: {{ .Values.alert.configmap.MAIL_SMTP_SSL_TRUST | quote }} + ENTERPRISE_WECHAT_ENABLE: {{ .Values.alert.configmap.ENTERPRISE_WECHAT_ENABLE | quote }} + ENTERPRISE_WECHAT_CORP_ID: {{ .Values.alert.configmap.ENTERPRISE_WECHAT_CORP_ID | quote }} + ENTERPRISE_WECHAT_SECRET: {{ .Values.alert.configmap.ENTERPRISE_WECHAT_SECRET | quote }} + ENTERPRISE_WECHAT_AGENT_ID: {{ .Values.alert.configmap.ENTERPRISE_WECHAT_AGENT_ID | quote }} + ENTERPRISE_WECHAT_USERS: {{ .Values.alert.configmap.ENTERPRISE_WECHAT_USERS | quote }} +{{- end }} \ No newline at end of file diff --git a/charts/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml b/charts/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml new file mode 100644 index 0000000000..8cce068276 --- /dev/null +++ b/charts/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +{{- if .Values.master.configmap }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "dolphinscheduler.fullname" . }}-master + labels: + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +data: + MASTER_EXEC_THREADS: {{ .Values.master.configmap.MASTER_EXEC_THREADS | quote }} + MASTER_EXEC_TASK_NUM: {{ .Values.master.configmap.MASTER_EXEC_TASK_NUM | quote }} + MASTER_HEARTBEAT_INTERVAL: {{ .Values.master.configmap.MASTER_HEARTBEAT_INTERVAL | quote }} + MASTER_TASK_COMMIT_RETRYTIMES: {{ .Values.master.configmap.MASTER_TASK_COMMIT_RETRYTIMES | quote }} + MASTER_TASK_COMMIT_INTERVAL: {{ .Values.master.configmap.MASTER_TASK_COMMIT_INTERVAL | quote }} + MASTER_MAX_CPULOAD_AVG: {{ .Values.master.configmap.MASTER_MAX_CPULOAD_AVG | quote }} + MASTER_RESERVED_MEMORY: {{ .Values.master.configmap.MASTER_RESERVED_MEMORY | quote }} +{{- end }} \ No newline at end of file diff --git a/charts/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml b/charts/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml new file mode 100644 index 0000000000..be7391fb32 --- /dev/null +++ b/charts/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml @@ -0,0 +1,39 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +{{- if .Values.worker.configmap }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "dolphinscheduler.fullname" . }}-worker + labels: + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +data: + WORKER_EXEC_THREADS: {{ .Values.worker.configmap.WORKER_EXEC_THREADS | quote }} + WORKER_HEARTBEAT_INTERVAL: {{ .Values.worker.configmap.WORKER_HEARTBEAT_INTERVAL | quote }} + WORKER_FETCH_TASK_NUM: {{ .Values.worker.configmap.WORKER_FETCH_TASK_NUM | quote }} + WORKER_MAX_CPULOAD_AVG: {{ .Values.worker.configmap.WORKER_MAX_CPULOAD_AVG | quote }} + WORKER_RESERVED_MEMORY: {{ .Values.worker.configmap.WORKER_RESERVED_MEMORY | quote }} + DOLPHINSCHEDULER_DATA_BASEDIR_PATH: {{ include "dolphinscheduler.worker.base.dir" . | quote }} + DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH: {{ include "dolphinscheduler.worker.data.download.dir" . | quote }} + DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH: {{ include "dolphinscheduler.worker.process.exec.dir" . | quote }} + dolphinscheduler_env.sh: |- + {{- range .Values.worker.configmap.DOLPHINSCHEDULER_ENV }} + {{ . }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml b/charts/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml new file mode 100644 index 0000000000..26026f74b3 --- /dev/null +++ b/charts/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml @@ -0,0 +1,228 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "dolphinscheduler.fullname" . }}-alert + labels: + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-alert + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/component: alert +spec: + replicas: {{ .Values.alert.replicas }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-alert + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/component: alert + strategy: + type: {{ .Values.alert.strategy.type | quote }} + rollingUpdate: + maxSurge: {{ .Values.alert.strategy.rollingUpdate.maxSurge | quote }} + maxUnavailable: {{ .Values.alert.strategy.rollingUpdate.maxUnavailable | quote }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-alert + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/component: alert + spec: + {{- if .Values.alert.affinity }} + affinity: {{- toYaml .Values.alert.affinity | nindent 8 }} + {{- end }} + {{- if .Values.alert.nodeSelector }} + nodeSelector: {{- toYaml .Values.alert.nodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.alert.tolerations }} + tolerations: {{- toYaml . | nindent 8 }} + {{- end }} + initContainers: + - name: init-postgresql + image: busybox:1.31.0 + command: + - /bin/sh + - -ec + - | + while ! nc -z ${POSTGRESQL_HOST} ${POSTGRESQL_PORT}; do + counter=$((counter+1)) + if [ $counter == 5 ]; then + echo "Error: Couldn't connect to postgresql." + exit 1 + fi + echo "Trying to connect to postgresql at ${POSTGRESQL_HOST}:${POSTGRESQL_PORT}. Attempt $counter." + sleep 60 + done + env: + - name: POSTGRESQL_HOST + {{- if .Values.postgresql.enabled }} + value: {{ template "dolphinscheduler.postgresql.fullname" . }} + {{- else }} + value: {{ .Values.externalDatabase.host | quote }} + {{- end }} + - name: POSTGRESQL_PORT + {{- if .Values.postgresql.enabled }} + value: "5432" + {{- else }} + value: {{ .Values.externalDatabase.port }} + {{- end }} + containers: + - name: {{ include "dolphinscheduler.fullname" . }}-alert + image: {{ include "dolphinscheduler.image.repository" . | quote }} + args: + - "alert-server" + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: TZ + value: {{ .Values.timezone }} + - name: XLS_FILE_PATH + valueFrom: + configMapKeyRef: + key: XLS_FILE_PATH + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_SERVER_HOST + valueFrom: + configMapKeyRef: + key: MAIL_SERVER_HOST + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_SERVER_PORT + valueFrom: + configMapKeyRef: + key: MAIL_SERVER_PORT + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_SENDER + valueFrom: + configMapKeyRef: + key: MAIL_SENDER + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_USER + valueFrom: + configMapKeyRef: + key: MAIL_USER + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_PASSWD + valueFrom: + configMapKeyRef: + key: MAIL_PASSWD + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_SMTP_STARTTLS_ENABLE + valueFrom: + configMapKeyRef: + key: MAIL_SMTP_STARTTLS_ENABLE + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_SMTP_SSL_ENABLE + valueFrom: + configMapKeyRef: + key: MAIL_SMTP_SSL_ENABLE + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_SMTP_SSL_TRUST + valueFrom: + configMapKeyRef: + key: MAIL_SMTP_SSL_TRUST + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: ENTERPRISE_WECHAT_ENABLE + valueFrom: + configMapKeyRef: + key: ENTERPRISE_WECHAT_ENABLE + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: ENTERPRISE_WECHAT_CORP_ID + valueFrom: + configMapKeyRef: + key: ENTERPRISE_WECHAT_CORP_ID + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: ENTERPRISE_WECHAT_SECRET + valueFrom: + configMapKeyRef: + key: ENTERPRISE_WECHAT_SECRET + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: ENTERPRISE_WECHAT_AGENT_ID + valueFrom: + configMapKeyRef: + key: ENTERPRISE_WECHAT_AGENT_ID + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: ENTERPRISE_WECHAT_USERS + valueFrom: + configMapKeyRef: + key: ENTERPRISE_WECHAT_USERS + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: POSTGRESQL_HOST + {{- if .Values.postgresql.enabled }} + value: {{ template "dolphinscheduler.postgresql.fullname" . }} + {{- else }} + value: {{ .Values.externalDatabase.host | quote }} + {{- end }} + - name: POSTGRESQL_PORT + {{- if .Values.postgresql.enabled }} + value: "5432" + {{- else }} + value: {{ .Values.externalDatabase.port }} + {{- end }} + - name: POSTGRESQL_USERNAME + {{- if .Values.postgresql.enabled }} + value: {{ .Values.postgresql.postgresqlUsername }} + {{- else }} + value: {{ .Values.externalDatabase.username | quote }} + {{- end }} + - name: POSTGRESQL_PASSWORD + valueFrom: + secretKeyRef: + {{- if .Values.postgresql.enabled }} + name: {{ template "dolphinscheduler.postgresql.fullname" . }} + key: postgresql-password + {{- else }} + name: {{ printf "%s-%s" .Release.Name "externaldb" }} + key: db-password + {{- end }} + {{- if .Values.alert.livenessProbe.enabled }} + livenessProbe: + exec: + command: + - sh + - /root/checkpoint.sh + - worker-server + initialDelaySeconds: {{ .Values.alert.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.alert.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.alert.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.alert.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.alert.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.alert.readinessProbe.enabled }} + readinessProbe: + exec: + command: + - sh + - /root/checkpoint.sh + - worker-server + initialDelaySeconds: {{ .Values.alert.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.alert.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.alert.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.alert.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.alert.readinessProbe.failureThreshold }} + {{- end }} + volumeMounts: + - mountPath: "/opt/dolphinscheduler/logs" + name: {{ include "dolphinscheduler.fullname" . }}-alert + volumes: + - name: {{ include "dolphinscheduler.fullname" . }}-alert + {{- if .Values.alert.persistentVolumeClaim.enabled }} + persistentVolumeClaim: + claimName: {{ include "dolphinscheduler.fullname" . }}-alert + {{- else }} + emptyDir: {} + {{- end }} \ No newline at end of file diff --git a/charts/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml b/charts/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml new file mode 100644 index 0000000000..926ce3c062 --- /dev/null +++ b/charts/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml @@ -0,0 +1,161 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "dolphinscheduler.fullname" . }}-api + labels: + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/component: api +spec: + replicas: {{ .Values.api.replicas }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/component: api + strategy: + type: {{ .Values.api.strategy.type | quote }} + rollingUpdate: + maxSurge: {{ .Values.api.strategy.rollingUpdate.maxSurge | quote }} + maxUnavailable: {{ .Values.api.strategy.rollingUpdate.maxUnavailable | quote }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/component: api + spec: + {{- if .Values.api.affinity }} + affinity: {{- toYaml .Values.api.affinity | nindent 8 }} + {{- end }} + {{- if .Values.api.nodeSelector }} + nodeSelector: {{- toYaml .Values.api.nodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.api.tolerations }} + tolerations: {{- toYaml . | nindent 8 }} + {{- end }} + initContainers: + - name: init-postgresql + image: busybox:1.31.0 + command: + - /bin/sh + - -ec + - | + while ! nc -z ${POSTGRESQL_HOST} ${POSTGRESQL_PORT}; do + counter=$((counter+1)) + if [ $counter == 5 ]; then + echo "Error: Couldn't connect to postgresql." + exit 1 + fi + echo "Trying to connect to postgresql at ${POSTGRESQL_HOST}:${POSTGRESQL_PORT}. Attempt $counter." + sleep 60 + done + env: + - name: POSTGRESQL_HOST + {{- if .Values.postgresql.enabled }} + value: {{ template "dolphinscheduler.postgresql.fullname" . }} + {{- else }} + value: {{ .Values.externalDatabase.host | quote }} + {{- end }} + - name: POSTGRESQL_PORT + {{- if .Values.postgresql.enabled }} + value: "5432" + {{- else }} + value: {{ .Values.externalDatabase.port }} + {{- end }} + containers: + - name: {{ include "dolphinscheduler.fullname" . }}-api + image: {{ include "dolphinscheduler.image.repository" . | quote }} + args: + - "api-server" + ports: + - containerPort: 12345 + name: tcp-port + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: TZ + value: {{ .Values.timezone }} + - name: POSTGRESQL_HOST + {{- if .Values.postgresql.enabled }} + value: {{ template "dolphinscheduler.postgresql.fullname" . }} + {{- else }} + value: {{ .Values.externalDatabase.host | quote }} + {{- end }} + - name: POSTGRESQL_PORT + {{- if .Values.postgresql.enabled }} + value: "5432" + {{- else }} + value: {{ .Values.externalDatabase.port }} + {{- end }} + - name: POSTGRESQL_USERNAME + {{- if .Values.postgresql.enabled }} + value: {{ .Values.postgresql.postgresqlUsername }} + {{- else }} + value: {{ .Values.externalDatabase.username | quote }} + {{- end }} + - name: POSTGRESQL_PASSWORD + valueFrom: + secretKeyRef: + {{- if .Values.postgresql.enabled }} + name: {{ template "dolphinscheduler.postgresql.fullname" . }} + key: postgresql-password + {{- else }} + name: {{ printf "%s-%s" .Release.Name "externaldb" }} + key: db-password + {{- end }} + - name: ZOOKEEPER_QUORUM + {{- if .Values.zookeeper.enabled }} + value: "{{ template "dolphinscheduler.zookeeper.quorum" . }}" + {{- else }} + value: {{ .Values.externalZookeeper.zookeeperQuorum }} + {{- end }} + {{- if .Values.api.livenessProbe.enabled }} + livenessProbe: + tcpSocket: + port: 12345 + initialDelaySeconds: {{ .Values.api.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.api.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.api.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.api.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.api.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.api.readinessProbe.enabled }} + readinessProbe: + tcpSocket: + port: 12345 + initialDelaySeconds: {{ .Values.api.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.api.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.api.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.api.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.api.readinessProbe.failureThreshold }} + {{- end }} + volumeMounts: + - mountPath: "/opt/dolphinscheduler/logs" + name: {{ include "dolphinscheduler.fullname" . }}-api + volumes: + - name: {{ include "dolphinscheduler.fullname" . }}-api + {{- if .Values.api.persistentVolumeClaim.enabled }} + persistentVolumeClaim: + claimName: {{ include "dolphinscheduler.fullname" . }}-api + {{- else }} + emptyDir: {} + {{- end }} \ No newline at end of file diff --git a/charts/dolphinscheduler/templates/deployment-dolphinscheduler-frontend.yaml b/charts/dolphinscheduler/templates/deployment-dolphinscheduler-frontend.yaml new file mode 100644 index 0000000000..aea09f107f --- /dev/null +++ b/charts/dolphinscheduler/templates/deployment-dolphinscheduler-frontend.yaml @@ -0,0 +1,102 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "dolphinscheduler.fullname" . }}-frontend + labels: + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-frontend + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/component: frontend +spec: + replicas: {{ .Values.frontend.replicas }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-frontend + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/component: frontend + strategy: + type: {{ .Values.frontend.strategy.type | quote }} + rollingUpdate: + maxSurge: {{ .Values.frontend.strategy.rollingUpdate.maxSurge | quote }} + maxUnavailable: {{ .Values.frontend.strategy.rollingUpdate.maxUnavailable | quote }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-frontend + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/component: frontend + spec: + {{- if .Values.frontend.affinity }} + affinity: {{- toYaml .Values.frontend.affinity | nindent 8 }} + {{- end }} + {{- if .Values.frontend.nodeSelector }} + nodeSelector: {{- toYaml .Values.frontend.nodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.frontend.tolerations }} + tolerations: {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: {{ include "dolphinscheduler.fullname" . }}-frontend + image: {{ include "dolphinscheduler.image.repository" . | quote }} + args: + - "frontend" + ports: + - containerPort: 8888 + name: tcp-port + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: TZ + value: {{ .Values.timezone }} + - name: FRONTEND_API_SERVER_HOST + value: '{{ include "dolphinscheduler.fullname" . }}-api' + - name: FRONTEND_API_SERVER_PORT + value: "12345" + {{- if .Values.frontend.livenessProbe.enabled }} + livenessProbe: + tcpSocket: + port: 8888 + initialDelaySeconds: {{ .Values.frontend.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.frontend.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.frontend.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.frontend.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.frontend.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.frontend.readinessProbe.enabled }} + readinessProbe: + tcpSocket: + port: 8888 + initialDelaySeconds: {{ .Values.frontend.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.frontend.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.frontend.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.frontend.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.frontend.readinessProbe.failureThreshold }} + {{- end }} + volumeMounts: + - mountPath: "/var/log/nginx" + name: {{ include "dolphinscheduler.fullname" . }}-frontend + volumes: + - name: {{ include "dolphinscheduler.fullname" . }}-frontend + {{- if .Values.frontend.persistentVolumeClaim.enabled }} + persistentVolumeClaim: + claimName: {{ include "dolphinscheduler.fullname" . }}-frontend + {{- else }} + emptyDir: {} + {{- end }} \ No newline at end of file diff --git a/charts/dolphinscheduler/templates/ingress.yaml b/charts/dolphinscheduler/templates/ingress.yaml new file mode 100644 index 0000000000..d0f923dcf1 --- /dev/null +++ b/charts/dolphinscheduler/templates/ingress.yaml @@ -0,0 +1,43 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +{{- if .Values.ingress.enabled }} +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: {{ include "dolphinscheduler.fullname" . }} + labels: + app.kubernetes.io/name: {{ include "dolphinscheduler.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +spec: + rules: + - host: {{ .Values.ingress.host }} + http: + paths: + - path: {{ .Values.ingress.path }} + backend: + serviceName: {{ include "dolphinscheduler.fullname" . }}-frontend + servicePort: tcp-port + {{- if .Values.ingress.tls.enabled }} + tls: + hosts: + {{- range .Values.ingress.tls.hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .Values.ingress.tls.secretName }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/dolphinscheduler/templates/pvc-dolphinscheduler-alert.yaml b/charts/dolphinscheduler/templates/pvc-dolphinscheduler-alert.yaml new file mode 100644 index 0000000000..7f74cd94ae --- /dev/null +++ b/charts/dolphinscheduler/templates/pvc-dolphinscheduler-alert.yaml @@ -0,0 +1,35 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +{{- if .Values.alert.persistentVolumeClaim.enabled }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "dolphinscheduler.fullname" . }}-alert + labels: + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-alert + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +spec: + accessModes: + {{- range .Values.alert.persistentVolumeClaim.accessModes }} + - {{ . | quote }} + {{- end }} + storageClassName: {{ .Values.alert.persistentVolumeClaim.storageClassName | quote }} + resources: + requests: + storage: {{ .Values.alert.persistentVolumeClaim.storage | quote }} +{{- end }} \ No newline at end of file diff --git a/charts/dolphinscheduler/templates/pvc-dolphinscheduler-api.yaml b/charts/dolphinscheduler/templates/pvc-dolphinscheduler-api.yaml new file mode 100644 index 0000000000..c1074cc2b1 --- /dev/null +++ b/charts/dolphinscheduler/templates/pvc-dolphinscheduler-api.yaml @@ -0,0 +1,35 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +{{- if .Values.api.persistentVolumeClaim.enabled }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "dolphinscheduler.fullname" . }}-api + labels: + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +spec: + accessModes: + {{- range .Values.api.persistentVolumeClaim.accessModes }} + - {{ . | quote }} + {{- end }} + storageClassName: {{ .Values.api.persistentVolumeClaim.storageClassName | quote }} + resources: + requests: + storage: {{ .Values.api.persistentVolumeClaim.storage | quote }} +{{- end }} \ No newline at end of file diff --git a/charts/dolphinscheduler/templates/pvc-dolphinscheduler-frontend.yaml b/charts/dolphinscheduler/templates/pvc-dolphinscheduler-frontend.yaml new file mode 100644 index 0000000000..ac9fe02a9e --- /dev/null +++ b/charts/dolphinscheduler/templates/pvc-dolphinscheduler-frontend.yaml @@ -0,0 +1,35 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +{{- if .Values.frontend.persistentVolumeClaim.enabled }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "dolphinscheduler.fullname" . }}-frontend + labels: + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-frontend + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +spec: + accessModes: + {{- range .Values.frontend.persistentVolumeClaim.accessModes }} + - {{ . | quote }} + {{- end }} + storageClassName: {{ .Values.frontend.persistentVolumeClaim.storageClassName | quote }} + resources: + requests: + storage: {{ .Values.frontend.persistentVolumeClaim.storage | quote }} +{{- end }} \ No newline at end of file diff --git a/charts/dolphinscheduler/templates/secret-external-postgresql.yaml b/charts/dolphinscheduler/templates/secret-external-postgresql.yaml new file mode 100644 index 0000000000..16d026afc6 --- /dev/null +++ b/charts/dolphinscheduler/templates/secret-external-postgresql.yaml @@ -0,0 +1,29 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +{{- if not .Values.postgresql.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ printf "%s-%s" .Release.Name "externaldb" }} + labels: + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-postgresql + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +type: Opaque +data: + db-password: {{ .Values.externalDatabase.password | b64enc | quote }} +{{- end }} \ No newline at end of file diff --git a/charts/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml b/charts/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml new file mode 100644 index 0000000000..ac974128b7 --- /dev/null +++ b/charts/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml @@ -0,0 +1,247 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "dolphinscheduler.fullname" . }}-master + labels: + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/component: master +spec: + podManagementPolicy: {{ .Values.master.podManagementPolicy }} + replicas: {{ .Values.master.replicas }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/component: master + serviceName: {{ template "dolphinscheduler.fullname" . }}-master-headless + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/component: master + spec: + {{- if .Values.master.affinity }} + affinity: {{- toYaml .Values.master.affinity | nindent 8 }} + {{- end }} + {{- if .Values.master.nodeSelector }} + nodeSelector: {{- toYaml .Values.master.nodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.master.tolerations }} + tolerations: {{- toYaml . | nindent 8 }} + {{- end }} + initContainers: + - name: init-zookeeper + image: busybox:1.31.0 + command: + - /bin/sh + - -ec + - | + echo "${ZOOKEEPER_QUORUM}" | awk -F ',' 'BEGIN{ i=1 }{ while( i <= NF ){ print $i; i++ } }' | while read line; do + while ! nc -z ${line%:*} ${line#*:}; do + counter=$((counter+1)) + if [ $counter == 5 ]; then + echo "Error: Couldn't connect to zookeeper." + exit 1 + fi + echo "Trying to connect to zookeeper at ${line}. Attempt $counter." + sleep 60 + done + done + env: + - name: ZOOKEEPER_QUORUM + {{- if .Values.zookeeper.enabled }} + value: "{{ template "dolphinscheduler.zookeeper.quorum" . }}" + {{- else }} + value: {{ .Values.externalZookeeper.zookeeperQuorum }} + {{- end }} + - name: init-postgresql + image: busybox:1.31.0 + command: + - /bin/sh + - -ec + - | + while ! nc -z ${POSTGRESQL_HOST} ${POSTGRESQL_PORT}; do + counter=$((counter+1)) + if [ $counter == 5 ]; then + echo "Error: Couldn't connect to postgresql." + exit 1 + fi + echo "Trying to connect to postgresql at ${POSTGRESQL_HOST}:${POSTGRESQL_PORT}. Attempt $counter." + sleep 60 + done + env: + - name: POSTGRESQL_HOST + {{- if .Values.postgresql.enabled }} + value: {{ template "dolphinscheduler.postgresql.fullname" . }} + {{- else }} + value: {{ .Values.externalDatabase.host | quote }} + {{- end }} + - name: POSTGRESQL_PORT + {{- if .Values.postgresql.enabled }} + value: "5432" + {{- else }} + value: {{ .Values.externalDatabase.port }} + {{- end }} + containers: + - name: {{ include "dolphinscheduler.fullname" . }}-master + image: {{ include "dolphinscheduler.image.repository" . | quote }} + args: + - "master-server" + ports: + - containerPort: 8888 + name: unused-tcp-port + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: TZ + value: {{ .Values.timezone }} + - name: MASTER_EXEC_THREADS + valueFrom: + configMapKeyRef: + name: {{ include "dolphinscheduler.fullname" . }}-master + key: MASTER_EXEC_THREADS + - name: MASTER_EXEC_TASK_NUM + valueFrom: + configMapKeyRef: + name: {{ include "dolphinscheduler.fullname" . }}-master + key: MASTER_EXEC_TASK_NUM + - name: MASTER_HEARTBEAT_INTERVAL + valueFrom: + configMapKeyRef: + name: {{ include "dolphinscheduler.fullname" . }}-master + key: MASTER_HEARTBEAT_INTERVAL + - name: MASTER_TASK_COMMIT_RETRYTIMES + valueFrom: + configMapKeyRef: + name: {{ include "dolphinscheduler.fullname" . }}-master + key: MASTER_TASK_COMMIT_RETRYTIMES + - name: MASTER_TASK_COMMIT_INTERVAL + valueFrom: + configMapKeyRef: + name: {{ include "dolphinscheduler.fullname" . }}-master + key: MASTER_TASK_COMMIT_INTERVAL + - name: MASTER_MAX_CPULOAD_AVG + valueFrom: + configMapKeyRef: + name: {{ include "dolphinscheduler.fullname" . }}-master + key: MASTER_MAX_CPULOAD_AVG + - name: MASTER_RESERVED_MEMORY + valueFrom: + configMapKeyRef: + name: {{ include "dolphinscheduler.fullname" . }}-master + key: MASTER_RESERVED_MEMORY + - name: POSTGRESQL_HOST + {{- if .Values.postgresql.enabled }} + value: {{ template "dolphinscheduler.postgresql.fullname" . }} + {{- else }} + value: {{ .Values.externalDatabase.host | quote }} + {{- end }} + - name: POSTGRESQL_PORT + {{- if .Values.postgresql.enabled }} + value: "5432" + {{- else }} + value: {{ .Values.externalDatabase.port }} + {{- end }} + - name: POSTGRESQL_USERNAME + {{- if .Values.postgresql.enabled }} + value: {{ .Values.postgresql.postgresqlUsername }} + {{- else }} + value: {{ .Values.externalDatabase.username | quote }} + {{- end }} + - name: POSTGRESQL_PASSWORD + valueFrom: + secretKeyRef: + {{- if .Values.postgresql.enabled }} + name: {{ template "dolphinscheduler.postgresql.fullname" . }} + key: postgresql-password + {{- else }} + name: {{ printf "%s-%s" .Release.Name "externaldb" }} + key: db-password + {{- end }} + - name: TASK_QUEUE + {{- if .Values.zookeeper.enabled }} + value: {{ .Values.zookeeper.taskQueue }} + {{- else }} + value: {{ .Values.externalZookeeper.taskQueue }} + {{- end }} + - name: ZOOKEEPER_QUORUM + {{- if .Values.zookeeper.enabled }} + value: {{ template "dolphinscheduler.zookeeper.quorum" . }} + {{- else }} + value: {{ .Values.externalZookeeper.zookeeperQuorum }} + {{- end }} + {{- if .Values.master.livenessProbe.enabled }} + livenessProbe: + exec: + command: + - sh + - /root/checkpoint.sh + - master-server + initialDelaySeconds: {{ .Values.master.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.master.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.master.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.master.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.master.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.master.readinessProbe.enabled }} + readinessProbe: + exec: + command: + - sh + - /root/checkpoint.sh + - master-server + initialDelaySeconds: {{ .Values.master.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.master.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.master.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.master.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.master.readinessProbe.failureThreshold }} + {{- end }} + volumeMounts: + - mountPath: "/opt/dolphinscheduler/logs" + name: {{ include "dolphinscheduler.fullname" . }}-master + volumes: + - name: {{ include "dolphinscheduler.fullname" . }}-master + {{- if .Values.master.persistentVolumeClaim.enabled }} + persistentVolumeClaim: + claimName: {{ include "dolphinscheduler.fullname" . }}-master + {{- else }} + emptyDir: {} + {{- end }} + {{- if .Values.master.persistentVolumeClaim.enabled }} + volumeClaimTemplates: + - metadata: + name: {{ include "dolphinscheduler.fullname" . }}-master + labels: + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + spec: + accessModes: + {{- range .Values.master.persistentVolumeClaim.accessModes }} + - {{ . | quote }} + {{- end }} + storageClassName: {{ .Values.master.persistentVolumeClaim.storageClassName | quote }} + resources: + requests: + storage: {{ .Values.master.persistentVolumeClaim.storage | quote }} + {{- end }} diff --git a/charts/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml b/charts/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml new file mode 100644 index 0000000000..a2407978b4 --- /dev/null +++ b/charts/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml @@ -0,0 +1,275 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "dolphinscheduler.fullname" . }}-worker + labels: + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/component: worker +spec: + podManagementPolicy: {{ .Values.worker.podManagementPolicy }} + replicas: {{ .Values.worker.replicas }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/component: worker + serviceName: {{ template "dolphinscheduler.fullname" . }}-worker-headless + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/component: worker + spec: + {{- if .Values.worker.affinity }} + affinity: {{- toYaml .Values.worker.affinity | nindent 8 }} + {{- end }} + {{- if .Values.worker.nodeSelector }} + nodeSelector: {{- toYaml .Values.worker.nodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.worker.tolerations }} + tolerations: {{- toYaml . | nindent 8 }} + {{- end }} + initContainers: + - name: init-zookeeper + image: busybox:1.31.0 + command: + - /bin/sh + - -ec + - | + echo "${ZOOKEEPER_QUORUM}" | awk -F ',' 'BEGIN{ i=1 }{ while( i <= NF ){ print $i; i++ } }' | while read line; do + while ! nc -z ${line%:*} ${line#*:}; do + counter=$((counter+1)) + if [ $counter == 5 ]; then + echo "Error: Couldn't connect to zookeeper." + exit 1 + fi + echo "Trying to connect to zookeeper at ${line}. Attempt $counter." + sleep 60 + done + done + env: + - name: ZOOKEEPER_QUORUM + {{- if .Values.zookeeper.enabled }} + value: "{{ template "dolphinscheduler.zookeeper.quorum" . }}" + {{- else }} + value: {{ .Values.externalZookeeper.zookeeperQuorum }} + {{- end }} + - name: init-postgresql + image: busybox:1.31.0 + command: + - /bin/sh + - -ec + - | + while ! nc -z ${POSTGRESQL_HOST} ${POSTGRESQL_PORT}; do + counter=$((counter+1)) + if [ $counter == 5 ]; then + echo "Error: Couldn't connect to postgresql." + exit 1 + fi + echo "Trying to connect to postgresql at ${POSTGRESQL_HOST}:${POSTGRESQL_PORT}. Attempt $counter." + sleep 60 + done + env: + - name: POSTGRESQL_HOST + {{- if .Values.postgresql.enabled }} + value: {{ template "dolphinscheduler.postgresql.fullname" . }} + {{- else }} + value: {{ .Values.externalDatabase.host | quote }} + {{- end }} + - name: POSTGRESQL_PORT + {{- if .Values.postgresql.enabled }} + value: "5432" + {{- else }} + value: {{ .Values.externalDatabase.port }} + {{- end }} + containers: + - name: {{ include "dolphinscheduler.fullname" . }}-worker + image: {{ include "dolphinscheduler.image.repository" . | quote }} + args: + - "worker-server" + ports: + - containerPort: 50051 + name: "logs-port" + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: TZ + value: {{ .Values.timezone }} + - name: WORKER_EXEC_THREADS + valueFrom: + configMapKeyRef: + name: {{ include "dolphinscheduler.fullname" . }}-worker + key: WORKER_EXEC_THREADS + - name: WORKER_FETCH_TASK_NUM + valueFrom: + configMapKeyRef: + name: {{ include "dolphinscheduler.fullname" . }}-worker + key: WORKER_FETCH_TASK_NUM + - name: WORKER_HEARTBEAT_INTERVAL + valueFrom: + configMapKeyRef: + name: {{ include "dolphinscheduler.fullname" . }}-worker + key: WORKER_HEARTBEAT_INTERVAL + - name: WORKER_MAX_CPULOAD_AVG + valueFrom: + configMapKeyRef: + name: {{ include "dolphinscheduler.fullname" . }}-worker + key: WORKER_MAX_CPULOAD_AVG + - name: WORKER_RESERVED_MEMORY + valueFrom: + configMapKeyRef: + name: {{ include "dolphinscheduler.fullname" . }}-worker + key: WORKER_RESERVED_MEMORY + - name: POSTGRESQL_HOST + {{- if .Values.postgresql.enabled }} + value: {{ template "dolphinscheduler.postgresql.fullname" . }} + {{- else }} + value: {{ .Values.externalDatabase.host | quote }} + {{- end }} + - name: POSTGRESQL_PORT + {{- if .Values.postgresql.enabled }} + value: "5432" + {{- else }} + value: {{ .Values.externalDatabase.port }} + {{- end }} + - name: POSTGRESQL_USERNAME + {{- if .Values.postgresql.enabled }} + value: {{ .Values.postgresql.postgresqlUsername }} + {{- else }} + value: {{ .Values.externalDatabase.username | quote }} + {{- end }} + - name: POSTGRESQL_PASSWORD + valueFrom: + secretKeyRef: + {{- if .Values.postgresql.enabled }} + name: {{ template "dolphinscheduler.postgresql.fullname" . }} + key: postgresql-password + {{- else }} + name: {{ printf "%s-%s" .Release.Name "externaldb" }} + key: db-password + {{- end }} + - name: TASK_QUEUE + {{- if .Values.zookeeper.enabled }} + value: {{ .Values.zookeeper.taskQueue }} + {{- else }} + value: {{ .Values.externalZookeeper.taskQueue }} + {{- end }} + - name: ZOOKEEPER_QUORUM + {{- if .Values.zookeeper.enabled }} + value: "{{ template "dolphinscheduler.zookeeper.quorum" . }}" + {{- else }} + value: {{ .Values.externalZookeeper.zookeeperQuorum }} + {{- end }} + {{- if .Values.worker.livenessProbe.enabled }} + livenessProbe: + exec: + command: + - sh + - /root/checkpoint.sh + - worker-server + initialDelaySeconds: {{ .Values.worker.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.worker.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.worker.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.worker.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.worker.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.worker.readinessProbe.enabled }} + readinessProbe: + exec: + command: + - sh + - /root/checkpoint.sh + - worker-server + initialDelaySeconds: {{ .Values.worker.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.worker.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.worker.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.worker.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.worker.readinessProbe.failureThreshold }} + {{- end }} + volumeMounts: + - mountPath: {{ include "dolphinscheduler.worker.base.dir" . | quote }} + name: {{ include "dolphinscheduler.fullname" . }}-worker-data + - mountPath: "/opt/dolphinscheduler/logs" + name: {{ include "dolphinscheduler.fullname" . }}-worker-logs + - mountPath: "/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh" + subPath: "dolphinscheduler_env.sh" + name: {{ include "dolphinscheduler.fullname" . }}-worker-configmap + volumes: + - name: {{ include "dolphinscheduler.fullname" . }}-worker-data + {{- if .Values.worker.persistentVolumeClaim.dataPersistentVolume.enabled }} + persistentVolumeClaim: + claimName: {{ include "dolphinscheduler.fullname" . }}-worker-data + {{- else }} + emptyDir: {} + {{- end }} + - name: {{ include "dolphinscheduler.fullname" . }}-worker-logs + {{- if .Values.worker.persistentVolumeClaim.logsPersistentVolume.enabled }} + persistentVolumeClaim: + claimName: {{ include "dolphinscheduler.fullname" . }}-worker-logs + {{- else }} + emptyDir: {} + {{- end }} + - name: {{ include "dolphinscheduler.fullname" . }}-worker-configmap + configMap: + defaultMode: 0777 + name: {{ include "dolphinscheduler.fullname" . }}-worker + items: + - key: dolphinscheduler_env.sh + path: dolphinscheduler_env.sh + {{- if .Values.worker.persistentVolumeClaim.enabled }} + volumeClaimTemplates: + {{- if .Values.worker.persistentVolumeClaim.dataPersistentVolume.enabled }} + - metadata: + name: {{ include "dolphinscheduler.fullname" . }}-worker-data + labels: + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker-data + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + spec: + accessModes: + {{- range .Values.worker.persistentVolumeClaim.dataPersistentVolume.accessModes }} + - {{ . | quote }} + {{- end }} + storageClassName: {{ .Values.worker.persistentVolumeClaim.dataPersistentVolume.storageClassName | quote }} + resources: + requests: + storage: {{ .Values.worker.persistentVolumeClaim.dataPersistentVolume.storage | quote }} + {{- end }} + {{- if .Values.worker.persistentVolumeClaim.logsPersistentVolume.enabled }} + - metadata: + name: {{ include "dolphinscheduler.fullname" . }}-worker-logs + labels: + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker-logs + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + spec: + accessModes: + {{- range .Values.worker.persistentVolumeClaim.logsPersistentVolume.accessModes }} + - {{ . | quote }} + {{- end }} + storageClassName: {{ .Values.worker.persistentVolumeClaim.logsPersistentVolume.storageClassName | quote }} + resources: + requests: + storage: {{ .Values.worker.persistentVolumeClaim.logsPersistentVolume.storage | quote }} + {{- end }} + {{- end }} diff --git a/charts/dolphinscheduler/templates/svc-dolphinscheduler-api.yaml b/charts/dolphinscheduler/templates/svc-dolphinscheduler-api.yaml new file mode 100644 index 0000000000..4d07ade242 --- /dev/null +++ b/charts/dolphinscheduler/templates/svc-dolphinscheduler-api.yaml @@ -0,0 +1,35 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +apiVersion: v1 +kind: Service +metadata: + name: {{ include "dolphinscheduler.fullname" . }}-api + labels: + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +spec: + ports: + - port: 12345 + targetPort: tcp-port + protocol: TCP + name: tcp-port + selector: + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/component: api \ No newline at end of file diff --git a/charts/dolphinscheduler/templates/svc-dolphinscheduler-frontend.yaml b/charts/dolphinscheduler/templates/svc-dolphinscheduler-frontend.yaml new file mode 100644 index 0000000000..60d0d6e7b5 --- /dev/null +++ b/charts/dolphinscheduler/templates/svc-dolphinscheduler-frontend.yaml @@ -0,0 +1,35 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +apiVersion: v1 +kind: Service +metadata: + name: {{ include "dolphinscheduler.fullname" . }}-frontend + labels: + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-frontend + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +spec: + ports: + - port: 8888 + targetPort: tcp-port + protocol: TCP + name: tcp-port + selector: + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-frontend + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/component: frontend \ No newline at end of file diff --git a/charts/dolphinscheduler/templates/svc-dolphinscheduler-master-headless.yaml b/charts/dolphinscheduler/templates/svc-dolphinscheduler-master-headless.yaml new file mode 100644 index 0000000000..7aaf0b4353 --- /dev/null +++ b/charts/dolphinscheduler/templates/svc-dolphinscheduler-master-headless.yaml @@ -0,0 +1,36 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +apiVersion: v1 +kind: Service +metadata: + name: {{ include "dolphinscheduler.fullname" . }}-master-headless + labels: + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master-headless + app.kubernetes.io/instance: {{ .Release.Name }}-master-headless + app.kubernetes.io/managed-by: {{ .Release.Service }} +spec: + clusterIP: "None" + ports: + - port: 8888 + targetPort: tcp-port + protocol: TCP + name: unused-tcp-port + selector: + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/component: master \ No newline at end of file diff --git a/charts/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml b/charts/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml new file mode 100644 index 0000000000..3e92a349d4 --- /dev/null +++ b/charts/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml @@ -0,0 +1,36 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +apiVersion: v1 +kind: Service +metadata: + name: {{ include "dolphinscheduler.fullname" . }}-worker-headless + labels: + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker-headless + app.kubernetes.io/instance: {{ .Release.Name }}-worker-headless + app.kubernetes.io/managed-by: {{ .Release.Service }} +spec: + clusterIP: "None" + ports: + - port: 50051 + targetPort: logs-port + protocol: TCP + name: logs-port + selector: + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/component: worker \ No newline at end of file diff --git a/charts/dolphinscheduler/values.yaml b/charts/dolphinscheduler/values.yaml new file mode 100644 index 0000000000..962a031a0c --- /dev/null +++ b/charts/dolphinscheduler/values.yaml @@ -0,0 +1,355 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Default values for dolphinscheduler-chart. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +nameOverride: "" +fullnameOverride: "" + +timezone: "Asia/Shanghai" + +image: + registry: "docker.io" + repository: "dolphinscheduler" + tag: "1.2.1" + pullPolicy: "IfNotPresent" + +imagePullSecrets: [] + +# If not exists external postgresql, by default, Dolphinscheduler's database will use it. +postgresql: + enabled: true + postgresqlUsername: "root" + postgresqlPassword: "root" + postgresqlDatabase: "dolphinscheduler" + persistence: + enabled: false + size: "20Gi" + storageClass: "-" + +# If exists external postgresql, and set postgresql.enable value to false. +# If postgresql.enable is false, Dolphinscheduler's database will use it. +externalDatabase: + host: "localhost" + port: "5432" + username: "root" + password: "root" + database: "dolphinscheduler" + +# If not exists external zookeeper, by default, Dolphinscheduler's zookeeper will use it. +zookeeper: + enabled: true + taskQueue: "zookeeper" + persistence: + enabled: false + size: "20Gi" + storageClass: "-" + +# If exists external zookeeper, and set zookeeper.enable value to false. +# If zookeeper.enable is false, Dolphinscheduler's zookeeper will use it. +externalZookeeper: + taskQueue: "zookeeper" + zookeeperQuorum: "127.0.0.1:2181" + +master: + podManagementPolicy: "Parallel" + replicas: "3" + # NodeSelector is a selector which must be true for the pod to fit on a node. + # Selector which must match a node's labels for the pod to be scheduled on that node. + # More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + nodeSelector: {} + # Tolerations are appended (excluding duplicates) to pods running with this RuntimeClass during admission, + # effectively unioning the set of nodes tolerated by the pod and the RuntimeClass. + tolerations: [] + # Affinity is a group of affinity scheduling rules. + # If specified, the pod's scheduling constraints. + # More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core + affinity: {} + ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. + ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + configmap: + MASTER_EXEC_THREADS: "100" + MASTER_EXEC_TASK_NUM: "20" + MASTER_HEARTBEAT_INTERVAL: "10" + MASTER_TASK_COMMIT_RETRYTIMES: "5" + MASTER_TASK_COMMIT_INTERVAL: "1000" + MASTER_MAX_CPULOAD_AVG: "100" + MASTER_RESERVED_MEMORY: "0.1" + livenessProbe: + enabled: true + initialDelaySeconds: "30" + periodSeconds: "30" + timeoutSeconds: "5" + failureThreshold: "3" + successThreshold: "1" + ## Periodic probe of container service readiness. Container will be removed from service endpoints if the probe fails. Cannot be updated. + ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + readinessProbe: + enabled: true + initialDelaySeconds: "30" + periodSeconds: "30" + timeoutSeconds: "5" + failureThreshold: "3" + successThreshold: "1" + ## volumeClaimTemplates is a list of claims that pods are allowed to reference. + ## The StatefulSet controller is responsible for mapping network identities to claims in a way that maintains the identity of a pod. + ## Every claim in this list must have at least one matching (by name) volumeMount in one container in the template. + ## A claim in this list takes precedence over any volumes in the template, with the same name. + persistentVolumeClaim: + enabled: false + accessModes: + - "ReadWriteOnce" + storageClassName: "-" + storage: "20Gi" + +worker: + podManagementPolicy: "Parallel" + replicas: "3" + # NodeSelector is a selector which must be true for the pod to fit on a node. + # Selector which must match a node's labels for the pod to be scheduled on that node. + # More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + nodeSelector: {} + # Tolerations are appended (excluding duplicates) to pods running with this RuntimeClass during admission, + # effectively unioning the set of nodes tolerated by the pod and the RuntimeClass. + tolerations: [] + # Affinity is a group of affinity scheduling rules. + # If specified, the pod's scheduling constraints. + # More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core + affinity: {} + ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. + ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + livenessProbe: + enabled: true + initialDelaySeconds: "30" + periodSeconds: "30" + timeoutSeconds: "5" + failureThreshold: "3" + successThreshold: "1" + ## Periodic probe of container service readiness. Container will be removed from service endpoints if the probe fails. Cannot be updated. + ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + readinessProbe: + enabled: true + initialDelaySeconds: "30" + periodSeconds: "30" + timeoutSeconds: "5" + failureThreshold: "3" + successThreshold: "1" + configmap: + WORKER_EXEC_THREADS: "100" + WORKER_HEARTBEAT_INTERVAL: "10" + WORKER_FETCH_TASK_NUM: "3" + WORKER_MAX_CPULOAD_AVG: "100" + WORKER_RESERVED_MEMORY: "0.1" + DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler" + DOLPHINSCHEDULER_ENV: + - "export HADOOP_HOME=/opt/soft/hadoop" + - "export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop" + - "export SPARK_HOME1=/opt/soft/spark1" + - "export SPARK_HOME2=/opt/soft/spark2" + - "export PYTHON_HOME=/opt/soft/python" + - "export JAVA_HOME=/opt/soft/java" + - "export HIVE_HOME=/opt/soft/hive" + - "export FLINK_HOME=/opt/soft/flink" + - "export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$PATH" + ## volumeClaimTemplates is a list of claims that pods are allowed to reference. + ## The StatefulSet controller is responsible for mapping network identities to claims in a way that maintains the identity of a pod. + ## Every claim in this list must have at least one matching (by name) volumeMount in one container in the template. + ## A claim in this list takes precedence over any volumes in the template, with the same name. + persistentVolumeClaim: + enabled: false + ## dolphinscheduler data volume + dataPersistentVolume: + enabled: false + accessModes: + - "ReadWriteOnce" + storageClassName: "-" + storage: "20Gi" + ## dolphinscheduler logs volume + logsPersistentVolume: + enabled: false + accessModes: + - "ReadWriteOnce" + storageClassName: "-" + storage: "20Gi" + +alert: + strategy: + type: "RollingUpdate" + rollingUpdate: + maxSurge: "25%" + maxUnavailable: "25%" + replicas: "1" + # NodeSelector is a selector which must be true for the pod to fit on a node. + # Selector which must match a node's labels for the pod to be scheduled on that node. + # More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + nodeSelector: {} + # Tolerations are appended (excluding duplicates) to pods running with this RuntimeClass during admission, + # effectively unioning the set of nodes tolerated by the pod and the RuntimeClass. + tolerations: [] + # Affinity is a group of affinity scheduling rules. + # If specified, the pod's scheduling constraints. + # More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core + affinity: {} + ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. + ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + configmap: + XLS_FILE_PATH: "/tmp/xls" + MAIL_SERVER_HOST: "" + MAIL_SERVER_PORT: "" + MAIL_SENDER: "" + MAIL_USER: "" + MAIL_PASSWD: "" + MAIL_SMTP_STARTTLS_ENABLE: false + MAIL_SMTP_SSL_ENABLE: false + MAIL_SMTP_SSL_TRUST: "" + ENTERPRISE_WECHAT_ENABLE: false + ENTERPRISE_WECHAT_CORP_ID: "" + ENTERPRISE_WECHAT_SECRET: "" + ENTERPRISE_WECHAT_AGENT_ID: "" + ENTERPRISE_WECHAT_USERS: "" + livenessProbe: + enabled: true + initialDelaySeconds: "30" + periodSeconds: "30" + timeoutSeconds: "5" + failureThreshold: "3" + successThreshold: "1" + ## Periodic probe of container service readiness. Container will be removed from service endpoints if the probe fails. Cannot be updated. + ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + readinessProbe: + enabled: true + initialDelaySeconds: "30" + periodSeconds: "30" + timeoutSeconds: "5" + failureThreshold: "3" + successThreshold: "1" + ## volumeClaimTemplates is a list of claims that pods are allowed to reference. + ## The StatefulSet controller is responsible for mapping network identities to claims in a way that maintains the identity of a pod. + ## Every claim in this list must have at least one matching (by name) volumeMount in one container in the template. + ## A claim in this list takes precedence over any volumes in the template, with the same name. + persistentVolumeClaim: + enabled: false + accessModes: + - "ReadWriteOnce" + storageClassName: "-" + storage: "20Gi" + +api: + strategy: + type: "RollingUpdate" + rollingUpdate: + maxSurge: "25%" + maxUnavailable: "25%" + replicas: "1" + # NodeSelector is a selector which must be true for the pod to fit on a node. + # Selector which must match a node's labels for the pod to be scheduled on that node. + # More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + nodeSelector: {} + # Tolerations are appended (excluding duplicates) to pods running with this RuntimeClass during admission, + # effectively unioning the set of nodes tolerated by the pod and the RuntimeClass. + tolerations: [] + # Affinity is a group of affinity scheduling rules. + # If specified, the pod's scheduling constraints. + # More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core + affinity: {} + ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. + ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + livenessProbe: + enabled: true + initialDelaySeconds: "30" + periodSeconds: "30" + timeoutSeconds: "5" + failureThreshold: "3" + successThreshold: "1" + ## Periodic probe of container service readiness. Container will be removed from service endpoints if the probe fails. Cannot be updated. + ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + readinessProbe: + enabled: true + initialDelaySeconds: "30" + periodSeconds: "30" + timeoutSeconds: "5" + failureThreshold: "3" + successThreshold: "1" + ## volumeClaimTemplates is a list of claims that pods are allowed to reference. + ## The StatefulSet controller is responsible for mapping network identities to claims in a way that maintains the identity of a pod. + ## Every claim in this list must have at least one matching (by name) volumeMount in one container in the template. + ## A claim in this list takes precedence over any volumes in the template, with the same name. + persistentVolumeClaim: + enabled: false + accessModes: + - "ReadWriteOnce" + storageClassName: "-" + storage: "20Gi" + +frontend: + strategy: + type: "RollingUpdate" + rollingUpdate: + maxSurge: "25%" + maxUnavailable: "25%" + replicas: "1" + # NodeSelector is a selector which must be true for the pod to fit on a node. + # Selector which must match a node's labels for the pod to be scheduled on that node. + # More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + nodeSelector: {} + # Tolerations are appended (excluding duplicates) to pods running with this RuntimeClass during admission, + # effectively unioning the set of nodes tolerated by the pod and the RuntimeClass. + tolerations: [] + # Affinity is a group of affinity scheduling rules. + # If specified, the pod's scheduling constraints. + # More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core + affinity: {} + ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. + ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + livenessProbe: + enabled: true + initialDelaySeconds: "30" + periodSeconds: "30" + timeoutSeconds: "5" + failureThreshold: "3" + successThreshold: "1" + ## Periodic probe of container service readiness. Container will be removed from service endpoints if the probe fails. Cannot be updated. + ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + readinessProbe: + enabled: true + initialDelaySeconds: "30" + periodSeconds: "30" + timeoutSeconds: "5" + failureThreshold: "3" + successThreshold: "1" + ## volumeClaimTemplates is a list of claims that pods are allowed to reference. + ## The StatefulSet controller is responsible for mapping network identities to claims in a way that maintains the identity of a pod. + ## Every claim in this list must have at least one matching (by name) volumeMount in one container in the template. + ## A claim in this list takes precedence over any volumes in the template, with the same name. + persistentVolumeClaim: + enabled: false + accessModes: + - "ReadWriteOnce" + storageClassName: "-" + storage: "20Gi" + +ingress: + enabled: false + host: "dolphinscheduler.org" + path: "/" + tls: + enabled: false + hosts: + - "dolphinscheduler.org" + secretName: "dolphinscheduler-tls" \ No newline at end of file diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index de5908583c..7e9c4e57cb 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. version: '2' services: zookeeper: diff --git a/dockerfile/hooks/check b/dockerfile/hooks/check index 21f3f4db12..fdb1902311 100644 --- a/dockerfile/hooks/check +++ b/dockerfile/hooks/check @@ -16,7 +16,7 @@ # limitations under the License. # echo "------ dolphinscheduler check - server - status -------" -sleep 20 +sleep 60 server_num=$(docker top `docker container list | grep '/sbin/tini' | awk '{print $1}'`| grep java | grep "dolphinscheduler" | awk -F 'classpath ' '{print $2}' | awk '{print $2}' | sort | uniq -c | wc -l) if [ $server_num -eq 5 ] then @@ -25,3 +25,11 @@ else echo "Server start failed "$server_num exit 1 fi +ready=`curl http://127.0.0.1:8888/dolphinscheduler/login -d 'userName=admin&userPassword=dolphinscheduler123' -v | grep "login success" | wc -l` +if [ $ready -eq 1 ] +then + echo "Servers is ready" +else + echo "Servers is not ready" + exit 1 +fi diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/ExcelUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/ExcelUtils.java index b2e71a8980..366e2828c5 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/ExcelUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/ExcelUtils.java @@ -26,6 +26,7 @@ import org.apache.poi.ss.usermodel.HorizontalAlignment; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.util.*; @@ -102,7 +103,11 @@ public class ExcelUtils { for (int i = 0; i < headerList.size(); i++) { sheet.setColumnWidth(i, headerList.get(i).length() * 800); + } + File file = new File(xlsFilePath); + if (!file.exists()) { + file.mkdirs(); } //setting file output diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/ExcelUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/ExcelUtilsTest.java index 3ef43aeef4..8ee62358dd 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/ExcelUtilsTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/ExcelUtilsTest.java @@ -89,4 +89,14 @@ public class ExcelUtilsTest { ExcelUtils.genExcelFile(incorrectContent1, title, xlsFilePath); } + + /** + * Test GenExcelFile (check directory) + */ + @Test + public void testGenExcelFileByCheckDir() { + ExcelUtils.genExcelFile("[{\"a\": \"a\"},{\"a\": \"a\"}]", "t", "/tmp/xls"); + File file = new File("/tmp/xls" + Constants.SINGLE_SLASH + "t" + Constants.EXCEL_SUFFIX_XLS); + file.delete(); + } } \ No newline at end of file diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AlertGroupController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AlertGroupController.java index e9bffa510b..140434ee43 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AlertGroupController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AlertGroupController.java @@ -93,11 +93,11 @@ public class AlertGroupController extends BaseController{ public Result list(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser) { logger.info("login user {}, query all alertGroup", loginUser.getUserName()); - try{ + try { HashMap result = alertGroupService.queryAlertgroup(); return returnDataList(result); - }catch (Exception e){ - logger.error(Status.QUERY_ALL_ALERTGROUP_ERROR.getMsg(),e); + } catch (Exception e) { + logger.error(Status.QUERY_ALL_ALERTGROUP_ERROR.getMsg(), e); return error(Status.QUERY_ALL_ALERTGROUP_ERROR.getCode(), Status.QUERY_ALL_ALERTGROUP_ERROR.getMsg()); } } @@ -214,12 +214,20 @@ public class AlertGroupController extends BaseController{ @GetMapping(value = "/verify-group-name") @ResponseStatus(HttpStatus.OK) public Result verifyGroupName(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @RequestParam(value ="groupName") String groupName - ) { - logger.info("login user {}, verfiy group name: {}", - loginUser.getUserName(),groupName); + @RequestParam(value ="groupName") String groupName) { + logger.info("login user {}, verify group name: {}", loginUser.getUserName(), groupName); - return alertGroupService.verifyGroupName(loginUser, groupName); + boolean exist= alertGroupService.existGroupName(groupName); + Result result = new Result(); + if (exist) { + logger.error("group {} has exist, can't create again.", groupName); + result.setCode(Status.ALERT_GROUP_EXIST.getCode()); + result.setMsg(Status.ALERT_GROUP_EXIST.getMsg()); + } else { + result.setCode(Status.SUCCESS.getCode()); + result.setMsg(Status.SUCCESS.getMsg()); + } + return result; } /** diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java index 881c93f2f7..89e6134609 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java @@ -16,18 +16,19 @@ */ package org.apache.dolphinscheduler.api.controller; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiImplicitParam; +import io.swagger.annotations.ApiImplicitParams; +import io.swagger.annotations.ApiOperation; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.service.DataSourceService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.DbConnectType; import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; -import io.swagger.annotations.Api; -import io.swagger.annotations.ApiImplicitParam; -import io.swagger.annotations.ApiImplicitParams; -import io.swagger.annotations.ApiOperation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -76,6 +77,7 @@ public class DataSourceController extends BaseController { @ApiImplicitParam(name = "database", value = "DATABASE_NAME",required = true, dataType ="String"), @ApiImplicitParam(name = "userName", value = "USER_NAME",required = true, dataType ="String"), @ApiImplicitParam(name = "password", value = "PASSWORD", dataType ="String"), + @ApiImplicitParam(name = "connectType", value = "CONNECT_TYPE", dataType = "DbConnectType"), @ApiImplicitParam(name = "other", value = "DATA_SOURCE_OTHER", dataType ="String") }) @PostMapping(value = "/create") @@ -90,11 +92,12 @@ public class DataSourceController extends BaseController { @RequestParam(value = "principal") String principal, @RequestParam(value = "userName") String userName, @RequestParam(value = "password") String password, + @RequestParam(value = "connectType") DbConnectType connectType, @RequestParam(value = "other") String other) { - logger.info("login user {} create datasource name: {}, note: {}, type: {}, host: {},port: {},database : {},principal: {},userName : {} other: {}", - loginUser.getUserName(), name, note, type, host,port,database,principal,userName,other); + logger.info("login user {} create datasource name: {}, note: {}, type: {}, host: {}, port: {}, database : {}, principal: {}, userName : {}, connectType: {}, other: {}", + loginUser.getUserName(), name, note, type, host, port, database, principal, userName, connectType, other); try { - String parameter = dataSourceService.buildParameter(name, note, type, host, port, database,principal,userName, password, other); + String parameter = dataSourceService.buildParameter(name, note, type, host, port, database, principal, userName, password, connectType, other); Map result = dataSourceService.createDataSource(loginUser, name, note, type, parameter); return returnDataList(result); @@ -133,6 +136,7 @@ public class DataSourceController extends BaseController { @ApiImplicitParam(name = "database", value = "DATABASE_NAME",required = true, dataType ="String"), @ApiImplicitParam(name = "userName", value = "USER_NAME",required = true, dataType ="String"), @ApiImplicitParam(name = "password", value = "PASSWORD", dataType ="String"), + @ApiImplicitParam(name = "connectType", value = "CONNECT_TYPE", dataType = "DbConnectType"), @ApiImplicitParam(name = "other", value = "DATA_SOURCE_OTHER", dataType ="String") }) @PostMapping(value = "/update") @@ -148,11 +152,12 @@ public class DataSourceController extends BaseController { @RequestParam(value = "principal") String principal, @RequestParam(value = "userName") String userName, @RequestParam(value = "password") String password, + @RequestParam(value = "connectType") DbConnectType connectType, @RequestParam(value = "other") String other) { - logger.info("login user {} updateProcessInstance datasource name: {}, note: {}, type: {}, other: {}", - loginUser.getUserName(), name, note, type, other); + logger.info("login user {} updateProcessInstance datasource name: {}, note: {}, type: {}, connectType: {}, other: {}", + loginUser.getUserName(), name, note, type, connectType, other); try { - String parameter = dataSourceService.buildParameter(name, note, type, host, port, database,principal, userName, password, other); + String parameter = dataSourceService.buildParameter(name, note, type, host, port, database,principal, userName, password, connectType, other); Map dataSource = dataSourceService.updateDataSource(id, loginUser, name, note, type, parameter); return returnDataList(dataSource); } catch (Exception e) { @@ -277,6 +282,7 @@ public class DataSourceController extends BaseController { @ApiImplicitParam(name = "database", value = "DATABASE_NAME",required = true, dataType ="String"), @ApiImplicitParam(name = "userName", value = "USER_NAME",required = true, dataType ="String"), @ApiImplicitParam(name = "password", value = "PASSWORD", dataType ="String"), + @ApiImplicitParam(name = "connectType", value = "CONNECT_TYPE", dataType = "DbConnectType"), @ApiImplicitParam(name = "other", value = "DATA_SOURCE_OTHER", dataType ="String") }) @PostMapping(value = "/connect") @@ -291,11 +297,12 @@ public class DataSourceController extends BaseController { @RequestParam(value = "principal") String principal, @RequestParam(value = "userName") String userName, @RequestParam(value = "password") String password, + @RequestParam(value = "connectType") DbConnectType connectType, @RequestParam(value = "other") String other) { - logger.info("login user {}, connect datasource: {} failure, note: {}, type: {}, other: {}", - loginUser.getUserName(), name, note, type, other); + logger.info("login user {}, connect datasource: {} failure, note: {}, type: {}, connectType: {}, other: {}", + loginUser.getUserName(), name, note, type, connectType, other); try { - String parameter = dataSourceService.buildParameter(name, note, type, host, port, database,principal,userName, password, other); + String parameter = dataSourceService.buildParameter(name, note, type, host, port, database, principal, userName, password, connectType, other); Boolean isConnection = dataSourceService.checkConnection(type, parameter); Result result = new Result(); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java index 102f116575..5faf7a4f14 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java @@ -391,7 +391,7 @@ public class ProcessInstanceController extends BaseController{ } } } - if(deleteFailedIdList.size() > 0){ + if(!deleteFailedIdList.isEmpty()){ putMsg(result, Status.BATCH_DELETE_PROCESS_INSTANCE_BY_IDS_ERROR, String.join(",", deleteFailedIdList)); }else{ putMsg(result, Status.SUCCESS); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java index 7bac6614ee..1d83fcea2b 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java @@ -60,6 +60,50 @@ public class ResourcesController extends BaseController{ @Autowired private UdfFuncService udfFuncService; + /** + * create resource + * + * @param loginUser login user + * @param alias alias + * @param description description + * @param type type + * @return create result code + */ + + /** + * + * @param loginUser login user + * @param type type + * @param alias alias + * @param description description + * @param pid parent id + * @param currentDir current directory + * @return + */ + @ApiOperation(value = "createDirctory", notes= "CREATE_RESOURCE_NOTES") + @ApiImplicitParams({ + @ApiImplicitParam(name = "type", value = "RESOURCE_TYPE", required = true, dataType ="ResourceType"), + @ApiImplicitParam(name = "name", value = "RESOURCE_NAME", required = true, dataType ="String"), + @ApiImplicitParam(name = "description", value = "RESOURCE_DESC", dataType ="String"), + @ApiImplicitParam(name = "file", value = "RESOURCE_FILE", required = true, dataType = "MultipartFile") + }) + @PostMapping(value = "/directory/create") + public Result createDirectory(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + @RequestParam(value = "type") ResourceType type, + @RequestParam(value ="name") String alias, + @RequestParam(value = "description", required = false) String description, + @RequestParam(value ="pid") int pid, + @RequestParam(value ="currentDir") String currentDir) { + try { + logger.info("login user {}, create resource, type: {}, resource alias: {}, desc: {}, file: {},{}", + loginUser.getUserName(),type, alias, description,pid,currentDir); + return resourceService.createDirectory(loginUser,alias, description,type ,pid,currentDir); + } catch (Exception e) { + logger.error(CREATE_RESOURCE_ERROR.getMsg(),e); + return error(CREATE_RESOURCE_ERROR.getCode(), CREATE_RESOURCE_ERROR.getMsg()); + } + } + /** * create resource * @@ -80,13 +124,15 @@ public class ResourcesController extends BaseController{ @PostMapping(value = "/create") public Result createResource(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam(value = "type") ResourceType type, - @RequestParam(value ="name")String alias, + @RequestParam(value ="name") String alias, @RequestParam(value = "description", required = false) String description, - @RequestParam("file") MultipartFile file) { + @RequestParam("file") MultipartFile file, + @RequestParam(value ="pid") int pid, + @RequestParam(value ="currentDir") String currentDir) { try { logger.info("login user {}, create resource, type: {}, resource alias: {}, desc: {}, file: {},{}", loginUser.getUserName(),type, alias, description, file.getName(), file.getOriginalFilename()); - return resourceService.createResource(loginUser,alias, description,type ,file); + return resourceService.createResource(loginUser,alias, description,type ,file,pid,currentDir); } catch (Exception e) { logger.error(CREATE_RESOURCE_ERROR.getMsg(),e); return error(CREATE_RESOURCE_ERROR.getCode(), CREATE_RESOURCE_ERROR.getMsg()); @@ -108,8 +154,7 @@ public class ResourcesController extends BaseController{ @ApiImplicitParam(name = "id", value = "RESOURCE_ID", required = true, dataType ="Int", example = "100"), @ApiImplicitParam(name = "type", value = "RESOURCE_TYPE", required = true, dataType ="ResourceType"), @ApiImplicitParam(name = "name", value = "RESOURCE_NAME", required = true, dataType ="String"), - @ApiImplicitParam(name = "description", value = "RESOURCE_DESC", dataType ="String"), - @ApiImplicitParam(name = "file", value = "RESOURCE_FILE", required = true,dataType = "MultipartFile") + @ApiImplicitParam(name = "description", value = "RESOURCE_DESC", dataType ="String") }) @PostMapping(value = "/update") public Result updateResource(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @@ -120,7 +165,7 @@ public class ResourcesController extends BaseController{ try { logger.info("login user {}, update resource, type: {}, resource alias: {}, desc: {}", loginUser.getUserName(),type, alias, description); - return resourceService.updateResource(loginUser,resourceId,alias, description,type); + return resourceService.updateResource(loginUser,resourceId,alias,description,type); } catch (Exception e) { logger.error(UPDATE_RESOURCE_ERROR.getMsg(),e); return error(Status.UPDATE_RESOURCE_ERROR.getCode(), Status.UPDATE_RESOURCE_ERROR.getMsg()); @@ -144,7 +189,7 @@ public class ResourcesController extends BaseController{ @RequestParam(value ="type") ResourceType type ){ try{ - logger.info("query resource list, login user:{}, resource type:{}", loginUser.getUserName(), type.toString()); + logger.info("query resource list, login user:{}, resource type:{}", loginUser.getUserName(), type); Map result = resourceService.queryResourceList(loginUser, type); return returnDataList(result); }catch (Exception e){ @@ -166,6 +211,7 @@ public class ResourcesController extends BaseController{ @ApiOperation(value = "queryResourceListPaging", notes= "QUERY_RESOURCE_LIST_PAGING_NOTES") @ApiImplicitParams({ @ApiImplicitParam(name = "type", value = "RESOURCE_TYPE", required = true, dataType ="ResourceType"), + @ApiImplicitParam(name = "id", value = "RESOURCE_ID", required = true, dataType ="int"), @ApiImplicitParam(name = "searchVal", value = "SEARCH_VAL", dataType ="String"), @ApiImplicitParam(name = "pageNo", value = "PAGE_NO", dataType = "Int", example = "1"), @ApiImplicitParam(name = "pageSize", value = "PAGE_SIZE", dataType ="Int",example = "20") @@ -174,20 +220,21 @@ public class ResourcesController extends BaseController{ @ResponseStatus(HttpStatus.OK) public Result queryResourceListPaging(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam(value ="type") ResourceType type, + @RequestParam(value ="id") int id, @RequestParam("pageNo") Integer pageNo, @RequestParam(value = "searchVal", required = false) String searchVal, @RequestParam("pageSize") Integer pageSize ){ try{ logger.info("query resource list, login user:{}, resource type:{}, search value:{}", - loginUser.getUserName(), type.toString(), searchVal); + loginUser.getUserName(), type, searchVal); Map result = checkPageParams(pageNo, pageSize); if(result.get(Constants.STATUS) != Status.SUCCESS){ return returnDataListPaging(result); } searchVal = ParameterUtils.handleEscapes(searchVal); - result = resourceService.queryResourceListPaging(loginUser,type,searchVal,pageNo, pageSize); + result = resourceService.queryResourceListPaging(loginUser,id,type,searchVal,pageNo, pageSize); return returnDataListPaging(result); }catch (Exception e){ logger.error(QUERY_RESOURCES_LIST_PAGING.getMsg(),e); @@ -227,32 +274,89 @@ public class ResourcesController extends BaseController{ * verify resource by alias and type * * @param loginUser login user - * @param alias resource name - * @param type resource type + * @param fullName resource full name + * @param type resource type * @return true if the resource name not exists, otherwise return false */ @ApiOperation(value = "verifyResourceName", notes= "VERIFY_RESOURCE_NAME_NOTES") @ApiImplicitParams({ @ApiImplicitParam(name = "type", value = "RESOURCE_TYPE", required = true, dataType ="ResourceType"), - @ApiImplicitParam(name = "name", value = "RESOURCE_NAME", required = true, dataType ="String") + @ApiImplicitParam(name = "fullName", value = "RESOURCE_FULL_NAME", required = true, dataType ="String") }) @GetMapping(value = "/verify-name") @ResponseStatus(HttpStatus.OK) public Result verifyResourceName(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @RequestParam(value ="name") String alias, + @RequestParam(value ="fullName") String fullName, @RequestParam(value ="type") ResourceType type ) { try { logger.info("login user {}, verfiy resource alias: {},resource type: {}", - loginUser.getUserName(), alias,type); + loginUser.getUserName(), fullName,type); - return resourceService.verifyResourceName(alias,type,loginUser); + return resourceService.verifyResourceName(fullName,type,loginUser); } catch (Exception e) { logger.error(VERIFY_RESOURCE_BY_NAME_AND_TYPE_ERROR.getMsg(), e); return error(Status.VERIFY_RESOURCE_BY_NAME_AND_TYPE_ERROR.getCode(), Status.VERIFY_RESOURCE_BY_NAME_AND_TYPE_ERROR.getMsg()); } } + /** + * query resources jar list + * + * @param loginUser login user + * @param type resource type + * @return resource list + */ + @ApiOperation(value = "queryResourceJarList", notes= "QUERY_RESOURCE_LIST_NOTES") + @ApiImplicitParams({ + @ApiImplicitParam(name = "type", value = "RESOURCE_TYPE", required = true, dataType ="ResourceType") + }) + @GetMapping(value="/list/jar") + @ResponseStatus(HttpStatus.OK) + public Result queryResourceJarList(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + @RequestParam(value ="type") ResourceType type + ){ + try{ + logger.info("query resource list, login user:{}, resource type:{}", loginUser.getUserName(), type.toString()); + Map result = resourceService.queryResourceJarList(loginUser, type); + return returnDataList(result); + }catch (Exception e){ + logger.error(QUERY_RESOURCES_LIST_ERROR.getMsg(),e); + return error(Status.QUERY_RESOURCES_LIST_ERROR.getCode(), Status.QUERY_RESOURCES_LIST_ERROR.getMsg()); + } + } + + /** + * query resource by full name and type + * + * @param loginUser login user + * @param fullName resource full name + * @param type resource type + * @return true if the resource name not exists, otherwise return false + */ + @ApiOperation(value = "queryResource", notes= "QUERY_BY_RESOURCE_NAME") + @ApiImplicitParams({ + @ApiImplicitParam(name = "type", value = "RESOURCE_TYPE", required = true, dataType ="ResourceType"), + @ApiImplicitParam(name = "fullName", value = "RESOURCE_FULL_NAME", required = true, dataType ="String") + }) + @GetMapping(value = "/queryResource") + @ResponseStatus(HttpStatus.OK) + public Result queryResource(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + @RequestParam(value ="fullName",required = false) String fullName, + @RequestParam(value ="id",required = false) Integer id, + @RequestParam(value ="type") ResourceType type + ) { + try { + logger.info("login user {}, query resource by full name: {} or id: {},resource type: {}", + loginUser.getUserName(), fullName,id,type); + + return resourceService.queryResource(fullName,id,type); + } catch (Exception e) { + logger.error(RESOURCE_NOT_EXIST.getMsg(), e); + return error(Status.RESOURCE_NOT_EXIST.getCode(), Status.RESOURCE_NOT_EXIST.getMsg()); + } + } + /** * view resource file online * @@ -310,16 +414,18 @@ public class ResourcesController extends BaseController{ @RequestParam(value ="fileName")String fileName, @RequestParam(value ="suffix")String fileSuffix, @RequestParam(value = "description", required = false) String description, - @RequestParam(value = "content") String content + @RequestParam(value = "content") String content, + @RequestParam(value ="pid") int pid, + @RequestParam(value ="currentDir") String currentDir ) { try{ logger.info("login user {}, online create resource! fileName : {}, type : {}, suffix : {},desc : {},content : {}", - loginUser.getUserName(),fileName,type,fileSuffix,description,content); + loginUser.getUserName(),fileName,type,fileSuffix,description,content,pid,currentDir); if(StringUtils.isEmpty(content)){ logger.error("resource file contents are not allowed to be empty"); return error(Status.RESOURCE_FILE_IS_EMPTY.getCode(), RESOURCE_FILE_IS_EMPTY.getMsg()); } - return resourceService.onlineCreateResource(loginUser,type,fileName,fileSuffix,description,content); + return resourceService.onlineCreateResource(loginUser,type,fileName,fileSuffix,description,content,pid,currentDir); }catch (Exception e){ logger.error(CREATE_RESOURCE_FILE_ON_LINE_ERROR.getMsg(),e); return error(Status.CREATE_RESOURCE_FILE_ON_LINE_ERROR.getCode(), Status.CREATE_RESOURCE_FILE_ON_LINE_ERROR.getMsg()); @@ -384,6 +490,9 @@ public class ResourcesController extends BaseController{ .ok() .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + file.getFilename() + "\"") .body(file); + }catch (RuntimeException e){ + logger.error(e.getMessage(),e); + return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(e.getMessage()); }catch (Exception e){ logger.error(DOWNLOAD_RESOURCE_FILE_ERROR.getMsg(),e); return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(Status.DOWNLOAD_RESOURCE_FILE_ERROR.getMsg()); @@ -426,8 +535,6 @@ public class ResourcesController extends BaseController{ @RequestParam(value = "resourceId") int resourceId) { logger.info("login user {}, create udf function, type: {}, funcName: {},argTypes: {} ,database: {},desc: {},resourceId: {}", loginUser.getUserName(),type, funcName, argTypes,database,description, resourceId); - Result result = new Result(); - try { return udfFuncService.createUdfFunction(loginUser,funcName,className,argTypes,database,description,type,resourceId); } catch (Exception e) { @@ -563,7 +670,7 @@ public class ResourcesController extends BaseController{ public Result queryResourceList(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam("type") UdfType type){ try{ - logger.info("query datasource list, user:{}, type:{}", loginUser.getUserName(), type.toString()); + logger.info("query datasource list, user:{}, type:{}", loginUser.getUserName(), type); Map result = udfFuncService.queryResourceList(loginUser,type.ordinal()); return returnDataList(result); }catch (Exception e){ @@ -660,21 +767,21 @@ public class ResourcesController extends BaseController{ * @param userId user id * @return unauthorized result code */ - @ApiOperation(value = "unauthorizedFile", notes= "UNAUTHORIZED_FILE_NOTES") + @ApiOperation(value = "authorizeResourceTree", notes= "AUTHORIZE_RESOURCE_TREE_NOTES") @ApiImplicitParams({ @ApiImplicitParam(name = "userId", value = "USER_ID", required = true, dataType ="Int", example = "100") }) - @GetMapping(value = "/unauth-file") + @GetMapping(value = "/authorize-resource-tree") @ResponseStatus(HttpStatus.CREATED) - public Result unauthorizedFile(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + public Result authorizeResourceTree(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam("userId") Integer userId) { try{ - logger.info("resource unauthorized file, user:{}, unauthorized user id:{}", loginUser.getUserName(), userId); - Map result = resourceService.unauthorizedFile(loginUser, userId); + logger.info("all resource file, user:{}, user id:{}", loginUser.getUserName(), userId); + Map result = resourceService.authorizeResourceTree(loginUser, userId); return returnDataList(result); }catch (Exception e){ - logger.error(UNAUTHORIZED_FILE_RESOURCE_ERROR.getMsg(),e); - return error(Status.UNAUTHORIZED_FILE_RESOURCE_ERROR.getCode(), Status.UNAUTHORIZED_FILE_RESOURCE_ERROR.getMsg()); + logger.error(AUTHORIZE_RESOURCE_TREE.getMsg(),e); + return error(Status.AUTHORIZE_RESOURCE_TREE.getCode(), Status.AUTHORIZE_RESOURCE_TREE.getMsg()); } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/ProcessMeta.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/ProcessMeta.java index f14d8df097..bdb9e1f576 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/ProcessMeta.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/ProcessMeta.java @@ -106,9 +106,6 @@ public class ProcessMeta { */ private String scheduleWorkerGroupName; - public ProcessMeta() { - } - public String getProjectName() { return projectName; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/TaskCountDto.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/TaskCountDto.java index e7b182076d..6b0391f111 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/TaskCountDto.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/TaskCountDto.java @@ -43,36 +43,36 @@ public class TaskCountDto { } private void countTaskDtos(List taskInstanceStateCounts){ - int submitted_success = 0; - int running_exeution = 0; - int ready_pause = 0; + int submittedSuccess = 0; + int runningExeution = 0; + int readyPause = 0; int pause = 0; - int ready_stop = 0; + int readyStop = 0; int stop = 0; int failure = 0; int success = 0; - int need_fault_tolerance = 0; + int needFaultTolerance = 0; int kill = 0; - int waitting_thread = 0; + int waittingThread = 0; for(ExecuteStatusCount taskInstanceStateCount : taskInstanceStateCounts){ ExecutionStatus status = taskInstanceStateCount.getExecutionStatus(); totalCount += taskInstanceStateCount.getCount(); switch (status){ case SUBMITTED_SUCCESS: - submitted_success += taskInstanceStateCount.getCount(); + submittedSuccess += taskInstanceStateCount.getCount(); break; case RUNNING_EXEUTION: - running_exeution += taskInstanceStateCount.getCount(); + runningExeution += taskInstanceStateCount.getCount(); break; case READY_PAUSE: - ready_pause += taskInstanceStateCount.getCount(); + readyPause += taskInstanceStateCount.getCount(); break; case PAUSE: pause += taskInstanceStateCount.getCount(); break; case READY_STOP: - ready_stop += taskInstanceStateCount.getCount(); + readyStop += taskInstanceStateCount.getCount(); break; case STOP: stop += taskInstanceStateCount.getCount(); @@ -84,13 +84,13 @@ public class TaskCountDto { success += taskInstanceStateCount.getCount(); break; case NEED_FAULT_TOLERANCE: - need_fault_tolerance += taskInstanceStateCount.getCount(); + needFaultTolerance += taskInstanceStateCount.getCount(); break; case KILL: kill += taskInstanceStateCount.getCount(); break; case WAITTING_THREAD: - waitting_thread += taskInstanceStateCount.getCount(); + waittingThread += taskInstanceStateCount.getCount(); break; default: @@ -98,17 +98,17 @@ public class TaskCountDto { } } this.taskCountDtos = new ArrayList<>(); - this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.SUBMITTED_SUCCESS, submitted_success)); - this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.RUNNING_EXEUTION, running_exeution)); - this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.READY_PAUSE, ready_pause)); + this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.SUBMITTED_SUCCESS, submittedSuccess)); + this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.RUNNING_EXEUTION, runningExeution)); + this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.READY_PAUSE, readyPause)); this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.PAUSE, pause)); - this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.READY_STOP, ready_stop)); + this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.READY_STOP, readyStop)); this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.STOP, stop)); this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.FAILURE, failure)); this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.SUCCESS, success)); - this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.NEED_FAULT_TOLERANCE, need_fault_tolerance)); + this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.NEED_FAULT_TOLERANCE, needFaultTolerance)); this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.KILL, kill)); - this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.WAITTING_THREAD, waitting_thread)); + this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.WAITTING_THREAD, waittingThread)); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/Directory.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/Directory.java new file mode 100644 index 0000000000..289d5060bf --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/Directory.java @@ -0,0 +1,29 @@ +package org.apache.dolphinscheduler.api.dto.resources; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * directory + */ +public class Directory extends ResourceComponent{ + + @Override + public boolean isDirctory() { + return true; + } + +} diff --git a/dolphinscheduler-ui/src/sass/common/_mixin.scss b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/FileLeaf.java similarity index 85% rename from dolphinscheduler-ui/src/sass/common/_mixin.scss rename to dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/FileLeaf.java index c6a5afeef5..b9b91821f4 100644 --- a/dolphinscheduler-ui/src/sass/common/_mixin.scss +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/FileLeaf.java @@ -1,3 +1,5 @@ +package org.apache.dolphinscheduler.api.dto.resources; + /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -13,4 +15,10 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - */ \ No newline at end of file + */ +/** + * file leaf + */ +public class FileLeaf extends ResourceComponent{ + +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/ResourceComponent.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/ResourceComponent.java new file mode 100644 index 0000000000..fb0da702b3 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/ResourceComponent.java @@ -0,0 +1,193 @@ +package org.apache.dolphinscheduler.api.dto.resources; + +import com.alibaba.fastjson.annotation.JSONField; +import com.alibaba.fastjson.annotation.JSONType; +import org.apache.dolphinscheduler.common.enums.ResourceType; + +import java.util.ArrayList; +import java.util.List; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * resource component + */ +@JSONType(orders={"id","pid","name","fullName","description","isDirctory","children","type"}) +public abstract class ResourceComponent { + public ResourceComponent() { + } + + public ResourceComponent(int id, int pid, String name, String fullName, String description, boolean isDirctory) { + this.id = id; + this.pid = pid; + this.name = name; + this.fullName = fullName; + this.description = description; + this.isDirctory = isDirctory; + int directoryFlag = isDirctory ? 1:0; + this.idValue = String.format("%s_%s",id,directoryFlag); + } + + + /** + * id + */ + @JSONField(ordinal = 1) + protected int id; + /** + * parent id + */ + @JSONField(ordinal = 2) + protected int pid; + /** + * name + */ + @JSONField(ordinal = 3) + protected String name; + /** + * current directory + */ + protected String currentDir; + /** + * full name + */ + @JSONField(ordinal = 4) + protected String fullName; + /** + * description + */ + @JSONField(ordinal = 5) + protected String description; + /** + * is directory + */ + @JSONField(ordinal = 6) + protected boolean isDirctory; + /** + * id value + */ + @JSONField(ordinal = 7) + protected String idValue; + /** + * resoruce type + */ + @JSONField(ordinal = 8) + protected ResourceType type; + /** + * children + */ + @JSONField(ordinal = 8) + protected List children = new ArrayList<>(); + + /** + * add resource component + * @param resourceComponent resource component + */ + public void add(ResourceComponent resourceComponent){ + children.add(resourceComponent); + } + + public String getName(){ + return this.name; + } + + public String getDescription(){ + return this.description; + } + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + + public int getPid() { + return pid; + } + + public void setPid(int pid) { + this.pid = pid; + } + + public void setName(String name) { + this.name = name; + } + + public String getFullName() { + return fullName; + } + + public void setFullName(String fullName) { + this.fullName = fullName; + } + + public void setDescription(String description) { + this.description = description; + } + + public boolean isDirctory() { + return isDirctory; + } + + public void setDirctory(boolean dirctory) { + isDirctory = dirctory; + } + + public String getIdValue() { + return idValue; + } + + public void setIdValue(int id,boolean isDirctory) { + int directoryFlag = isDirctory ? 1:0; + this.idValue = String.format("%s_%s",id,directoryFlag); + } + + public ResourceType getType() { + return type; + } + + public void setType(ResourceType type) { + this.type = type; + } + + public List getChildren() { + return children; + } + + public void setChildren(List children) { + this.children = children; + } + + @Override + public String toString() { + return "ResourceComponent{" + + "id=" + id + + ", pid=" + pid + + ", name='" + name + '\'' + + ", currentDir='" + currentDir + '\'' + + ", fullName='" + fullName + '\'' + + ", description='" + description + '\'' + + ", isDirctory=" + isDirctory + + ", idValue='" + idValue + '\'' + + ", type=" + type + + ", children=" + children + + '}'; + } + +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/IFilter.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/IFilter.java new file mode 100644 index 0000000000..ce6ce3a011 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/IFilter.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.api.dto.resources.filter; + +import org.apache.dolphinscheduler.dao.entity.Resource; + +import java.util.List; + +/** + * interface filter + */ +public interface IFilter { + List filter(); +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilter.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilter.java new file mode 100644 index 0000000000..c918a160af --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilter.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.api.dto.resources.filter; + +import org.apache.dolphinscheduler.dao.entity.Resource; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * resource filter + */ +public class ResourceFilter implements IFilter { + /** + * resource suffix + */ + private String suffix; + /** + * resource list + */ + private List resourceList; + + /** + * parent list + */ + //Set parentList = new HashSet<>(); + + /** + * constructor + * @param suffix resource suffix + * @param resourceList resource list + */ + public ResourceFilter(String suffix, List resourceList) { + this.suffix = suffix; + this.resourceList = resourceList; + } + + /** + * file filter + * @return file filtered by suffix + */ + public Set fileFilter(){ + Set resources = resourceList.stream().filter(t -> { + String alias = t.getAlias(); + return alias.endsWith(suffix); + }).collect(Collectors.toSet()); + return resources; + } + + /** + * list all parent dir + * @return parent resource dir set + */ + Set listAllParent(){ + Set parentList = new HashSet<>(); + Set filterFileList = fileFilter(); + for(Resource file:filterFileList){ + parentList.add(file); + setAllParent(file,parentList); + } + return parentList; + + } + + /** + * list all parent dir + * @param resource resource + * @return parent resource dir set + */ + private void setAllParent(Resource resource,Set parentList){ + for (Resource resourceTemp : resourceList) { + if (resourceTemp.getId() == resource.getPid()) { + parentList.add(resourceTemp); + setAllParent(resourceTemp,parentList); + } + } + } + + @Override + public List filter() { + return new ArrayList<>(listAllParent()); + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitor.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitor.java new file mode 100644 index 0000000000..5cf118800a --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitor.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.api.dto.resources.visitor; + + +import org.apache.dolphinscheduler.api.dto.resources.Directory; +import org.apache.dolphinscheduler.api.dto.resources.FileLeaf; +import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent; +import org.apache.dolphinscheduler.dao.entity.Resource; + +import java.util.ArrayList; +import java.util.List; + +/** + * resource tree visitor + */ +public class ResourceTreeVisitor implements Visitor{ + + /** + * resource list + */ + private List resourceList; + + public ResourceTreeVisitor() { + } + + /** + * constructor + * @param resourceList resource list + */ + public ResourceTreeVisitor(List resourceList) { + this.resourceList = resourceList; + } + + /** + * visit + * @return resoruce component + */ + public ResourceComponent visit() { + ResourceComponent rootDirectory = new Directory(); + for (Resource resource : resourceList) { + // judge whether is root node + if (rootNode(resource)){ + ResourceComponent tempResourceComponent = getResourceComponent(resource); + rootDirectory.add(tempResourceComponent); + tempResourceComponent.setChildren(setChildren(tempResourceComponent.getId(),resourceList)); + } + } + return rootDirectory; + } + + /** + * set children + * @param id id + * @param list resource list + * @return resource component list + */ + public static List setChildren(int id, List list ){ + List childList = new ArrayList<>(); + for (Resource resource : list) { + if (id == resource.getPid()){ + ResourceComponent tempResourceComponent = getResourceComponent(resource); + childList.add(tempResourceComponent); + } + } + for (ResourceComponent resourceComponent : childList) { + resourceComponent.setChildren(setChildren(resourceComponent.getId(),list)); + } + if (childList.size()==0){ + return new ArrayList<>(); + } + return childList; + } + + /** + * Determine whether it is the root node + * @param resource resource + * @return true if it is the root node + */ + public boolean rootNode(Resource resource) { + + boolean isRootNode = true; + if(resource.getPid() != -1 ){ + for (Resource parent : resourceList) { + if (resource.getPid() == parent.getId()) { + isRootNode = false; + break; + } + } + } + return isRootNode; + } + + /** + * get resource component by resource + * @param resource resource + * @return resource component + */ + private static ResourceComponent getResourceComponent(Resource resource) { + ResourceComponent tempResourceComponent; + if(resource.isDirectory()){ + tempResourceComponent = new Directory(); + }else{ + tempResourceComponent = new FileLeaf(); + } + tempResourceComponent.setName(resource.getAlias()); + tempResourceComponent.setFullName(resource.getFullName().replaceFirst("/","")); + tempResourceComponent.setId(resource.getId()); + tempResourceComponent.setPid(resource.getPid()); + tempResourceComponent.setIdValue(resource.getId(),resource.isDirectory()); + tempResourceComponent.setDescription(resource.getDescription()); + tempResourceComponent.setType(resource.getType()); + return tempResourceComponent; + } + +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/Visitor.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/Visitor.java new file mode 100644 index 0000000000..3dfce7c7c1 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/Visitor.java @@ -0,0 +1,31 @@ +package org.apache.dolphinscheduler.api.dto.resources.visitor; + + +import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Visitor + */ +public interface Visitor { + /** + * visit + * @return resource component + */ + ResourceComponent visit(); +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java index 9955463f8e..416dc0ef54 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java @@ -97,7 +97,7 @@ public enum Status { VERIFY_UDF_FUNCTION_NAME_ERROR( 10070,"verify udf function name error", "UDF函数名称验证错误"), DELETE_UDF_FUNCTION_ERROR( 10071,"delete udf function error", "删除UDF函数错误"), AUTHORIZED_FILE_RESOURCE_ERROR( 10072,"authorized file resource error", "授权资源文件错误"), - UNAUTHORIZED_FILE_RESOURCE_ERROR( 10073,"unauthorized file resource error", "查询未授权资源错误"), + AUTHORIZE_RESOURCE_TREE( 10073,"authorize resource tree display error","授权资源目录树错误"), UNAUTHORIZED_UDF_FUNCTION_ERROR( 10074,"unauthorized udf function error", "查询未授权UDF函数错误"), AUTHORIZED_UDF_FUNCTION_ERROR(10075,"authorized udf function error", "授权UDF函数错误"), CREATE_SCHEDULE_ERROR(10076,"create schedule error", "创建调度配置错误"), @@ -184,10 +184,12 @@ public enum Status { RESOURCE_SIZE_EXCEED_LIMIT(20007, "upload resource file size exceeds limit", "上传资源文件大小超过限制"), RESOURCE_SUFFIX_FORBID_CHANGE(20008, "resource suffix not allowed to be modified", "资源文件后缀不支持修改"), UDF_RESOURCE_SUFFIX_NOT_JAR(20009, "UDF resource suffix name must be jar", "UDF资源文件后缀名只支持[jar]"), - HDFS_COPY_FAIL(20009, "hdfs copy {0} -> {1} fail", "hdfs复制失败：[{0}] -> [{1}]"), - RESOURCE_FILE_EXIST(20010, "resource file {0} already exists in hdfs,please delete it or change name!", "资源文件[{0}]在hdfs中已存在，请删除或修改资源名"), - RESOURCE_FILE_NOT_EXIST(20011, "resource file {0} not exists in hdfs!", "资源文件[{0}]在hdfs中不存在"), - + HDFS_COPY_FAIL(20010, "hdfs copy {0} -> {1} fail", "hdfs复制失败：[{0}] -> [{1}]"), + RESOURCE_FILE_EXIST(20011, "resource file {0} already exists in hdfs,please delete it or change name!", "资源文件[{0}]在hdfs中已存在，请删除或修改资源名"), + RESOURCE_FILE_NOT_EXIST(20012, "resource file {0} not exists in hdfs!", "资源文件[{0}]在hdfs中不存在"), + UDF_RESOURCE_IS_BOUND(20013, "udf resource file is bound by UDF functions:{0}","udf函数绑定了资源文件[{0}]"), + RESOURCE_IS_USED(20014, "resource file is used by process definition","资源文件被上线的流程定义使用了"), + PARENT_RESOURCE_NOT_EXIST(20015, "parent resource not exist","父资源文件不存在"), USER_NO_OPERATION_PERM(30001, "user has no operation privilege", "当前用户没有操作权限"), diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertGroupService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertGroupService.java index 70310b6331..001a10d08a 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertGroupService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertGroupService.java @@ -16,17 +16,17 @@ */ package org.apache.dolphinscheduler.api.service; +import java.util.*; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.PageInfo; -import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.AlertType; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.dao.entity.AlertGroup; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.entity.UserAlertGroup; import org.apache.dolphinscheduler.dao.mapper.AlertGroupMapper; -import org.apache.dolphinscheduler.dao.mapper.UserAlertGroupMapper; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import org.slf4j.Logger; @@ -35,11 +35,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - /** * alert group service */ @@ -52,8 +47,7 @@ public class AlertGroupService extends BaseService{ private AlertGroupMapper alertGroupMapper; @Autowired - private UserAlertGroupMapper userAlertGroupMapper; - + private UserAlertGroupService userAlertGroupService; /** * query alert group list * @@ -122,7 +116,7 @@ public class AlertGroupService extends BaseService{ alertGroup.setCreateTime(now); alertGroup.setUpdateTime(now); - // insert + // insert int insert = alertGroupMapper.insert(alertGroup); if (insert > 0) { @@ -199,7 +193,7 @@ public class AlertGroupService extends BaseService{ return result; } - userAlertGroupMapper.deleteByAlertgroupId(id); + userAlertGroupService.deleteByAlertGroupId(id); alertGroupMapper.deleteById(id); putMsg(result, Status.SUCCESS); return result; @@ -223,22 +217,26 @@ public class AlertGroupService extends BaseService{ return result; } - userAlertGroupMapper.deleteByAlertgroupId(alertgroupId); + userAlertGroupService.deleteByAlertGroupId(alertgroupId); if (StringUtils.isEmpty(userIds)) { putMsg(result, Status.SUCCESS); return result; } String[] userIdsArr = userIds.split(","); - + Date now = new Date(); + List alertGroups = new ArrayList<>(userIds.length()); for (String userId : userIdsArr) { - Date now = new Date(); UserAlertGroup userAlertGroup = new UserAlertGroup(); userAlertGroup.setAlertgroupId(alertgroupId); userAlertGroup.setUserId(Integer.parseInt(userId)); userAlertGroup.setCreateTime(now); userAlertGroup.setUpdateTime(now); - userAlertGroupMapper.insert(userAlertGroup); + alertGroups.add(userAlertGroup); + } + + if (CollectionUtils.isNotEmpty(alertGroups)) { + userAlertGroupService.saveBatch(alertGroups); } putMsg(result, Status.SUCCESS); @@ -248,22 +246,11 @@ public class AlertGroupService extends BaseService{ /** * verify group name exists * - * @param loginUser login user * @param groupName group name * @return check result code */ - public Result verifyGroupName(User loginUser, String groupName) { - Result result = new Result(); + public boolean existGroupName(String groupName) { List alertGroup = alertGroupMapper.queryByGroupName(groupName); - if (alertGroup != null && alertGroup.size() > 0) { - logger.error("group {} has exist, can't create again.", groupName); - result.setCode(Status.ALERT_GROUP_EXIST.getCode()); - result.setMsg(Status.ALERT_GROUP_EXIST.getMsg()); - } else { - result.setCode(Status.SUCCESS.getCode()); - result.setMsg(Status.SUCCESS.getMsg()); - } - - return result; + return CollectionUtils.isNotEmpty(alertGroup); } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java index 6a732fed0e..afa13b7414 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java @@ -17,10 +17,15 @@ package org.apache.dolphinscheduler.api.service; import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.TypeReference; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.DbConnectType; import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; @@ -30,10 +35,6 @@ import org.apache.dolphinscheduler.dao.entity.Resource; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper; import org.apache.dolphinscheduler.dao.mapper.DataSourceUserMapper; -import com.alibaba.fastjson.JSONObject; -import com.alibaba.fastjson.TypeReference; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; @@ -473,12 +474,19 @@ public class DataSourceService extends BaseService{ * @return datasource parameter */ public String buildParameter(String name, String desc, DbType type, String host, - String port, String database,String principal,String userName, - String password, String other) { + String port, String database, String principal, String userName, + String password, DbConnectType connectType, String other) { + + String address = buildAddress(type, host, port, connectType); - String address = buildAddress(type, host, port); + String jdbcUrl; + if (Constants.ORACLE.equals(type.name()) + && connectType == DbConnectType.ORACLE_SID) { + jdbcUrl = address + ":" + database; + } else { + jdbcUrl = address + "/" + database; + } - String jdbcUrl = address + "/" + database; if (CommonUtils.getKerberosStartupState() && (type == DbType.HIVE || type == DbType.SPARK)){ jdbcUrl += ";principal=" + principal; @@ -531,7 +539,7 @@ public class DataSourceService extends BaseService{ } - private String buildAddress(DbType type, String host, String port) { + private String buildAddress(DbType type, String host, String port, DbConnectType connectType) { StringBuilder sb = new StringBuilder(); if (Constants.MYSQL.equals(type.name())) { sb.append(Constants.JDBC_MYSQL); @@ -552,7 +560,11 @@ public class DataSourceService extends BaseService{ sb.append(Constants.JDBC_CLICKHOUSE); sb.append(host).append(":").append(port); } else if (Constants.ORACLE.equals(type.name())) { - sb.append(Constants.JDBC_ORACLE); + if (connectType == DbConnectType.ORACLE_SID) { + sb.append(Constants.JDBC_ORACLE_SID); + } else { + sb.append(Constants.JDBC_ORACLE_SERVICE_NAME); + } sb.append(host).append(":").append(port); } else if (Constants.SQLSERVER.equals(type.name())) { sb.append(Constants.JDBC_SQLSERVER); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java index 86b507f0a0..dda960d2e6 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java @@ -98,7 +98,7 @@ public class ExecutorService extends BaseService{ String receivers, String receiversCc, RunMode runMode, Priority processInstancePriority, int workerGroupId, Integer timeout) throws ParseException { Map result = new HashMap<>(5); - // timeout is valid + // timeout is invalid if (timeout <= 0 || timeout > MAX_TASK_TIMEOUT) { putMsg(result,Status.TASK_TIMEOUT_PARAMS_ERROR); return result; @@ -453,7 +453,7 @@ public class ExecutorService extends BaseService{ TaskDependType nodeDep, FailureStrategy failureStrategy, String startNodeList, String schedule, WarningType warningType, int excutorId, int warningGroupId, - RunMode runMode,Priority processInstancePriority, int workerGroupId) throws ParseException { + RunMode runMode,Priority processInstancePriority, int workerGroupId){ /** * instantiate command schedule instance @@ -496,6 +496,7 @@ public class ExecutorService extends BaseService{ } } + // determine whether to complement if(commandType == CommandType.COMPLEMENT_DATA){ runMode = (runMode == null) ? RunMode.RUN_MODE_SERIAL : runMode; if(null != start && null != end && start.before(end)){ @@ -540,7 +541,7 @@ public class ExecutorService extends BaseService{ processDefineId, schedule); } }else{ - command.setCommandParam(JSONUtils.toJson(cmdParam)); + return processService.createCommand(command); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java index 22e3593a52..0639aba113 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java @@ -38,11 +38,9 @@ import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.model.TaskNodeRelation; import org.apache.dolphinscheduler.common.process.ProcessDag; import org.apache.dolphinscheduler.common.process.Property; +import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.thread.Stopper; -import org.apache.dolphinscheduler.common.utils.CollectionUtils; -import org.apache.dolphinscheduler.common.utils.DateUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.dao.entity.*; import org.apache.dolphinscheduler.dao.mapper.*; import org.apache.dolphinscheduler.dao.utils.DagHelper; @@ -148,7 +146,7 @@ public class ProcessDefinitionService extends BaseDAGService { //custom global params List globalParamsList = processData.getGlobalParams(); - if (globalParamsList != null && globalParamsList.size() > 0) { + if (CollectionUtils.isNotEmpty(globalParamsList)) { Set globalParamsSet = new HashSet<>(globalParamsList); globalParamsList = new ArrayList<>(globalParamsSet); processDefine.setGlobalParamList(globalParamsList); @@ -162,6 +160,31 @@ public class ProcessDefinitionService extends BaseDAGService { return result; } + /** + * get resource ids + * @param processData process data + * @return resource ids + */ + private String getResourceIds(ProcessData processData) { + List tasks = processData.getTasks(); + Set resourceIds = new HashSet<>(); + for(TaskNode taskNode : tasks){ + String taskParameter = taskNode.getParams(); + AbstractParameters params = TaskParametersUtils.getParameters(taskNode.getType(),taskParameter); + Set tempSet = params.getResourceFilesList().stream().map(t->t.getId()).collect(Collectors.toSet()); + resourceIds.addAll(tempSet); + } + + StringBuilder sb = new StringBuilder(); + for(int i : resourceIds) { + if (sb.length() > 0) { + sb.append(","); + } + sb.append(i); + } + return sb.toString(); + } + /** * query proccess definition list @@ -284,20 +307,19 @@ public class ProcessDefinitionService extends BaseDAGService { if ((checkProcessJson.get(Constants.STATUS) != Status.SUCCESS)) { return checkProcessJson; } - ProcessDefinition processDefinition = processService.findProcessDefineById(id); - if (processDefinition == null) { + ProcessDefinition processDefine = processService.findProcessDefineById(id); + if (processDefine == null) { // check process definition exists putMsg(result, Status.PROCESS_DEFINE_NOT_EXIST, id); return result; - } else if (processDefinition.getReleaseState() == ReleaseState.ONLINE) { + } else if (processDefine.getReleaseState() == ReleaseState.ONLINE) { // online can not permit edit - putMsg(result, Status.PROCESS_DEFINE_NOT_ALLOWED_EDIT, processDefinition.getName()); + putMsg(result, Status.PROCESS_DEFINE_NOT_ALLOWED_EDIT, processDefine.getName()); return result; } else { putMsg(result, Status.SUCCESS); } - ProcessDefinition processDefine = processService.findProcessDefineById(id); Date now = new Date(); processDefine.setId(id); @@ -314,7 +336,7 @@ public class ProcessDefinitionService extends BaseDAGService { //custom global params List globalParamsList = new ArrayList<>(); - if (processData.getGlobalParams() != null && processData.getGlobalParams().size() > 0) { + if (CollectionUtils.isNotEmpty(processData.getGlobalParams())) { Set userDefParamsSet = new HashSet<>(processData.getGlobalParams()); globalParamsList = new ArrayList<>(userDefParamsSet); } @@ -453,12 +475,11 @@ public class ProcessDefinitionService extends BaseDAGService { ProcessDefinition processDefinition = processDefineMapper.selectById(id); switch (state) { - case ONLINE: { + case ONLINE: processDefinition.setReleaseState(state); processDefineMapper.updateById(processDefinition); break; - } - case OFFLINE: { + case OFFLINE: processDefinition.setReleaseState(state); processDefineMapper.updateById(processDefinition); List scheduleList = scheduleMapper.selectAllByProcessDefineArray( @@ -473,11 +494,9 @@ public class ProcessDefinitionService extends BaseDAGService { SchedulerService.deleteSchedule(project.getId(), schedule.getId()); } break; - } - default: { + default: putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "releaseState"); return result; - } } putMsg(result, Status.SUCCESS); @@ -949,7 +968,9 @@ public class ProcessDefinitionService extends BaseDAGService { return result; } + String processDefinitionJson = processDefinition.getProcessDefinitionJson(); + ProcessData processData = JSONUtils.parseObject(processDefinitionJson, ProcessData.class); //process data check @@ -1166,6 +1187,7 @@ public class ProcessDefinitionService extends BaseDAGService { private DAG genDagGraph(ProcessDefinition processDefinition) throws Exception { String processDefinitionJson = processDefinition.getProcessDefinitionJson(); + ProcessData processData = JSONUtils.parseObject(processDefinitionJson, ProcessData.class); //check process data diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java index 3530c66ff6..ab5580cc3e 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java @@ -204,14 +204,8 @@ public class ProcessInstanceService extends BaseDAGService { } } - Set exclusionSet = new HashSet<>(); - exclusionSet.add(Constants.CLASS); - exclusionSet.add("locations"); - exclusionSet.add("connects"); - exclusionSet.add("processInstanceJson"); - pageInfo.setTotalCount((int) processInstanceList.getTotal()); - pageInfo.setLists(CollectionUtils.getListByExclusion(processInstances, exclusionSet)); + pageInfo.setLists(processInstances); result.put(Constants.DATA_LIST, pageInfo); putMsg(result, Status.SUCCESS); return result; @@ -239,7 +233,7 @@ public class ProcessInstanceService extends BaseDAGService { } ProcessInstance processInstance = processService.findProcessInstanceDetailById(processId); List taskInstanceList = processService.findValidTaskListByProcessId(processId); - AddDependResultForTaskList(taskInstanceList); + addDependResultForTaskList(taskInstanceList); Map resultMap = new HashMap<>(); resultMap.put(PROCESS_INSTANCE_STATE, processInstance.getState().toString()); resultMap.put(TASK_LIST, taskInstanceList); @@ -253,9 +247,9 @@ public class ProcessInstanceService extends BaseDAGService { * add dependent result for dependent task * @param taskInstanceList */ - private void AddDependResultForTaskList(List taskInstanceList) throws IOException { + private void addDependResultForTaskList(List taskInstanceList) throws IOException { for(TaskInstance taskInstance: taskInstanceList){ - if(taskInstance.getTaskType().toUpperCase().equals(TaskType.DEPENDENT.toString())){ + if(taskInstance.getTaskType().equalsIgnoreCase(TaskType.DEPENDENT.toString())){ Result logResult = loggerService.queryLog( taskInstance.getId(), 0, 4098); if(logResult.getCode() == Status.SUCCESS.ordinal()){ @@ -414,11 +408,10 @@ public class ProcessInstanceService extends BaseDAGService { processInstance.setProcessInstanceJson(processInstanceJson); processInstance.setGlobalParams(globalParams); } -// int update = processDao.updateProcessInstance(processInstanceId, processInstanceJson, -// globalParams, schedule, flag, locations, connects); + int update = processService.updateProcessInstance(processInstance); int updateDefine = 1; - if (syncDefine && StringUtils.isNotEmpty(processInstanceJson)) { + if (Boolean.TRUE.equals(syncDefine) && StringUtils.isNotEmpty(processInstanceJson)) { processDefinition.setProcessDefinitionJson(processInstanceJson); processDefinition.setGlobalParams(originDefParams); processDefinition.setLocations(locations); @@ -544,7 +537,7 @@ public class ProcessInstanceService extends BaseDAGService { nodeValueSb.append(ipSb); } - logger.info("delete task queue node : {}",nodeValueSb.toString()); + logger.info("delete task queue node : {}",nodeValueSb); tasksQueue.removeNode(org.apache.dolphinscheduler.common.Constants.DOLPHINSCHEDULER_TASKS_QUEUE, nodeValueSb.toString()); } @@ -621,7 +614,7 @@ public class ProcessInstanceService extends BaseDAGService { Map localParamsMap = new HashMap<>(); localParamsMap.put("taskType",taskNode.getType()); localParamsMap.put("localParamsList",localParamsList); - if (localParamsList.size() > 0) { + if (CollectionUtils.isNotEmpty(localParamsList)) { localUserDefParams.put(taskNode.getName(), localParamsMap); } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/QueueService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/QueueService.java index 862c895c92..cba1b5f2bb 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/QueueService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/QueueService.java @@ -20,6 +20,7 @@ import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.dao.entity.Queue; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.QueueMapper; @@ -43,7 +44,7 @@ import java.util.Map; @Service public class QueueService extends BaseService { - private static final Logger logger = LoggerFactory.getLogger(TenantService.class); + private static final Logger logger = LoggerFactory.getLogger(QueueService.class); @Autowired private QueueMapper queueMapper; @@ -186,19 +187,16 @@ public class QueueService extends BaseService { } // check queue name is exist - if (!queueName.equals(queueObj.getQueueName())) { - if (checkQueueNameExist(queueName)) { - putMsg(result, Status.QUEUE_NAME_EXIST, queueName); - return result; - } + if (!queueName.equals(queueObj.getQueueName()) + && checkQueueNameExist(queueName)) { + putMsg(result, Status.QUEUE_NAME_EXIST, queueName); + return result; } // check queue value is exist - if (!queue.equals(queueObj.getQueue())) { - if (checkQueueExist(queue)) { - putMsg(result, Status.QUEUE_VALUE_EXIST, queue); - return result; - } + if (!queue.equals(queueObj.getQueue()) && checkQueueExist(queue)) { + putMsg(result, Status.QUEUE_VALUE_EXIST, queue); + return result; } // check old queue using by any user @@ -267,7 +265,7 @@ public class QueueService extends BaseService { * @return true if the queue not exists, otherwise return false */ private boolean checkQueueExist(String queue) { - return queueMapper.queryAllQueueList(queue, null).size() > 0; + return CollectionUtils.isNotEmpty(queueMapper.queryAllQueueList(queue, null)); } /** @@ -278,7 +276,7 @@ public class QueueService extends BaseService { * @return true if the queue name not exists, otherwise return false */ private boolean checkQueueNameExist(String queueName) { - return queueMapper.queryAllQueueList(null, queueName).size() > 0; + return CollectionUtils.isNotEmpty(queueMapper.queryAllQueueList(null, queueName)); } /** @@ -290,7 +288,7 @@ public class QueueService extends BaseService { * @return true if need to update user */ private boolean checkIfQueueIsInUsing (String oldQueue, String newQueue) { - return !oldQueue.equals(newQueue) && userMapper.queryUserListByQueue(oldQueue).size() > 0; + return !oldQueue.equals(newQueue) && CollectionUtils.isNotEmpty(userMapper.queryUserListByQueue(oldQueue)); } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java index c1adb8874d..ff87aadbc7 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java @@ -16,18 +16,21 @@ */ package org.apache.dolphinscheduler.api.service; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.serializer.SerializerFeature; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import org.apache.commons.collections.BeanMap; +import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent; +import org.apache.dolphinscheduler.api.dto.resources.filter.ResourceFilter; +import org.apache.dolphinscheduler.api.dto.resources.visitor.ResourceTreeVisitor; +import org.apache.dolphinscheduler.api.dto.resources.visitor.Visitor; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ResourceType; -import org.apache.dolphinscheduler.common.utils.FileUtils; -import org.apache.dolphinscheduler.common.utils.HadoopUtils; -import org.apache.dolphinscheduler.common.utils.PropertyUtils; -import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.dao.entity.Resource; import org.apache.dolphinscheduler.dao.entity.Tenant; import org.apache.dolphinscheduler.dao.entity.UdfFunc; @@ -42,6 +45,7 @@ import org.springframework.web.multipart.MultipartFile; import java.text.MessageFormat; import java.util.*; +import java.util.stream.Collectors; import static org.apache.dolphinscheduler.common.Constants.*; @@ -68,6 +72,82 @@ public class ResourcesService extends BaseService { @Autowired private ResourceUserMapper resourceUserMapper; + @Autowired + private ProcessDefinitionMapper processDefinitionMapper; + + /** + * create directory + * + * @param loginUser login user + * @param name alias + * @param description description + * @param type type + * @param pid parent id + * @param currentDir current directory + * @return create directory result + */ + @Transactional(rollbackFor = Exception.class) + public Result createDirectory(User loginUser, + String name, + String description, + ResourceType type, + int pid, + String currentDir) { + Result result = new Result(); + // if hdfs not startup + if (!PropertyUtils.getResUploadStartupState()){ + logger.error("resource upload startup state: {}", PropertyUtils.getResUploadStartupState()); + putMsg(result, Status.HDFS_NOT_STARTUP); + return result; + } + String fullName = currentDir.equals("/") ? String.format("%s%s",currentDir,name):String.format("%s/%s",currentDir,name); + + if (pid != -1) { + Resource parentResource = resourcesMapper.selectById(pid); + + if (parentResource == null) { + putMsg(result, Status.PARENT_RESOURCE_NOT_EXIST); + return result; + } + + if (!hasPerm(loginUser, parentResource.getUserId())) { + putMsg(result, Status.USER_NO_OPERATION_PERM); + return result; + } + } + + + if (checkResourceExists(fullName, 0, type.ordinal())) { + logger.error("resource directory {} has exist, can't recreate", fullName); + putMsg(result, Status.RESOURCE_EXIST); + return result; + } + + Date now = new Date(); + + Resource resource = new Resource(pid,name,fullName,true,description,name,loginUser.getId(),type,0,now,now); + + try { + resourcesMapper.insert(resource); + + putMsg(result, Status.SUCCESS); + Map dataMap = new BeanMap(resource); + Map resultMap = new HashMap(); + for (Map.Entry entry: dataMap.entrySet()) { + if (!"class".equalsIgnoreCase(entry.getKey().toString())) { + resultMap.put(entry.getKey().toString(), entry.getValue()); + } + } + result.setData(resultMap); + } catch (Exception e) { + logger.error("resource already exists, can't recreate ", e); + throw new RuntimeException("resource already exists, can't recreate"); + } + //create directory in hdfs + createDirecotry(loginUser,fullName,type,result); + return result; + } + /** * create resource * @@ -76,6 +156,8 @@ public class ResourcesService extends BaseService { * @param desc description * @param file file * @param type type + * @param pid parent id + * @param currentDir current directory * @return create result code */ @Transactional(rollbackFor = Exception.class) @@ -83,7 +165,9 @@ public class ResourcesService extends BaseService { String name, String desc, ResourceType type, - MultipartFile file) { + MultipartFile file, + int pid, + String currentDir) { Result result = new Result(); // if hdfs not startup @@ -126,7 +210,8 @@ public class ResourcesService extends BaseService { } // check resoure name exists - if (checkResourceExists(name, 0, type.ordinal())) { + String fullName = currentDir.equals("/") ? String.format("%s%s",currentDir,name):String.format("%s/%s",currentDir,name); + if (checkResourceExists(fullName, 0, type.ordinal())) { logger.error("resource {} has exist, can't recreate", name); putMsg(result, Status.RESOURCE_EXIST); return result; @@ -134,14 +219,16 @@ public class ResourcesService extends BaseService { Date now = new Date(); - Resource resource = new Resource(name,file.getOriginalFilename(),desc,loginUser.getId(),type,file.getSize(),now,now); + + + Resource resource = new Resource(pid,name,fullName,false,desc,file.getOriginalFilename(),loginUser.getId(),type,file.getSize(),now,now); try { resourcesMapper.insert(resource); putMsg(result, Status.SUCCESS); Map dataMap = new BeanMap(resource); - Map resultMap = new HashMap(); + Map resultMap = new HashMap<>(); for (Map.Entry entry: dataMap.entrySet()) { if (!"class".equalsIgnoreCase(entry.getKey().toString())) { resultMap.put(entry.getKey().toString(), entry.getValue()); @@ -154,7 +241,7 @@ public class ResourcesService extends BaseService { } // fail upload - if (!upload(loginUser, name, file, type)) { + if (!upload(loginUser, fullName, file, type)) { logger.error("upload resource: {} file: {} failed.", name, file.getOriginalFilename()); putMsg(result, Status.HDFS_OPERATION_ERROR); throw new RuntimeException(String.format("upload resource: %s file: %s failed.", name, file.getOriginalFilename())); @@ -165,14 +252,14 @@ public class ResourcesService extends BaseService { /** * check resource is exists * - * @param alias alias + * @param fullName fullName * @param userId user id * @param type type * @return true if resource exists */ - private boolean checkResourceExists(String alias, int userId, int type ){ + private boolean checkResourceExists(String fullName, int userId, int type ){ - List resources = resourcesMapper.queryResourceList(alias, userId, type); + List resources = resourcesMapper.queryResourceList(fullName, userId, type); if (resources != null && resources.size() > 0) { return true; } @@ -180,16 +267,14 @@ public class ResourcesService extends BaseService { } - /** * update resource - * - * @param loginUser login user - * @param name alias - * @param resourceId resource id - * @param type resource type - * @param desc description - * @return update result code + * @param loginUser login user + * @param resourceId resource id + * @param name name + * @param desc description + * @param type resource type + * @return update result code */ @Transactional(rollbackFor = Exception.class) public Result updateResource(User loginUser, @@ -223,7 +308,10 @@ public class ResourcesService extends BaseService { } //check resource aleady exists - if (!resource.getAlias().equals(name) && checkResourceExists(name, 0, type.ordinal())) { + String originFullName = resource.getFullName(); + + String fullName = String.format("%s%s",originFullName.substring(0,originFullName.lastIndexOf("/")+1),name); + if (!resource.getAlias().equals(name) && checkResourceExists(fullName, 0, type.ordinal())) { logger.error("resource {} already exists, can't recreate", name); putMsg(result, Status.RESOURCE_EXIST); return result; @@ -234,25 +322,41 @@ public class ResourcesService extends BaseService { if (StringUtils.isEmpty(tenantCode)){ return result; } - - //get the file suffix + String nameWithSuffix = name; String originResourceName = resource.getAlias(); - String suffix = originResourceName.substring(originResourceName.lastIndexOf(".")); + if (!resource.isDirectory()) { + //get the file suffix - //if the name without suffix then add it ,else use the origin name - String nameWithSuffix = name; - if(!name.endsWith(suffix)){ - nameWithSuffix = nameWithSuffix + suffix; + String suffix = originResourceName.substring(originResourceName.lastIndexOf(".")); + + //if the name without suffix then add it ,else use the origin name + if(!name.endsWith(suffix)){ + nameWithSuffix = nameWithSuffix + suffix; + } } // updateResource data + List childrenResource = listAllChildren(resource); + String oldFullName = resource.getFullName(); Date now = new Date(); + resource.setAlias(nameWithSuffix); + resource.setFullName(fullName); resource.setDescription(desc); resource.setUpdateTime(now); try { resourcesMapper.updateById(resource); + if (resource.isDirectory() && CollectionUtils.isNotEmpty(childrenResource)) { + List childResourceList = new ArrayList<>(); + List resourceList = resourcesMapper.listResourceByIds(childrenResource.toArray(new Integer[childrenResource.size()])); + childResourceList = resourceList.stream().map(t -> { + t.setFullName(t.getFullName().replaceFirst(oldFullName, fullName)); + t.setUpdateTime(now); + return t; + }).collect(Collectors.toList()); + resourcesMapper.batchUpdateResource(childResourceList); + } putMsg(result, Status.SUCCESS); Map dataMap = new BeanMap(resource); @@ -274,15 +378,9 @@ public class ResourcesService extends BaseService { // get file hdfs path // delete hdfs file by type - String originHdfsFileName = ""; - String destHdfsFileName = ""; - if (resource.getType().equals(ResourceType.FILE)) { - originHdfsFileName = HadoopUtils.getHdfsFilename(tenantCode, originResourceName); - destHdfsFileName = HadoopUtils.getHdfsFilename(tenantCode, name); - } else if (resource.getType().equals(ResourceType.UDF)) { - originHdfsFileName = HadoopUtils.getHdfsUdfFilename(tenantCode, originResourceName); - destHdfsFileName = HadoopUtils.getHdfsUdfFilename(tenantCode, name); - } + String originHdfsFileName = HadoopUtils.getHdfsFileName(resource.getType(),tenantCode,originFullName); + String destHdfsFileName = HadoopUtils.getHdfsFileName(resource.getType(),tenantCode,fullName); + try { if (HadoopUtils.getInstance().exists(originHdfsFileName)) { logger.info("hdfs copy {} -> {}", originHdfsFileName, destHdfsFileName); @@ -310,7 +408,7 @@ public class ResourcesService extends BaseService { * @param pageSize page size * @return resource list page */ - public Map queryResourceListPaging(User loginUser, ResourceType type, String searchVal, Integer pageNo, Integer pageSize) { + public Map queryResourceListPaging(User loginUser, int direcotryId, ResourceType type, String searchVal, Integer pageNo, Integer pageSize) { HashMap result = new HashMap<>(5); Page page = new Page(pageNo, pageSize); @@ -319,7 +417,7 @@ public class ResourcesService extends BaseService { userId= 0; } IPage resourceIPage = resourcesMapper.queryResourcePaging(page, - userId, type.ordinal(), searchVal); + userId,direcotryId, type.ordinal(), searchVal); PageInfo pageInfo = new PageInfo(pageNo, pageSize); pageInfo.setTotalCount((int)resourceIPage.getTotal()); pageInfo.setLists(resourceIPage.getRecords()); @@ -328,17 +426,46 @@ public class ResourcesService extends BaseService { return result; } + /** + * create direcoty + * @param loginUser login user + * @param fullName full name + * @param type resource type + * @param result Result + */ + private void createDirecotry(User loginUser,String fullName,ResourceType type,Result result){ + // query tenant + String tenantCode = tenantMapper.queryById(loginUser.getTenantId()).getTenantCode(); + String directoryName = HadoopUtils.getHdfsFileName(type,tenantCode,fullName); + String resourceRootPath = HadoopUtils.getHdfsDir(type,tenantCode); + try { + if (!HadoopUtils.getInstance().exists(resourceRootPath)) { + createTenantDirIfNotExists(tenantCode); + } + + if (!HadoopUtils.getInstance().mkdir(directoryName)) { + logger.error("create resource directory {} of hdfs failed",directoryName); + putMsg(result,Status.HDFS_OPERATION_ERROR); + throw new RuntimeException(String.format("create resource directory: %s failed.", directoryName)); + } + } catch (Exception e) { + logger.error("create resource directory {} of hdfs failed",directoryName); + putMsg(result,Status.HDFS_OPERATION_ERROR); + throw new RuntimeException(String.format("create resource directory: %s failed.", directoryName)); + } + } + /** * upload file to hdfs * - * @param loginUser - * @param name - * @param file + * @param loginUser login user + * @param fullName full name + * @param file file */ - private boolean upload(User loginUser, String name, MultipartFile file, ResourceType type) { + private boolean upload(User loginUser, String fullName, MultipartFile file, ResourceType type) { // save to local String fileSuffix = FileUtils.suffix(file.getOriginalFilename()); - String nameSuffix = FileUtils.suffix(name); + String nameSuffix = FileUtils.suffix(fullName); // determine file suffix if (!(StringUtils.isNotEmpty(fileSuffix) && fileSuffix.equalsIgnoreCase(nameSuffix))) { @@ -351,15 +478,8 @@ public class ResourcesService extends BaseService { // save file to hdfs, and delete original file - String hdfsFilename = ""; - String resourcePath = ""; - if (type.equals(ResourceType.FILE)) { - hdfsFilename = HadoopUtils.getHdfsFilename(tenantCode, name); - resourcePath = HadoopUtils.getHdfsResDir(tenantCode); - } else if (type.equals(ResourceType.UDF)) { - hdfsFilename = HadoopUtils.getHdfsUdfFilename(tenantCode, name); - resourcePath = HadoopUtils.getHdfsUdfDir(tenantCode); - } + String hdfsFilename = HadoopUtils.getHdfsFileName(type,tenantCode,fullName); + String resourcePath = HadoopUtils.getHdfsDir(type,tenantCode); try { // if tenant dir not exists if (!HadoopUtils.getInstance().exists(resourcePath)) { @@ -384,13 +504,59 @@ public class ResourcesService extends BaseService { public Map queryResourceList(User loginUser, ResourceType type) { Map result = new HashMap<>(5); - List resourceList; + + Set allResourceList = getAllResources(loginUser, type); + Visitor resourceTreeVisitor = new ResourceTreeVisitor(new ArrayList<>(allResourceList)); + //JSONArray jsonArray = JSON.parseArray(JSON.toJSONString(resourceTreeVisitor.visit().getChildren(), SerializerFeature.SortField)); + result.put(Constants.DATA_LIST, resourceTreeVisitor.visit().getChildren()); + putMsg(result,Status.SUCCESS); + + return result; + } + + /** + * get all resources + * @param loginUser login user + * @return all resource set + */ + private Set getAllResources(User loginUser, ResourceType type) { int userId = loginUser.getId(); + boolean listChildren = true; if(isAdmin(loginUser)){ userId = 0; + listChildren = false; + } + List resourceList = resourcesMapper.queryResourceListAuthored(userId, type.ordinal()); + Set allResourceList = new HashSet<>(resourceList); + if (listChildren) { + Set authorizedIds = new HashSet<>(); + List authorizedDirecoty = resourceList.stream().filter(t->t.getUserId() != loginUser.getId() && t.isDirectory()).collect(Collectors.toList()); + if (CollectionUtils.isNotEmpty(authorizedDirecoty)) { + for(Resource resource : authorizedDirecoty){ + authorizedIds.addAll(listAllChildren(resource)); + } + List childrenResources = resourcesMapper.listResourceByIds(authorizedIds.toArray(new Integer[authorizedIds.size()])); + allResourceList.addAll(childrenResources); + } } - resourceList = resourcesMapper.queryResourceListAuthored(userId, type.ordinal()); - result.put(Constants.DATA_LIST, resourceList); + return allResourceList; + } + + /** + * query resource list + * + * @param loginUser login user + * @param type resource type + * @return resource list + */ + public Map queryResourceJarList(User loginUser, ResourceType type) { + + Map result = new HashMap<>(5); + + Set allResourceList = getAllResources(loginUser, type); + List resources = new ResourceFilter(".jar",new ArrayList<>(allResourceList)).filter(); + Visitor resourceTreeVisitor = new ResourceTreeVisitor(resources); + result.put(Constants.DATA_LIST, resourceTreeVisitor.visit().getChildren()); putMsg(result,Status.SUCCESS); return result; @@ -426,23 +592,51 @@ public class ResourcesService extends BaseService { putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } + //if resource type is UDF,need check whether it is bound by UDF functon + if (resource.getType() == (ResourceType.UDF)) { + List udfFuncs = udfFunctionMapper.listUdfByResourceId(new int[]{resourceId}); + if (CollectionUtils.isNotEmpty(udfFuncs)) { + logger.error("can't be deleted,because it is bound by UDF functions:{}",udfFuncs.toString()); + putMsg(result,Status.UDF_RESOURCE_IS_BOUND,udfFuncs.get(0).getFuncName()); + return result; + } + } - Tenant tenant = tenantMapper.queryById(loginUser.getTenantId()); - if (tenant == null){ - putMsg(result, Status.TENANT_NOT_EXIST); + String tenantCode = getTenantCode(resource.getUserId(),result); + if (StringUtils.isEmpty(tenantCode)){ + return result; + } + + // get all resource id of process definitions those is released + Map> resourceProcessMap = getResourceProcessMap(); + Set resourceIdSet = resourceProcessMap.keySet(); + // get all children of the resource + List allChildren = listAllChildren(resource); + + if (resourceIdSet.contains(resource.getPid())) { + logger.error("can't be deleted,because it is used of process definition"); + putMsg(result, Status.RESOURCE_IS_USED); + return result; + } + resourceIdSet.retainAll(allChildren); + if (CollectionUtils.isNotEmpty(resourceIdSet)) { + logger.error("can't be deleted,because it is used of process definition"); + for (Integer resId : resourceIdSet) { + logger.error("resource id:{} is used of process definition {}",resId,resourceProcessMap.get(resId)); + } + putMsg(result, Status.RESOURCE_IS_USED); return result; } - String hdfsFilename = ""; - // delete hdfs file by type - String tenantCode = tenant.getTenantCode(); - hdfsFilename = getHdfsFileName(resource, tenantCode, hdfsFilename); + // get hdfs file by type + String hdfsFilename = HadoopUtils.getHdfsFileName(resource.getType(), tenantCode, resource.getFullName()); //delete data in database - resourcesMapper.deleteById(resourceId); + resourcesMapper.deleteIds(allChildren.toArray(new Integer[allChildren.size()])); resourceUserMapper.deleteResourceUser(0, resourceId); + //delete file on hdfs - HadoopUtils.getInstance().delete(hdfsFilename, false); + HadoopUtils.getInstance().delete(hdfsFilename, true); putMsg(result, Status.SUCCESS); return result; @@ -451,15 +645,15 @@ public class ResourcesService extends BaseService { /** * verify resource by name and type * @param loginUser login user - * @param name resource alias - * @param type resource type + * @param fullName resource full name + * @param type resource type * @return true if the resource name not exists, otherwise return false */ - public Result verifyResourceName(String name, ResourceType type,User loginUser) { + public Result verifyResourceName(String fullName, ResourceType type,User loginUser) { Result result = new Result(); putMsg(result, Status.SUCCESS); - if (checkResourceExists(name, 0, type.ordinal())) { - logger.error("resource type:{} name:{} has exist, can't create again.", type, name); + if (checkResourceExists(fullName, 0, type.ordinal())) { + logger.error("resource type:{} name:{} has exist, can't create again.", type, fullName); putMsg(result, Status.RESOURCE_EXIST); } else { // query tenant @@ -468,9 +662,9 @@ public class ResourcesService extends BaseService { String tenantCode = tenant.getTenantCode(); try { - String hdfsFilename = getHdfsFileName(type,tenantCode,name); + String hdfsFilename = HadoopUtils.getHdfsFileName(type,tenantCode,fullName); if(HadoopUtils.getInstance().exists(hdfsFilename)){ - logger.error("resource type:{} name:{} has exist in hdfs {}, can't create again.", type, name,hdfsFilename); + logger.error("resource type:{} name:{} has exist in hdfs {}, can't create again.", type, fullName,hdfsFilename); putMsg(result, Status.RESOURCE_FILE_EXIST,hdfsFilename); } @@ -487,6 +681,48 @@ public class ResourcesService extends BaseService { return result; } + /** + * verify resource by full name or pid and type + * @param fullName resource full name + * @param id resource id + * @param type resource type + * @return true if the resource full name or pid not exists, otherwise return false + */ + public Result queryResource(String fullName,Integer id,ResourceType type) { + Result result = new Result(); + if (StringUtils.isBlank(fullName) && id == null) { + logger.error("You must input one of fullName and pid"); + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR); + return result; + } + if (StringUtils.isNotBlank(fullName)) { + List resourceList = resourcesMapper.queryResource(fullName,type.ordinal()); + if (CollectionUtils.isEmpty(resourceList)) { + logger.error("resource file not exist, resource full name {} ", fullName); + putMsg(result, Status.RESOURCE_NOT_EXIST); + return result; + } + putMsg(result, Status.SUCCESS); + result.setData(resourceList.get(0)); + } else { + Resource resource = resourcesMapper.selectById(id); + if (resource == null) { + logger.error("resource file not exist, resource id {}", id); + putMsg(result, Status.RESOURCE_NOT_EXIST); + return result; + } + Resource parentResource = resourcesMapper.selectById(resource.getPid()); + if (parentResource == null) { + logger.error("parent resource file not exist, resource id {}", id); + putMsg(result, Status.RESOURCE_NOT_EXIST); + return result; + } + putMsg(result, Status.SUCCESS); + result.setData(parentResource); + } + return result; + } + /** * view resource file online * @@ -508,7 +744,7 @@ public class ResourcesService extends BaseService { // get resource by id Resource resource = resourcesMapper.selectById(resourceId); if (resource == null) { - logger.error("resouce file not exist, resource id {}", resourceId); + logger.error("resource file not exist, resource id {}", resourceId); putMsg(result, Status.RESOURCE_NOT_EXIST); return result; } @@ -518,7 +754,7 @@ public class ResourcesService extends BaseService { if (StringUtils.isNotEmpty(resourceViewSuffixs)) { List strList = Arrays.asList(resourceViewSuffixs.split(",")); if (!strList.contains(nameSuffix)) { - logger.error("resouce suffix {} not support view, resource id {}", nameSuffix, resourceId); + logger.error("resource suffix {} not support view, resource id {}", nameSuffix, resourceId); putMsg(result, Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW); return result; } @@ -530,7 +766,7 @@ public class ResourcesService extends BaseService { } // hdfs path - String hdfsFileName = HadoopUtils.getHdfsFilename(tenantCode, resource.getAlias()); + String hdfsFileName = HadoopUtils.getHdfsResourceFileName(tenantCode, resource.getFullName()); logger.info("resource hdfs path is {} ", hdfsFileName); try { if(HadoopUtils.getInstance().exists(hdfsFileName)){ @@ -566,7 +802,7 @@ public class ResourcesService extends BaseService { * @return create result code */ @Transactional(rollbackFor = Exception.class) - public Result onlineCreateResource(User loginUser, ResourceType type, String fileName, String fileSuffix, String desc, String content) { + public Result onlineCreateResource(User loginUser, ResourceType type, String fileName, String fileSuffix, String desc, String content,int pid,String currentDirectory) { Result result = new Result(); // if resource upload startup if (!PropertyUtils.getResUploadStartupState()){ @@ -588,15 +824,16 @@ public class ResourcesService extends BaseService { } String name = fileName.trim() + "." + nameSuffix; + String fullName = currentDirectory.equals("/") ? String.format("%s%s",currentDirectory,name):String.format("%s/%s",currentDirectory,name); - result = verifyResourceName(name,type,loginUser); + result = verifyResourceName(fullName,type,loginUser); if (!result.getCode().equals(Status.SUCCESS.getCode())) { return result; } // save data Date now = new Date(); - Resource resource = new Resource(name,name,desc,loginUser.getId(),type,content.getBytes().length,now,now); + Resource resource = new Resource(pid,name,fullName,false,desc,name,loginUser.getId(),type,content.getBytes().length,now,now); resourcesMapper.insert(resource); @@ -612,7 +849,7 @@ public class ResourcesService extends BaseService { String tenantCode = tenantMapper.queryById(loginUser.getTenantId()).getTenantCode(); - result = uploadContentToHdfs(name, tenantCode, content); + result = uploadContentToHdfs(fullName, tenantCode, content); if (!result.getCode().equals(Status.SUCCESS.getCode())) { throw new RuntimeException(result.getMsg()); } @@ -664,7 +901,7 @@ public class ResourcesService extends BaseService { resourcesMapper.updateById(resource); - result = uploadContentToHdfs(resource.getAlias(), tenantCode, content); + result = uploadContentToHdfs(resource.getFullName(), tenantCode, content); if (!result.getCode().equals(Status.SUCCESS.getCode())) { throw new RuntimeException(result.getMsg()); } @@ -672,10 +909,10 @@ public class ResourcesService extends BaseService { } /** - * @param resourceName - * @param tenantCode - * @param content - * @return + * @param resourceName resource name + * @param tenantCode tenant code + * @param content content + * @return result */ private Result uploadContentToHdfs(String resourceName, String tenantCode, String content) { Result result = new Result(); @@ -691,8 +928,8 @@ public class ResourcesService extends BaseService { return result; } - // get file hdfs path - hdfsFileName = HadoopUtils.getHdfsFilename(tenantCode, resourceName); + // get resource file hdfs path + hdfsFileName = HadoopUtils.getHdfsResourceFileName(tenantCode, resourceName); String resourcePath = HadoopUtils.getHdfsResDir(tenantCode); logger.info("resource hdfs path is {} ", hdfsFileName); @@ -736,21 +973,50 @@ public class ResourcesService extends BaseService { logger.error("download file not exist, resource id {}", resourceId); return null; } + if (resource.isDirectory()) { + logger.error("resource id {} is directory,can't download it", resourceId); + throw new RuntimeException("cant't download directory"); + } User user = userMapper.queryDetailsById(resource.getUserId()); String tenantCode = tenantMapper.queryById(user.getTenantId()).getTenantCode(); - String hdfsFileName = ""; - hdfsFileName = getHdfsFileName(resource, tenantCode, hdfsFileName); + String hdfsFileName = HadoopUtils.getHdfsFileName(resource.getType(), tenantCode, resource.getAlias()); String localFileName = FileUtils.getDownloadFilename(resource.getAlias()); logger.info("resource hdfs path is {} ", hdfsFileName); HadoopUtils.getInstance().copyHdfsToLocal(hdfsFileName, localFileName, false, true); - org.springframework.core.io.Resource file = org.apache.dolphinscheduler.api.utils.FileUtils.file2Resource(localFileName); - return file; + return org.apache.dolphinscheduler.api.utils.FileUtils.file2Resource(localFileName); } + /** + * list all file + * + * @param loginUser login user + * @param userId user id + * @return unauthorized result code + */ + public Map authorizeResourceTree(User loginUser, Integer userId) { + + Map result = new HashMap<>(); + if (checkAdmin(loginUser, result)) { + return result; + } + List resourceList = resourcesMapper.queryResourceExceptUserId(userId); + List list ; + if (CollectionUtils.isNotEmpty(resourceList)) { + Visitor visitor = new ResourceTreeVisitor(resourceList); + list = visitor.visit().getChildren(); + }else { + list = new ArrayList<>(0); + } + + result.put(Constants.DATA_LIST, list); + putMsg(result,Status.SUCCESS); + return result; + } + /** * unauthorized file * @@ -765,7 +1031,7 @@ public class ResourcesService extends BaseService { return result; } List resourceList = resourcesMapper.queryResourceExceptUserId(userId); - List list ; + List list ; if (resourceList != null && resourceList.size() > 0) { Set resourceSet = new HashSet<>(resourceList); List authedResourceList = resourcesMapper.queryAuthorizedResourceList(userId); @@ -775,15 +1041,12 @@ public class ResourcesService extends BaseService { }else { list = new ArrayList<>(0); } - - result.put(Constants.DATA_LIST, list); + Visitor visitor = new ResourceTreeVisitor(list); + result.put(Constants.DATA_LIST, visitor.visit().getChildren()); putMsg(result,Status.SUCCESS); return result; } - - - /** * unauthorized udf function * @@ -801,7 +1064,7 @@ public class ResourcesService extends BaseService { List udfFuncList = udfFunctionMapper.queryUdfFuncExceptUserId(userId); List resultList = new ArrayList<>(); Set udfFuncSet = null; - if (udfFuncList != null && udfFuncList.size() > 0) { + if (CollectionUtils.isNotEmpty(udfFuncList)) { udfFuncSet = new HashSet<>(udfFuncList); List authedUDFFuncList = udfFunctionMapper.queryAuthedUdfFunc(userId); @@ -849,46 +1112,15 @@ public class ResourcesService extends BaseService { return result; } List authedResources = resourcesMapper.queryAuthorizedResourceList(userId); - - result.put(Constants.DATA_LIST, authedResources); + Visitor visitor = new ResourceTreeVisitor(authedResources); + logger.info(JSON.toJSONString(visitor.visit(), SerializerFeature.SortField)); + String jsonTreeStr = JSON.toJSONString(visitor.visit().getChildren(), SerializerFeature.SortField); + logger.info(jsonTreeStr); + result.put(Constants.DATA_LIST, visitor.visit().getChildren()); putMsg(result,Status.SUCCESS); return result; } - /** - * get hdfs file name - * - * @param resource resource - * @param tenantCode tenant code - * @param hdfsFileName hdfs file name - * @return hdfs file name - */ - private String getHdfsFileName(Resource resource, String tenantCode, String hdfsFileName) { - if (resource.getType().equals(ResourceType.FILE)) { - hdfsFileName = HadoopUtils.getHdfsFilename(tenantCode, resource.getAlias()); - } else if (resource.getType().equals(ResourceType.UDF)) { - hdfsFileName = HadoopUtils.getHdfsUdfFilename(tenantCode, resource.getAlias()); - } - return hdfsFileName; - } - - /** - * get hdfs file name - * - * @param resourceType resource type - * @param tenantCode tenant code - * @param hdfsFileName hdfs file name - * @return hdfs file name - */ - private String getHdfsFileName(ResourceType resourceType, String tenantCode, String hdfsFileName) { - if (resourceType.equals(ResourceType.FILE)) { - hdfsFileName = HadoopUtils.getHdfsFilename(tenantCode, hdfsFileName); - } else if (resourceType.equals(ResourceType.UDF)) { - hdfsFileName = HadoopUtils.getHdfsUdfFilename(tenantCode, hdfsFileName); - } - return hdfsFileName; - } - /** * get authorized resource list * @@ -897,10 +1129,9 @@ public class ResourcesService extends BaseService { */ private void getAuthorizedResourceList(Set resourceSet, List authedResourceList) { Set authedResourceSet = null; - if (authedResourceList != null && authedResourceList.size() > 0) { + if (CollectionUtils.isNotEmpty(authedResourceList)) { authedResourceSet = new HashSet<>(authedResourceList); resourceSet.removeAll(authedResourceSet); - } } @@ -929,4 +1160,69 @@ public class ResourcesService extends BaseService { return tenant.getTenantCode(); } + /** + * list all children id + * @param resource resource + * @return all children id + */ + List listAllChildren(Resource resource){ + List childList = new ArrayList<>(); + if (resource.getId() != -1) { + childList.add(resource.getId()); + } + + if(resource.isDirectory()){ + listAllChildren(resource.getId(),childList); + } + return childList; + } + + /** + * list all children id + * @param resourceId resource id + * @param childList child list + */ + void listAllChildren(int resourceId,List childList){ + + List children = resourcesMapper.listChildren(resourceId); + for(int chlidId:children){ + childList.add(chlidId); + listAllChildren(chlidId,childList); + } + } + + /** + * get resource process map key is resource id,value is the set of process definition + * @return resource process definition map + */ + private Map> getResourceProcessMap(){ + Map map = new HashMap<>(); + Map> result = new HashMap<>(); + List> list = processDefinitionMapper.listResources(); + if (CollectionUtils.isNotEmpty(list)) { + for (Map tempMap : list) { + + map.put((Integer) tempMap.get("id"), (String)tempMap.get("resource_ids")); + } + } + + for (Map.Entry entry : map.entrySet()) { + Integer mapKey = entry.getKey(); + String[] arr = entry.getValue().split(","); + Set mapValues = Arrays.stream(arr).map(Integer::parseInt).collect(Collectors.toSet()); + for (Integer value : mapValues) { + if (result.containsKey(value)) { + Set set = result.get(value); + set.add(mapKey); + result.put(value, set); + } else { + Set set = new HashSet<>(); + set.add(mapKey); + result.put(value, set); + } + } + } + return result; + } + } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskInstanceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskInstanceService.java index e4fec54395..170278e02f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskInstanceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskInstanceService.java @@ -32,8 +32,6 @@ import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; import org.apache.dolphinscheduler.service.process.ProcessService; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -46,8 +44,6 @@ import java.util.*; @Service public class TaskInstanceService extends BaseService { - private static final Logger logger = LoggerFactory.getLogger(TaskInstanceService.class); - @Autowired ProjectMapper projectMapper; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskRecordService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskRecordService.java index 54b6a1889c..54eba5c2d6 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskRecordService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskRecordService.java @@ -21,8 +21,6 @@ import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.dao.TaskRecordDao; import org.apache.dolphinscheduler.dao.entity.TaskRecord; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; import java.util.HashMap; @@ -37,8 +35,6 @@ import static org.apache.dolphinscheduler.common.Constants.*; @Service public class TaskRecordService extends BaseService{ - private static final Logger logger = LoggerFactory.getLogger(TaskRecordService.class); - /** * query task record list paging * diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TenantService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TenantService.java index 12b4656a40..2fded4d32f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TenantService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TenantService.java @@ -310,7 +310,7 @@ public class TenantService extends BaseService{ Map result = new HashMap<>(5); List resourceList = tenantMapper.queryByTenantCode(tenantCode); - if (resourceList != null && resourceList.size() > 0) { + if (CollectionUtils.isNotEmpty(resourceList)) { result.put(Constants.DATA_LIST, resourceList); putMsg(result, Status.SUCCESS); } else { @@ -346,6 +346,6 @@ public class TenantService extends BaseService{ */ private boolean checkTenantExists(String tenantCode) { List tenants = tenantMapper.queryByTenantCode(tenantCode); - return (tenants != null && tenants.size() > 0); + return CollectionUtils.isNotEmpty(tenants); } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java index 249c7ec8df..8a0bf748bb 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java @@ -118,7 +118,7 @@ public class UdfFuncService extends BaseService{ } udf.setDescription(desc); udf.setResourceId(resourceId); - udf.setResourceName(resource.getAlias()); + udf.setResourceName(resource.getFullName()); udf.setType(type); udf.setCreateTime(now); @@ -226,7 +226,7 @@ public class UdfFuncService extends BaseService{ } udf.setDescription(desc); udf.setResourceId(resourceId); - udf.setResourceName(resource.getAlias()); + udf.setResourceName(resource.getFullName()); udf.setType(type); udf.setUpdateTime(now); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UserAlertGroupService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UserAlertGroupService.java new file mode 100644 index 0000000000..502185709f --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UserAlertGroupService.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.api.service; + +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import org.apache.dolphinscheduler.dao.entity.UserAlertGroup; +import org.apache.dolphinscheduler.dao.mapper.UserAlertGroupMapper; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +/** + * + */ +@Service +public class UserAlertGroupService extends ServiceImpl { + + @Autowired + private UserAlertGroupMapper userAlertGroupMapper; + + boolean deleteByAlertGroupId(Integer groupId) { + return userAlertGroupMapper.deleteByAlertgroupId(groupId) >= 1; + } + +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkerGroupService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkerGroupService.java index c44c446d5c..7d47a8fb0d 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkerGroupService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkerGroupService.java @@ -111,7 +111,7 @@ public class WorkerGroupService extends BaseService { List workerGroupList = workerGroupMapper.queryWorkerGroupByName(workerGroup.getName()); - if(workerGroupList.size() > 0 ){ + if(CollectionUtils.isNotEmpty(workerGroupList)){ // new group has same name.. if(workerGroup.getId() == 0){ return true; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/DataSourceControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/DataSourceControllerTest.java index f80ce8556e..5ed7310c47 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/DataSourceControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/DataSourceControllerTest.java @@ -39,6 +39,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. * data source controller test */ public class DataSourceControllerTest extends AbstractControllerTest{ + private static Logger logger = LoggerFactory.getLogger(DataSourceControllerTest.class); @Ignore @@ -95,6 +96,7 @@ public class DataSourceControllerTest extends AbstractControllerTest{ + @Ignore @Test public void testQueryDataSource() throws Exception { MultiValueMap paramsMap = new LinkedMultiValueMap<>(); @@ -169,6 +171,7 @@ public class DataSourceControllerTest extends AbstractControllerTest{ } + @Ignore @Test public void testConnectionTest() throws Exception { MultiValueMap paramsMap = new LinkedMultiValueMap<>(); @@ -248,6 +251,7 @@ public class DataSourceControllerTest extends AbstractControllerTest{ + @Ignore @Test public void testDelete() throws Exception { MultiValueMap paramsMap = new LinkedMultiValueMap<>(); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoginControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoginControllerTest.java index f5a28d01ae..5be7b0711c 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoginControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoginControllerTest.java @@ -36,7 +36,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. * login controller test */ public class LoginControllerTest extends AbstractControllerTest{ - private static Logger logger = LoggerFactory.getLogger(SchedulerControllerTest.class); + private static Logger logger = LoggerFactory.getLogger(LoginControllerTest.class); @Test diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskRecordControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskRecordControllerTest.java index 943e14607b..ad4a165ca1 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskRecordControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskRecordControllerTest.java @@ -33,7 +33,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; public class TaskRecordControllerTest extends AbstractControllerTest { - private static final Logger logger = LoggerFactory.getLogger(TaskInstanceController.class); + private static final Logger logger = LoggerFactory.getLogger(TaskRecordControllerTest.class); @Test public void testQueryTaskRecordListPaging() throws Exception { diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/UsersControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/UsersControllerTest.java index d1be6cb382..0798151299 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/UsersControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/UsersControllerTest.java @@ -37,7 +37,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. * users controller test */ public class UsersControllerTest extends AbstractControllerTest{ - private static Logger logger = LoggerFactory.getLogger(QueueControllerTest.class); + private static Logger logger = LoggerFactory.getLogger(UsersControllerTest.class); @Test public void testCreateUser() throws Exception { diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilterTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilterTest.java new file mode 100644 index 0000000000..8a4a16c4f0 --- /dev/null +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilterTest.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.api.dto.resources.filter; + +import org.apache.dolphinscheduler.dao.entity.Resource; +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; + +/** + * resource filter test + */ +public class ResourceFilterTest { + private static Logger logger = LoggerFactory.getLogger(ResourceFilterTest.class); + @Test + public void filterTest(){ + List allList = new ArrayList<>(); + + Resource resource1 = new Resource(3,-1,"b","/b",true); + Resource resource2 = new Resource(4,2,"a1.txt","/a/a1.txt",false); + Resource resource3 = new Resource(5,3,"b1.txt","/b/b1.txt",false); + Resource resource4 = new Resource(6,3,"b2.jar","/b/b2.jar",false); + Resource resource5 = new Resource(7,-1,"b2","/b2",true); + Resource resource6 = new Resource(8,-1,"b2","/b/b2",true); + Resource resource7 = new Resource(9,8,"c2.jar","/b/b2/c2.jar",false); + allList.add(resource1); + allList.add(resource2); + allList.add(resource3); + allList.add(resource4); + allList.add(resource5); + allList.add(resource6); + allList.add(resource7); + + + ResourceFilter resourceFilter = new ResourceFilter(".jar",allList); + List resourceList = resourceFilter.filter(); + Assert.assertNotNull(resourceList); + resourceList.stream().forEach(t-> logger.info(t.toString())); + } +} \ No newline at end of file diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitorTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitorTest.java new file mode 100644 index 0000000000..d1f8a12012 --- /dev/null +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitorTest.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.api.dto.resources.visitor; + +import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent; +import org.apache.dolphinscheduler.dao.entity.Resource; +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +/** + * resource tree visitor test + */ +public class ResourceTreeVisitorTest { + + @Test + public void visit() throws Exception { + List resourceList = new ArrayList<>(); + + Resource resource1 = new Resource(3,-1,"b","/b",true); + Resource resource2 = new Resource(4,2,"a1.txt","/a/a1.txt",false); + Resource resource3 = new Resource(5,3,"b1.txt","/b/b1.txt",false); + Resource resource4 = new Resource(6,3,"b2.jar","/b/b2.jar",false); + Resource resource5 = new Resource(7,-1,"b2","/b2",true); + Resource resource6 = new Resource(8,-1,"b2","/b/b2",true); + Resource resource7 = new Resource(9,8,"c2.jar","/b/b2/c2.jar",false); + resourceList.add(resource1); + resourceList.add(resource2); + resourceList.add(resource3); + resourceList.add(resource4); + resourceList.add(resource5); + resourceList.add(resource6); + resourceList.add(resource7); + + ResourceTreeVisitor resourceTreeVisitor = new ResourceTreeVisitor(resourceList); + ResourceComponent resourceComponent = resourceTreeVisitor.visit(); + Assert.assertNotNull(resourceComponent.getChildren()); + } + + @Test + public void rootNode() throws Exception { + List resourceList = new ArrayList<>(); + + Resource resource1 = new Resource(3,-1,"b","/b",true); + Resource resource2 = new Resource(4,2,"a1.txt","/a/a1.txt",false); + Resource resource3 = new Resource(5,3,"b1.txt","/b/b1.txt",false); + Resource resource4 = new Resource(6,3,"b2.jar","/b/b2.jar",false); + Resource resource5 = new Resource(7,-1,"b2","/b2",true); + Resource resource6 = new Resource(8,-1,"b2","/b/b2",true); + Resource resource7 = new Resource(9,8,"c2.jar","/b/b2/c2.jar",false); + resourceList.add(resource1); + resourceList.add(resource2); + resourceList.add(resource3); + resourceList.add(resource4); + resourceList.add(resource5); + resourceList.add(resource6); + resourceList.add(resource7); + + ResourceTreeVisitor resourceTreeVisitor = new ResourceTreeVisitor(resourceList); + Assert.assertTrue(resourceTreeVisitor.rootNode(resource1)); + Assert.assertTrue(resourceTreeVisitor.rootNode(resource2)); + Assert.assertFalse(resourceTreeVisitor.rootNode(resource3)); + + } + +} \ No newline at end of file diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/enums/StatusTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/enums/StatusTest.java index 0c9ddff791..4e31a71e9d 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/enums/StatusTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/enums/StatusTest.java @@ -28,7 +28,7 @@ public class StatusTest { @Test public void testGetCode() { - assertEquals(Status.SUCCESS.getCode(), 0); + assertEquals(0, Status.SUCCESS.getCode()); assertNotEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR.getCode(), 0); } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AlertGroupServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AlertGroupServiceTest.java index 4a31902af9..ab7dac4d60 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AlertGroupServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AlertGroupServiceTest.java @@ -18,9 +18,12 @@ package org.apache.dolphinscheduler.api.service; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.PageInfo; -import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.AlertType; import org.apache.dolphinscheduler.common.enums.UserType; @@ -31,9 +34,12 @@ import org.apache.dolphinscheduler.dao.mapper.AlertGroupMapper; import org.apache.dolphinscheduler.dao.mapper.UserAlertGroupMapper; import org.junit.After; import org.junit.Assert; +import static org.junit.Assert.assertEquals; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import static org.mockito.ArgumentMatchers.*; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; @@ -41,14 +47,6 @@ import org.mockito.junit.MockitoJUnitRunner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; - @RunWith(MockitoJUnitRunner.class) public class AlertGroupServiceTest { @@ -60,6 +58,8 @@ public class AlertGroupServiceTest { private AlertGroupMapper alertGroupMapper; @Mock private UserAlertGroupMapper userAlertGroupMapper; + @Mock + UserAlertGroupService userAlertGroupService; private String groupName = "AlertGroupServiceTest"; @@ -160,25 +160,34 @@ public class AlertGroupServiceTest { } + @Test - public void testGrantUser(){ + public void testGrantUser() { + + Integer groupId = 1; + + ArgumentCaptor groupArgument = ArgumentCaptor.forClass(Integer.class); + + Mockito.when(userAlertGroupService.deleteByAlertGroupId(anyInt())).thenReturn(true); + + Map result = alertGroupService.grantUser(getLoginUser(), groupId, "123,321"); + Mockito.verify(userAlertGroupService).deleteByAlertGroupId(groupArgument.capture()); - Map result = alertGroupService.grantUser(getLoginUser(),1,"123,321"); logger.info(result.toString()); - Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + assertEquals(groupArgument.getValue(), groupId); + assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); } + @Test - public void testVerifyGroupName(){ + public void testVerifyGroupName() { //group name not exist - Result result = alertGroupService.verifyGroupName(getLoginUser(), groupName); - logger.info(result.toString()); - Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg()); + boolean result = alertGroupService.existGroupName(groupName); + Assert.assertFalse(result); Mockito.when(alertGroupMapper.queryByGroupName(groupName)).thenReturn(getList()); //group name exist - result = alertGroupService.verifyGroupName(getLoginUser(), groupName); - logger.info(result.toString()); - Assert.assertEquals(Status.ALERT_GROUP_EXIST.getMsg(),result.getMsg()); + result = alertGroupService.existGroupName(groupName); + Assert.assertTrue(result); } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java index 6d07ebd99c..d73eba8bdc 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java @@ -24,10 +24,7 @@ import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ResourceType; import org.apache.dolphinscheduler.common.enums.UserType; -import org.apache.dolphinscheduler.common.utils.CollectionUtils; -import org.apache.dolphinscheduler.common.utils.FileUtils; -import org.apache.dolphinscheduler.common.utils.HadoopUtils; -import org.apache.dolphinscheduler.common.utils.PropertyUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.dao.entity.Resource; import org.apache.dolphinscheduler.dao.entity.Tenant; import org.apache.dolphinscheduler.dao.entity.UdfFunc; @@ -40,6 +37,7 @@ import org.junit.runner.RunWith; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; +import org.omg.CORBA.Any; import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; @@ -73,6 +71,8 @@ public class ResourcesServiceTest { private UserMapper userMapper; @Mock private UdfFuncMapper udfFunctionMapper; + @Mock + private ProcessDefinitionMapper processDefinitionMapper; @Before public void setUp() { @@ -96,14 +96,14 @@ public class ResourcesServiceTest { PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false); User user = new User(); //HDFS_NOT_STARTUP - Result result = resourcesService.createResource(user,"ResourcesServiceTest","ResourcesServiceTest",ResourceType.FILE,null); + Result result = resourcesService.createResource(user,"ResourcesServiceTest","ResourcesServiceTest",ResourceType.FILE,null,-1,"/"); logger.info(result.toString()); Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(),result.getMsg()); //RESOURCE_FILE_IS_EMPTY MockMultipartFile mockMultipartFile = new MockMultipartFile("test.pdf",new String().getBytes()); PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); - result = resourcesService.createResource(user,"ResourcesServiceTest","ResourcesServiceTest",ResourceType.FILE,mockMultipartFile); + result = resourcesService.createResource(user,"ResourcesServiceTest","ResourcesServiceTest",ResourceType.FILE,mockMultipartFile,-1,"/"); logger.info(result.toString()); Assert.assertEquals(Status.RESOURCE_FILE_IS_EMPTY.getMsg(),result.getMsg()); @@ -111,31 +111,42 @@ public class ResourcesServiceTest { mockMultipartFile = new MockMultipartFile("test.pdf","test.pdf","pdf",new String("test").getBytes()); PowerMockito.when(FileUtils.suffix("test.pdf")).thenReturn("pdf"); PowerMockito.when(FileUtils.suffix("ResourcesServiceTest.jar")).thenReturn("jar"); - result = resourcesService.createResource(user,"ResourcesServiceTest.jar","ResourcesServiceTest",ResourceType.FILE,mockMultipartFile); + result = resourcesService.createResource(user,"ResourcesServiceTest.jar","ResourcesServiceTest",ResourceType.FILE,mockMultipartFile,-1,"/"); logger.info(result.toString()); Assert.assertEquals(Status.RESOURCE_SUFFIX_FORBID_CHANGE.getMsg(),result.getMsg()); //UDF_RESOURCE_SUFFIX_NOT_JAR mockMultipartFile = new MockMultipartFile("ResourcesServiceTest.pdf","ResourcesServiceTest.pdf","pdf",new String("test").getBytes()); PowerMockito.when(FileUtils.suffix("ResourcesServiceTest.pdf")).thenReturn("pdf"); - result = resourcesService.createResource(user,"ResourcesServiceTest.pdf","ResourcesServiceTest",ResourceType.UDF,mockMultipartFile); + result = resourcesService.createResource(user,"ResourcesServiceTest.pdf","ResourcesServiceTest",ResourceType.UDF,mockMultipartFile,-1,"/"); logger.info(result.toString()); Assert.assertEquals(Status.UDF_RESOURCE_SUFFIX_NOT_JAR.getMsg(),result.getMsg()); - //UDF_RESOURCE_SUFFIX_NOT_JAR - Mockito.when(tenantMapper.queryById(0)).thenReturn(getTenant()); - Mockito.when(resourcesMapper.queryResourceList("ResourcesServiceTest.jar", 0, 1)).thenReturn(getResourceList()); - mockMultipartFile = new MockMultipartFile("ResourcesServiceTest.jar","ResourcesServiceTest.jar","pdf",new String("test").getBytes()); - result = resourcesService.createResource(user,"ResourcesServiceTest.jar","ResourcesServiceTest",ResourceType.UDF,mockMultipartFile); + } + + @Test + public void testCreateDirecotry(){ + + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false); + User user = new User(); + //HDFS_NOT_STARTUP + Result result = resourcesService.createDirectory(user,"directoryTest","directory test",ResourceType.FILE,-1,"/"); logger.info(result.toString()); - Assert.assertEquals(Status.RESOURCE_EXIST.getMsg(),result.getMsg()); + Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(),result.getMsg()); - //SUCCESS - Mockito.when(resourcesMapper.queryResourceList("ResourcesServiceTest.jar", 0, 1)).thenReturn(new ArrayList<>()); - result = resourcesService.createResource(user,"ResourcesServiceTest.jar","ResourcesServiceTest",ResourceType.UDF,mockMultipartFile); + //PARENT_RESOURCE_NOT_EXIST + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); + Mockito.when(resourcesMapper.selectById(Mockito.anyInt())).thenReturn(null); + result = resourcesService.createDirectory(user,"directoryTest","directory test",ResourceType.FILE,1,"/"); logger.info(result.toString()); - Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg()); + Assert.assertEquals(Status.PARENT_RESOURCE_NOT_EXIST.getMsg(),result.getMsg()); + //RESOURCE_EXIST + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); + Mockito.when(resourcesMapper.queryResourceList("/directoryTest", 0, 0)).thenReturn(getResourceList()); + result = resourcesService.createDirectory(user,"directoryTest","directory test",ResourceType.FILE,-1,"/"); + logger.info(result.toString()); + Assert.assertEquals(Status.RESOURCE_EXIST.getMsg(),result.getMsg()); } @@ -163,41 +174,46 @@ public class ResourcesServiceTest { //SUCCESS user.setId(1); - result = resourcesService.updateResource(user,1,"ResourcesServiceTest.jar","ResourcesServiceTest.jar",ResourceType.FILE); + Mockito.when(userMapper.queryDetailsById(1)).thenReturn(getUser()); + Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); + + result = resourcesService.updateResource(user,1,"ResourcesServiceTest.jar","ResourcesServiceTest",ResourceType.FILE); logger.info(result.toString()); Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg()); //RESOURCE_EXIST - Mockito.when(resourcesMapper.queryResourceList("ResourcesServiceTest1.jar", 0, 0)).thenReturn(getResourceList()); - result = resourcesService.updateResource(user,1,"ResourcesServiceTest1.jar","ResourcesServiceTest1.jar",ResourceType.FILE); + Mockito.when(resourcesMapper.queryResourceList("/ResourcesServiceTest1.jar", 0, 0)).thenReturn(getResourceList()); + result = resourcesService.updateResource(user,1,"ResourcesServiceTest1.jar","ResourcesServiceTest",ResourceType.FILE); logger.info(result.toString()); Assert.assertEquals(Status.RESOURCE_EXIST.getMsg(),result.getMsg()); //USER_NOT_EXIST - result = resourcesService.updateResource(user,1,"ResourcesServiceTest1.jar","ResourcesServiceTest1.jar",ResourceType.UDF); + Mockito.when(userMapper.queryDetailsById(Mockito.anyInt())).thenReturn(null); + result = resourcesService.updateResource(user,1,"ResourcesServiceTest1.jar","ResourcesServiceTest",ResourceType.UDF); logger.info(result.toString()); Assert.assertTrue(Status.USER_NOT_EXIST.getCode() == result.getCode()); //TENANT_NOT_EXIST Mockito.when(userMapper.queryDetailsById(1)).thenReturn(getUser()); - result = resourcesService.updateResource(user,1,"ResourcesServiceTest1.jar","ResourcesServiceTest1.jar",ResourceType.UDF); + Mockito.when(tenantMapper.queryById(Mockito.anyInt())).thenReturn(null); + result = resourcesService.updateResource(user,1,"ResourcesServiceTest1.jar","ResourcesServiceTest",ResourceType.UDF); logger.info(result.toString()); Assert.assertEquals(Status.TENANT_NOT_EXIST.getMsg(),result.getMsg()); //RESOURCE_NOT_EXIST Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); - PowerMockito.when(HadoopUtils.getHdfsFilename(Mockito.any(), Mockito.any())).thenReturn("test1"); + PowerMockito.when(HadoopUtils.getHdfsResourceFileName(Mockito.any(), Mockito.any())).thenReturn("test1"); try { Mockito.when(hadoopUtils.exists("test")).thenReturn(true); } catch (IOException e) { e.printStackTrace(); } - result = resourcesService.updateResource(user,1,"ResourcesServiceTest1.jar","ResourcesServiceTest1.jar",ResourceType.UDF); + result = resourcesService.updateResource(user,1,"ResourcesServiceTest1.jar","ResourcesServiceTest",ResourceType.UDF); logger.info(result.toString()); Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(),result.getMsg()); //SUCCESS - PowerMockito.when(HadoopUtils.getHdfsFilename(Mockito.any(), Mockito.any())).thenReturn("test"); + PowerMockito.when(HadoopUtils.getHdfsResourceFileName(Mockito.any(), Mockito.any())).thenReturn("test"); result = resourcesService.updateResource(user,1,"ResourcesServiceTest1.jar","ResourcesServiceTest1.jar",ResourceType.UDF); logger.info(result.toString()); Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg()); @@ -212,8 +228,8 @@ public class ResourcesServiceTest { resourcePage.setTotal(1); resourcePage.setRecords(getResourceList()); Mockito.when(resourcesMapper.queryResourcePaging(Mockito.any(Page.class), - Mockito.eq(0), Mockito.eq(0), Mockito.eq("test"))).thenReturn(resourcePage); - Map result = resourcesService.queryResourceListPaging(loginUser,ResourceType.FILE,"test",1,10); + Mockito.eq(0),Mockito.eq(-1), Mockito.eq(0), Mockito.eq("test"))).thenReturn(resourcePage); + Map result = resourcesService.queryResourceListPaging(loginUser,-1,ResourceType.FILE,"test",1,10); logger.info(result.toString()); Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); PageInfo pageInfo = (PageInfo) result.get(Constants.DATA_LIST); @@ -263,6 +279,7 @@ public class ResourcesServiceTest { //TENANT_NOT_EXIST loginUser.setUserType(UserType.ADMIN_USER); loginUser.setTenantId(2); + Mockito.when(userMapper.queryDetailsById(Mockito.anyInt())).thenReturn(loginUser); result = resourcesService.delete(loginUser,1); logger.info(result.toString()); Assert.assertEquals(Status.TENANT_NOT_EXIST.getMsg(), result.getMsg()); @@ -285,14 +302,20 @@ public class ResourcesServiceTest { User user = new User(); user.setId(1); - Mockito.when(resourcesMapper.queryResourceList("test", 0, 0)).thenReturn(getResourceList()); - Result result = resourcesService.verifyResourceName("test",ResourceType.FILE,user); + Mockito.when(resourcesMapper.queryResourceList("/ResourcesServiceTest.jar", 0, 0)).thenReturn(getResourceList()); + Result result = resourcesService.verifyResourceName("/ResourcesServiceTest.jar",ResourceType.FILE,user); logger.info(result.toString()); Assert.assertEquals(Status.RESOURCE_EXIST.getMsg(), result.getMsg()); //TENANT_NOT_EXIST Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); - result = resourcesService.verifyResourceName("test1",ResourceType.FILE,user); + String unExistFullName = "/test.jar"; + try { + Mockito.when(hadoopUtils.exists(unExistFullName)).thenReturn(false); + } catch (IOException e) { + logger.error("hadoop error",e); + } + result = resourcesService.verifyResourceName("/test.jar",ResourceType.FILE,user); logger.info(result.toString()); Assert.assertEquals(Status.TENANT_NOT_EXIST.getMsg(), result.getMsg()); @@ -304,10 +327,10 @@ public class ResourcesServiceTest { } catch (IOException e) { logger.error("hadoop error",e); } - PowerMockito.when(HadoopUtils.getHdfsFilename("123", "test1")).thenReturn("test"); - result = resourcesService.verifyResourceName("test1",ResourceType.FILE,user); + PowerMockito.when(HadoopUtils.getHdfsResourceFileName("123", "test1")).thenReturn("test"); + result = resourcesService.verifyResourceName("/ResourcesServiceTest.jar",ResourceType.FILE,user); logger.info(result.toString()); - Assert.assertTrue(Status.RESOURCE_FILE_EXIST.getCode()==result.getCode()); + Assert.assertTrue(Status.RESOURCE_EXIST.getCode()==result.getCode()); //SUCCESS result = resourcesService.verifyResourceName("test2",ResourceType.FILE,user); @@ -389,14 +412,14 @@ public class ResourcesServiceTest { PowerMockito.when(HadoopUtils.getHdfsUdfDir("udfDir")).thenReturn("udfDir"); User user = getUser(); //HDFS_NOT_STARTUP - Result result = resourcesService.onlineCreateResource(user,ResourceType.FILE,"test","jar","desc","content"); + Result result = resourcesService.onlineCreateResource(user,ResourceType.FILE,"test","jar","desc","content",-1,"/"); logger.info(result.toString()); Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(),result.getMsg()); //RESOURCE_SUFFIX_NOT_SUPPORT_VIEW PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); PowerMockito.when(FileUtils.getResourceViewSuffixs()).thenReturn("class"); - result = resourcesService.onlineCreateResource(user,ResourceType.FILE,"test","jar","desc","content"); + result = resourcesService.onlineCreateResource(user,ResourceType.FILE,"test","jar","desc","content",-1,"/"); logger.info(result.toString()); Assert.assertEquals(Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW.getMsg(),result.getMsg()); @@ -404,7 +427,7 @@ public class ResourcesServiceTest { try { PowerMockito.when(FileUtils.getResourceViewSuffixs()).thenReturn("jar"); Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); - result = resourcesService.onlineCreateResource(user, ResourceType.FILE, "test", "jar", "desc", "content"); + result = resourcesService.onlineCreateResource(user, ResourceType.FILE, "test", "jar", "desc", "content",-1,"/"); }catch (RuntimeException ex){ logger.info(result.toString()); Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), ex.getMessage()); @@ -413,7 +436,7 @@ public class ResourcesServiceTest { //SUCCESS Mockito.when(FileUtils.getUploadFilename(Mockito.anyString(), Mockito.anyString())).thenReturn("test"); PowerMockito.when(FileUtils.writeContent2File(Mockito.anyString(), Mockito.anyString())).thenReturn(true); - result = resourcesService.onlineCreateResource(user,ResourceType.FILE,"test","jar","desc","content"); + result = resourcesService.onlineCreateResource(user,ResourceType.FILE,"test","jar","desc","content",-1,"/"); logger.info(result.toString()); Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg()); @@ -584,13 +607,26 @@ public class ResourcesServiceTest { private Resource getResource(){ Resource resource = new Resource(); + resource.setPid(-1); resource.setUserId(1); resource.setDescription("ResourcesServiceTest.jar"); resource.setAlias("ResourcesServiceTest.jar"); + resource.setFullName("/ResourcesServiceTest.jar"); resource.setType(ResourceType.FILE); return resource; } + private Resource getUdfResource(){ + + Resource resource = new Resource(); + resource.setUserId(1); + resource.setDescription("udfTest"); + resource.setAlias("udfTest.jar"); + resource.setFullName("/udfTest.jar"); + resource.setType(ResourceType.UDF); + return resource; + } + private UdfFunc getUdfFunc(){ UdfFunc udfFunc = new UdfFunc(); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UserAlertGroupServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UserAlertGroupServiceTest.java new file mode 100644 index 0000000000..24b1d5a98b --- /dev/null +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UserAlertGroupServiceTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.api.service; + +import org.apache.dolphinscheduler.dao.mapper.UserAlertGroupMapper; +import static org.junit.Assert.assertEquals; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +/** + * + */ +@RunWith(MockitoJUnitRunner.class) +public class UserAlertGroupServiceTest { + + @InjectMocks + UserAlertGroupService userAlertGroupService; + + @Mock + UserAlertGroupMapper userAlertGroupMapper; + + @Test + public void deleteByAlertGroupId() { + + Integer groupId = 1; + userAlertGroupService.deleteByAlertGroupId(groupId); + ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(Integer.class); + + Mockito.verify(userAlertGroupMapper).deleteByAlertgroupId(argumentCaptor.capture()); + assertEquals(argumentCaptor.getValue(), groupId); + + } + +} \ No newline at end of file diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/CheckUtilsTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/CheckUtilsTest.java index 308ed8e9b6..ccc231fcf6 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/CheckUtilsTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/CheckUtilsTest.java @@ -43,6 +43,7 @@ import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; import java.util.Map; import static org.junit.Assert.*; @@ -173,7 +174,11 @@ public class CheckUtilsTest { // MapreduceParameters MapreduceParameters mapreduceParameters = new MapreduceParameters(); assertFalse(CheckUtils.checkTaskNodeParameters(JSONUtils.toJsonString(mapreduceParameters), TaskType.MR.toString())); - mapreduceParameters.setMainJar(new ResourceInfo()); + + ResourceInfo resourceInfoMapreduce = new ResourceInfo(); + resourceInfoMapreduce.setId(1); + resourceInfoMapreduce.setRes(""); + mapreduceParameters.setMainJar(resourceInfoMapreduce); mapreduceParameters.setProgramType(ProgramType.JAVA); assertTrue(CheckUtils.checkTaskNodeParameters(JSONUtils.toJsonString(mapreduceParameters), TaskType.MR.toString())); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java index 73655e7a9d..c46635c1e8 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java @@ -972,7 +972,8 @@ public final class Constants { public static final String JDBC_POSTGRESQL = "jdbc:postgresql://"; public static final String JDBC_HIVE_2 = "jdbc:hive2://"; public static final String JDBC_CLICKHOUSE = "jdbc:clickhouse://"; - public static final String JDBC_ORACLE = "jdbc:oracle:thin:@//"; + public static final String JDBC_ORACLE_SID = "jdbc:oracle:thin:@"; + public static final String JDBC_ORACLE_SERVICE_NAME = "jdbc:oracle:thin:@//"; public static final String JDBC_SQLSERVER = "jdbc:sqlserver://"; public static final String JDBC_DB2 = "jdbc:db2://"; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AuthorizationType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AuthorizationType.java index 1c371e799e..633f5f9623 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AuthorizationType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AuthorizationType.java @@ -23,13 +23,17 @@ import com.baomidou.mybatisplus.annotation.EnumValue; */ public enum AuthorizationType { /** - * 0 RESOURCE_FILE; + * 0 RESOURCE_FILE_ID; + * 0 RESOURCE_FILE_NAME; + * 1 UDF_FILE; * 1 DATASOURCE; * 2 UDF; */ - RESOURCE_FILE(0, "resource file"), - DATASOURCE(1, "data source"), - UDF(2, "udf function"); + RESOURCE_FILE_ID(0, "resource file id"), + RESOURCE_FILE_NAME(1, "resource file name"), + UDF_FILE(2, "udf file"), + DATASOURCE(3, "data source"), + UDF(4, "udf function"); AuthorizationType(int code, String descp){ this.code = code; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/DbConnectType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/DbConnectType.java new file mode 100644 index 0000000000..ef0f454ff6 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/DbConnectType.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.enums; + +import com.baomidou.mybatisplus.annotation.EnumValue; + +public enum DbConnectType { + + ORACLE_SERVICE_NAME(0, "Oracle Service Name"), + ORACLE_SID(1, "Oracle SID"); + + DbConnectType(int code, String descp) { + this.code = code; + this.descp = descp; + } + + @EnumValue + private final int code; + + private final String descp; + + public int getCode() { + return code; + } + + public String getDescp() { + return descp; + } + +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskStateType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskStateType.java index 695f0fd880..200f90709a 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskStateType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskStateType.java @@ -60,7 +60,7 @@ public enum TaskStateType { default: break; } - return null; + return new int[0]; } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ZKNodeType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ZKNodeType.java index 8982c2a838..b4b3c59321 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ZKNodeType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ZKNodeType.java @@ -22,10 +22,10 @@ package org.apache.dolphinscheduler.common.enums; public enum ZKNodeType { /** - * 0 do not send warning; - * 1 send if process success; - * 2 send if process failed; - * 3 send if process ending; + * 0 master node; + * 1 worker node; + * 2 dead_server node; + * 3 task_queue node; */ MASTER, WORKER, DEAD_SERVER, TASK_QUEUE; } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java index c0ad907dca..1ed398a693 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java @@ -293,14 +293,14 @@ public class TaskNode { public TaskTimeoutParameter getTaskTimeoutParameter() { if(StringUtils.isNotEmpty(this.getTimeout())){ String formatStr = String.format("%s,%s", TaskTimeoutStrategy.WARN.name(), TaskTimeoutStrategy.FAILED.name()); - String timeout = this.getTimeout().replace(formatStr,TaskTimeoutStrategy.WARNFAILED.name()); - return JSON.parseObject(timeout,TaskTimeoutParameter.class); + String taskTimeout = this.getTimeout().replace(formatStr,TaskTimeoutStrategy.WARNFAILED.name()); + return JSON.parseObject(taskTimeout,TaskTimeoutParameter.class); } return new TaskTimeoutParameter(false); } public boolean isConditionsTask(){ - return this.getType().toUpperCase().equals(TaskType.CONDITIONS.toString()); + return TaskType.CONDITIONS.toString().equalsIgnoreCase(this.getType()); } @Override diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/process/ResourceInfo.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/process/ResourceInfo.java index 3c95ac648b..a7fc0839eb 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/process/ResourceInfo.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/process/ResourceInfo.java @@ -23,6 +23,16 @@ public class ResourceInfo { /** * res the name of the resource that was uploaded */ + private int id; + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + private String res; public String getRes() { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/AbstractParameters.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/AbstractParameters.java index 2d0322a6d7..ae78caf881 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/AbstractParameters.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/AbstractParameters.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.common.task; import org.apache.dolphinscheduler.common.process.Property; +import org.apache.dolphinscheduler.common.process.ResourceInfo; import java.util.LinkedHashMap; import java.util.List; @@ -31,7 +32,7 @@ public abstract class AbstractParameters implements IParameters { public abstract boolean checkParameters(); @Override - public abstract List getResourceFilesList(); + public abstract List getResourceFilesList(); /** * local parameters diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/IParameters.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/IParameters.java index 8fb49eb1fa..88a2b54761 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/IParameters.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/IParameters.java @@ -16,6 +16,8 @@ */ package org.apache.dolphinscheduler.common.task; +import org.apache.dolphinscheduler.common.process.ResourceInfo; + import java.util.List; /** @@ -34,5 +36,5 @@ public interface IParameters { * * @return resource files list */ - List getResourceFilesList(); + List getResourceFilesList(); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/conditions/ConditionsParameters.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/conditions/ConditionsParameters.java index 5714b5ef3e..7f0f2c8079 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/conditions/ConditionsParameters.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/conditions/ConditionsParameters.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.common.task.conditions; import org.apache.dolphinscheduler.common.enums.DependentRelation; import org.apache.dolphinscheduler.common.model.DependentTaskModel; +import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.AbstractParameters; import java.util.List; @@ -41,7 +42,7 @@ public class ConditionsParameters extends AbstractParameters { } @Override - public List getResourceFilesList() { + public List getResourceFilesList() { return null; } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/datax/DataxParameters.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/datax/DataxParameters.java index f153360d63..872b3aa174 100755 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/datax/DataxParameters.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/datax/DataxParameters.java @@ -20,6 +20,7 @@ import java.util.ArrayList; import java.util.List; import org.apache.commons.lang.StringUtils; +import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.AbstractParameters; /** @@ -198,7 +199,7 @@ public class DataxParameters extends AbstractParameters { } @Override - public List getResourceFilesList() { + public List getResourceFilesList() { return new ArrayList<>(); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/dependent/DependentParameters.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/dependent/DependentParameters.java index 9ff1405722..5f2e0e1853 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/dependent/DependentParameters.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/dependent/DependentParameters.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.common.task.dependent; import org.apache.dolphinscheduler.common.enums.DependentRelation; import org.apache.dolphinscheduler.common.model.DependentTaskModel; +import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.AbstractParameters; import java.util.ArrayList; @@ -36,7 +37,7 @@ public class DependentParameters extends AbstractParameters { } @Override - public List getResourceFilesList() { + public List getResourceFilesList() { return new ArrayList<>(); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/flink/FlinkParameters.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/flink/FlinkParameters.java index 1fbd9ab354..05cbb1d794 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/flink/FlinkParameters.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/flink/FlinkParameters.java @@ -19,10 +19,10 @@ package org.apache.dolphinscheduler.common.task.flink; import org.apache.dolphinscheduler.common.enums.ProgramType; import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.AbstractParameters; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; -import java.util.Collections; +import java.util.ArrayList; import java.util.List; -import java.util.stream.Collectors; /** * spark parameters @@ -50,35 +50,35 @@ public class FlinkParameters extends AbstractParameters { private String mainArgs; /** - * slot个数 + * slot count */ private int slot; /** - *Yarn application的名字 + *Yarn application name */ private String appName; /** - * taskManager 数量 + * taskManager count */ private int taskManager; /** - * jobManagerMemory 内存大小 + * job manager memory */ private String jobManagerMemory ; /** - * taskManagerMemory内存大小 + * task manager memory */ private String taskManagerMemory; /** * resource list */ - private List resourceList; + private List resourceList = new ArrayList<>(); /** * The YARN queue to submit to @@ -207,16 +207,11 @@ public class FlinkParameters extends AbstractParameters { @Override - public List getResourceFilesList() { - if(resourceList != null ) { - List resourceFiles = resourceList.stream() - .map(ResourceInfo::getRes).collect(Collectors.toList()); - if(mainJar != null) { - resourceFiles.add(mainJar.getRes()); - } - return resourceFiles; + public List getResourceFilesList() { + if (mainJar != null && !resourceList.contains(mainJar)) { + resourceList.add(mainJar); } - return Collections.emptyList(); + return resourceList; } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/http/HttpParameters.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/http/HttpParameters.java index 00b01afce3..54284bd8b0 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/http/HttpParameters.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/http/HttpParameters.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.common.task.http; import org.apache.dolphinscheduler.common.enums.HttpCheckCondition; import org.apache.dolphinscheduler.common.enums.HttpMethod; import org.apache.dolphinscheduler.common.process.HttpProperty; +import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.commons.lang.StringUtils; @@ -62,7 +63,7 @@ public class HttpParameters extends AbstractParameters { } @Override - public List getResourceFilesList() { + public List getResourceFilesList() { return new ArrayList<>(); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/mr/MapreduceParameters.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/mr/MapreduceParameters.java index 31c9c7292f..5126e82e85 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/mr/MapreduceParameters.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/mr/MapreduceParameters.java @@ -19,10 +19,10 @@ package org.apache.dolphinscheduler.common.task.mr; import org.apache.dolphinscheduler.common.enums.ProgramType; import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.AbstractParameters; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; -import java.util.Collections; +import java.util.ArrayList; import java.util.List; -import java.util.stream.Collectors; public class MapreduceParameters extends AbstractParameters { @@ -54,7 +54,7 @@ public class MapreduceParameters extends AbstractParameters { /** * resource list */ - private List resourceList; + private List resourceList = new ArrayList<>(); /** * program type @@ -125,16 +125,12 @@ public class MapreduceParameters extends AbstractParameters { } @Override - public List getResourceFilesList() { - if(resourceList != null ) { - List resourceFiles = resourceList.stream() - .map(ResourceInfo::getRes).collect(Collectors.toList()); - if(mainJar != null) { - resourceFiles.add(mainJar.getRes()); - } - return resourceFiles; + public List getResourceFilesList() { + if (mainJar != null && !resourceList.contains(mainJar)) { + resourceList.add(mainJar); } - return Collections.emptyList(); + + return resourceList; } @Override diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/procedure/ProcedureParameters.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/procedure/ProcedureParameters.java index 56ae65547d..2811f10380 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/procedure/ProcedureParameters.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/procedure/ProcedureParameters.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.common.task.procedure; +import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.commons.lang.StringUtils; @@ -74,7 +75,7 @@ public class ProcedureParameters extends AbstractParameters { } @Override - public List getResourceFilesList() { + public List getResourceFilesList() { return new ArrayList<>(); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/python/PythonParameters.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/python/PythonParameters.java index ae9cb4c7da..35dbd8ed86 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/python/PythonParameters.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/python/PythonParameters.java @@ -21,7 +21,6 @@ import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.AbstractParameters; import java.util.List; -import java.util.stream.Collectors; public class PythonParameters extends AbstractParameters { /** @@ -56,12 +55,7 @@ public class PythonParameters extends AbstractParameters { } @Override - public List getResourceFilesList() { - if (resourceList != null) { - return resourceList.stream() - .map(p -> p.getRes()).collect(Collectors.toList()); - } - - return null; + public List getResourceFilesList() { + return this.resourceList; } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/shell/ShellParameters.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/shell/ShellParameters.java index 85b8acb46a..e11e59600b 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/shell/ShellParameters.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/shell/ShellParameters.java @@ -59,12 +59,7 @@ public class ShellParameters extends AbstractParameters { } @Override - public List getResourceFilesList() { - if (resourceList != null) { - return resourceList.stream() - .map(p -> p.getRes()).collect(Collectors.toList()); - } - - return null; + public List getResourceFilesList() { + return resourceList; } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/spark/SparkParameters.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/spark/SparkParameters.java index dbafddfddd..4e58201bf3 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/spark/SparkParameters.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/spark/SparkParameters.java @@ -19,9 +19,10 @@ package org.apache.dolphinscheduler.common.task.spark; import org.apache.dolphinscheduler.common.enums.ProgramType; import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.AbstractParameters; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import java.util.ArrayList; import java.util.List; -import java.util.stream.Collectors; /** * spark parameters @@ -77,7 +78,7 @@ public class SparkParameters extends AbstractParameters { /** * resource list */ - private List resourceList; + private List resourceList = new ArrayList<>(); /** * The YARN queue to submit to @@ -218,15 +219,12 @@ public class SparkParameters extends AbstractParameters { return mainJar != null && programType != null && sparkVersion != null; } - @Override - public List getResourceFilesList() { - if(resourceList !=null ) { - this.resourceList.add(mainJar); - return resourceList.stream() - .map(ResourceInfo::getRes).collect(Collectors.toList()); + public List getResourceFilesList() { + if (mainJar != null && !resourceList.contains(mainJar)) { + resourceList.add(mainJar); } - return null; + return resourceList; } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sql/SqlParameters.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sql/SqlParameters.java index d65204a386..4604234e8f 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sql/SqlParameters.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sql/SqlParameters.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.common.task.sql; +import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.commons.lang.StringUtils; @@ -189,7 +190,7 @@ public class SqlParameters extends AbstractParameters { } @Override - public List getResourceFilesList() { + public List getResourceFilesList() { return new ArrayList<>(); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sqoop/SqoopParameters.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sqoop/SqoopParameters.java index fb65df6c1b..7f02f42387 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sqoop/SqoopParameters.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sqoop/SqoopParameters.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.common.task.sqoop; +import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.utils.StringUtils; @@ -111,7 +112,7 @@ public class SqoopParameters extends AbstractParameters { } @Override - public List getResourceFilesList() { + public List getResourceFilesList() { return new ArrayList<>(); } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/subprocess/SubProcessParameters.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/subprocess/SubProcessParameters.java index c7784de8dd..46f0e8510c 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/subprocess/SubProcessParameters.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/subprocess/SubProcessParameters.java @@ -15,6 +15,7 @@ * limitations under the License. */ package org.apache.dolphinscheduler.common.task.subprocess; +import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.AbstractParameters; import java.util.ArrayList; @@ -42,7 +43,7 @@ public class SubProcessParameters extends AbstractParameters { } @Override - public List getResourceFilesList() { + public List getResourceFilesList() { return new ArrayList<>(); } } \ No newline at end of file diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/Stopper.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/Stopper.java index cad6914cb8..7353291054 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/Stopper.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/Stopper.java @@ -23,7 +23,7 @@ import java.util.concurrent.atomic.AtomicBoolean; */ public class Stopper { - private static volatile AtomicBoolean signal = new AtomicBoolean(false); + private static AtomicBoolean signal = new AtomicBoolean(false); public static final boolean isStopped(){ return signal.get(); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadPoolExecutors.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadPoolExecutors.java index f88ea6d127..2744803f21 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadPoolExecutors.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadPoolExecutors.java @@ -74,21 +74,21 @@ public class ThreadPoolExecutors { * @param event */ public void execute(final Runnable event) { - Executor executor = getExecutor(); - if (executor == null) { - logger.error("Cannot execute [" + event + "] because the executor is missing."); + Executor eventExecutor = getExecutor(); + if (eventExecutor == null) { + logger.error("Cannot execute [{}}] because the executor is missing.", event); } else { - executor.execute(event); + eventExecutor.execute(event); } } public Future submit(Runnable event) { - Executor executor = getExecutor(); - if (executor == null) { - logger.error("Cannot submit [" + event + "] because the executor is missing."); + Executor eventExecutor = getExecutor(); + if (eventExecutor == null) { + logger.error("Cannot submit [{}}] because the executor is missing.", event); } else { - return executor.submit(event); + return eventExecutor.submit(event); } return null; @@ -97,11 +97,11 @@ public class ThreadPoolExecutors { public Future submit(Callable task) { - Executor executor = getExecutor(); - if (executor == null) { - logger.error("Cannot submit [" + task + "] because the executor is missing."); + Executor taskExecutor = getExecutor(); + if (taskExecutor == null) { + logger.error("Cannot submit [{}] because the executor is missing.", task); } else { - return executor.submit(task); + return taskExecutor.submit(task); } return null; @@ -110,8 +110,8 @@ public class ThreadPoolExecutors { public void printStatus() { - Executor executor = getExecutor(); - executor.getStatus().dumpInfo(); + Executor printExecutor = getExecutor(); + printExecutor.getStatus().dumpInfo(); } @@ -125,7 +125,7 @@ public class ThreadPoolExecutors { List wasRunning = executor.threadPoolExecutor .shutdownNow(); if (!wasRunning.isEmpty()) { - logger.info(executor + " had " + wasRunning + " on shutdown"); + logger.info("{} had {} on shutdown", executor, wasRunning); } } } @@ -138,7 +138,7 @@ public class ThreadPoolExecutors { /** * how long to retain excess threads */ - final long keepAliveTimeInMillis = 1000; + static final long KEEP_ALIVE_TIME_IN_MILLIS = 1000; /** * the thread pool executor that services the requests */ @@ -146,7 +146,7 @@ public class ThreadPoolExecutors { /** * work queue to use - unbounded queue */ - final BlockingQueue q = new LinkedBlockingQueue(); + final BlockingQueue q = new LinkedBlockingQueue<>(); private final String name; private static final AtomicLong seqids = new AtomicLong(0); private final long id; @@ -156,7 +156,7 @@ public class ThreadPoolExecutors { this.name = name; //create the thread pool executor this.threadPoolExecutor = new TrackingThreadPoolExecutor( - maxThreads, maxThreads, keepAliveTimeInMillis, + maxThreads, maxThreads, KEEP_ALIVE_TIME_IN_MILLIS, TimeUnit.MILLISECONDS, q); // name the threads for this threadpool ThreadFactoryBuilder tfb = new ThreadFactoryBuilder(); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ConnectionUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ConnectionUtils.java index c1c3ff5d57..f8ea0e7188 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ConnectionUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ConnectionUtils.java @@ -16,86 +16,35 @@ */ package org.apache.dolphinscheduler.common.utils; +import java.util.Arrays; +import java.util.Objects; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.sql.*; - public class ConnectionUtils { - public static final Logger logger = LoggerFactory.getLogger(ConnectionUtils.class); - - private static ConnectionUtils instance; - - ConnectionUtils() { - } - - public static ConnectionUtils getInstance() { - if (null == instance) { - syncInit(); - } - return instance; - } - - private static synchronized void syncInit() { - if (instance == null) { - instance = new ConnectionUtils(); - } - } - - public void release(ResultSet rs, Statement stmt, Connection conn) { - try { - if (rs != null) { - rs.close(); - rs = null; - } - } catch (SQLException e) { - logger.error(e.getMessage(),e); - } finally { - try { - if (stmt != null) { - stmt.close(); - stmt = null; - } - } catch (SQLException e) { - logger.error(e.getMessage(),e); - } finally { - try { - if (conn != null) { - conn.close(); - conn = null; - } - } catch (SQLException e) { - logger.error(e.getMessage(),e); - } - } - } - } - - public static void releaseResource(ResultSet rs, PreparedStatement ps, Connection conn) { - ConnectionUtils.getInstance().release(rs,ps,conn); - if (null != rs) { - try { - rs.close(); - } catch (SQLException e) { - logger.error(e.getMessage(),e); - } - } - - if (null != ps) { - try { - ps.close(); - } catch (SQLException e) { - logger.error(e.getMessage(),e); - } - } - - if (null != conn) { - try { - conn.close(); - } catch (SQLException e) { - logger.error(e.getMessage(),e); - } - } - } + public static final Logger logger = LoggerFactory.getLogger(ConnectionUtils.class); + + private ConnectionUtils() { + throw new IllegalStateException("ConnectionUtils class"); + } + + /** + * release resource + * @param resources resources + */ + public static void releaseResource(AutoCloseable... resources) { + + if (resources == null || resources.length == 0) { + return; + } + Arrays.stream(resources).filter(Objects::nonNull) + .forEach(resource -> { + try { + resource.close(); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + }); + } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java index 6c42704b47..c89f3c835c 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java @@ -26,6 +26,7 @@ import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONException; import com.alibaba.fastjson.JSONObject; import org.apache.commons.io.IOUtils; +import org.apache.dolphinscheduler.common.enums.ResourceType; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.*; import org.apache.hadoop.fs.FileSystem; @@ -415,6 +416,22 @@ public class HadoopUtils implements Closeable { } } + /** + * hdfs resource dir + * + * @param tenantCode tenant code + * @return hdfs resource dir + */ + public static String getHdfsDir(ResourceType resourceType,String tenantCode) { + String hdfsDir = ""; + if (resourceType.equals(ResourceType.FILE)) { + hdfsDir = getHdfsResDir(tenantCode); + } else if (resourceType.equals(ResourceType.UDF)) { + hdfsDir = getHdfsUdfDir(tenantCode); + } + return hdfsDir; + } + /** * hdfs resource dir * @@ -450,22 +467,42 @@ public class HadoopUtils implements Closeable { * get absolute path and name for file on hdfs * * @param tenantCode tenant code - * @param filename file name + * @param fileName file name + * @return get absolute path and name for file on hdfs + */ + + /** + * get hdfs file name + * + * @param resourceType resource type + * @param tenantCode tenant code + * @param fileName file name + * @return hdfs file name + */ + public static String getHdfsFileName(ResourceType resourceType, String tenantCode, String fileName) { + return String.format("%s/%s", getHdfsDir(resourceType,tenantCode), fileName); + } + + /** + * get absolute path and name for resource file on hdfs + * + * @param tenantCode tenant code + * @param fileName file name * @return get absolute path and name for file on hdfs */ - public static String getHdfsFilename(String tenantCode, String filename) { - return String.format("%s/%s", getHdfsResDir(tenantCode), filename); + public static String getHdfsResourceFileName(String tenantCode, String fileName) { + return String.format("%s/%s", getHdfsResDir(tenantCode), fileName); } /** * get absolute path and name for udf file on hdfs * * @param tenantCode tenant code - * @param filename file name + * @param fileName file name * @return get absolute path and name for udf file on hdfs */ - public static String getHdfsUdfFilename(String tenantCode, String filename) { - return String.format("%s/%s", getHdfsUdfDir(tenantCode), filename); + public static String getHdfsUdfFileName(String tenantCode, String fileName) { + return String.format("%s/%s", getHdfsUdfDir(tenantCode), fileName); } /** diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java index b011c0bc4e..4df09d1c15 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java @@ -352,13 +352,7 @@ public class OSUtils { return sb.toString(); } finally { - if (br != null) { - try { - br.close(); - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - } + IOUtils.closeQuietly(br); } } @@ -408,7 +402,7 @@ public class OSUtils { * whether is windows * @return true if windows */ - public static boolean isWindows() { ; + public static boolean isWindows() { return getOSName().startsWith("Windows"); } @@ -425,13 +419,13 @@ public class OSUtils { * @return check memory and cpu usage */ public static Boolean checkResource(double systemCpuLoad, double systemReservedMemory){ - // judging usage + // system load average double loadAverage = OSUtils.loadAverage(); - // + // system available physical memory double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize(); if(loadAverage > systemCpuLoad || availablePhysicalMemorySize < systemReservedMemory){ - logger.warn("load or availablePhysicalMemorySize(G) is too high, it's availablePhysicalMemorySize(G):{},loadAvg:{}", availablePhysicalMemorySize , loadAverage); + logger.warn("load is too high or availablePhysicalMemorySize(G) is too low, it's availablePhysicalMemorySize(G):{},loadAvg:{}", availablePhysicalMemorySize , loadAverage); return false; }else{ return true; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java index ca928c94d5..725188d6b0 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java @@ -71,7 +71,7 @@ public class PropertyUtils { * * @return judge whether resource upload startup */ - public static Boolean getResUploadStartupState(){ + public static boolean getResUploadStartupState(){ String resUploadStartupType = PropertyUtils.getString(Constants.RES_UPLOAD_STARTUP_TYPE); ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType); return resUploadType == ResUploadType.HDFS || resUploadType == ResUploadType.S3; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/placeholder/TimePlaceholderUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/placeholder/TimePlaceholderUtils.java index 15e3282d38..35cb018399 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/placeholder/TimePlaceholderUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/placeholder/TimePlaceholderUtils.java @@ -35,12 +35,12 @@ public class TimePlaceholderUtils { /** * Prefix of the position to be replaced */ - public static final String placeholderPrefix = "$["; + public static final String PLACEHOLDER_PREFIX = "$["; /** * The suffix of the position to be replaced */ - public static final String placeholderSuffix = "]"; + public static final String PLACEHOLDER_SUFFIX = "]"; /** * Replaces all placeholders of format {@code ${name}} with the value returned @@ -66,7 +66,7 @@ public class TimePlaceholderUtils { * be ignored ({@code true}) or cause an exception ({@code false}) */ private static PropertyPlaceholderHelper getPropertyPlaceholderHelper(boolean ignoreUnresolvablePlaceholders) { - return new PropertyPlaceholderHelper(placeholderPrefix, placeholderSuffix, null, ignoreUnresolvablePlaceholders); + return new PropertyPlaceholderHelper(PLACEHOLDER_PREFIX, PLACEHOLDER_SUFFIX, null, ignoreUnresolvablePlaceholders); } /** @@ -503,7 +503,7 @@ public class TimePlaceholderUtils { * @return calculate need minutes */ public static Integer calcMinutes(String minuteExpression) { - int index = minuteExpression.indexOf("/"); + int index = minuteExpression.indexOf('/'); String calcExpression; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessEnvironmentForWin32.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessEnvironmentForWin32.java index 3dbe7cb50f..39fddfbad9 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessEnvironmentForWin32.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessEnvironmentForWin32.java @@ -46,22 +46,23 @@ final class ProcessEnvironmentForWin32 extends HashMap { return (String) o; } + @Override public String put(String key, String value) { return super.put(validateName(key), validateValue(value)); } - + @Override public String get(Object key) { return super.get(nonNullString(key)); } - + @Override public boolean containsKey(Object key) { return super.containsKey(nonNullString(key)); } - + @Override public boolean containsValue(Object value) { return super.containsValue(nonNullString(value)); } - + @Override public String remove(Object key) { return super.remove(nonNullString(key)); } @@ -92,6 +93,7 @@ final class ProcessEnvironmentForWin32 extends HashMap { public Entry next() { return new CheckedEntry(i.next()); } + @Override public void remove() { i.remove();} }; } @@ -110,10 +112,14 @@ final class ProcessEnvironmentForWin32 extends HashMap { private final Collection c; public CheckedValues(Collection c) {this.c = c;} public int size() {return c.size();} + @Override public boolean isEmpty() {return c.isEmpty();} + @Override public void clear() { c.clear();} public Iterator iterator() {return c.iterator();} + @Override public boolean contains(Object o) {return c.contains(nonNullString(o));} + @Override public boolean remove(Object o) {return c.remove(nonNullString(o));} } @@ -127,15 +133,15 @@ final class ProcessEnvironmentForWin32 extends HashMap { public boolean contains(Object o) {return s.contains(nonNullString(o));} public boolean remove(Object o) {return s.remove(nonNullString(o));} } - + @Override public Set keySet() { return new CheckedKeySet(super.keySet()); } - + @Override public Collection values() { return new CheckedValues(super.values()); } - + @Override public Set> entrySet() { return new CheckedEntrySet(super.entrySet()); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32.java index 9f2716a096..4f6d719ef3 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32.java @@ -32,6 +32,7 @@ import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; +import static com.sun.jna.platform.win32.WinBase.INVALID_HANDLE_VALUE; import static com.sun.jna.platform.win32.WinBase.STILL_ACTIVE; import static java.util.Objects.requireNonNull; @@ -112,7 +113,7 @@ public class ProcessImplForWin32 extends Process { // System-dependent portion of ProcessBuilderForWindows.start() static Process start(String username, String password, - String cmdarray[], + String[] cmdarray, java.util.Map environment, String dir, ProcessBuilderForWin32.Redirect[] redirects, @@ -177,10 +178,10 @@ public class ProcessImplForWin32 extends Process { private static class LazyPattern { // Escape-support version: - // "(\")((?:\\\\\\1|.)+?)\\1|([^\\s\"]+)"; + // "(\")((?:\\\\\\1|.)+?)\\1|([^\\s\"]+)" private static final Pattern PATTERN = Pattern.compile("[^\\s\"]+|\"[^\"]*\""); - }; + } /* Parses the command string parameter into the executable name and * program arguments. @@ -203,7 +204,7 @@ public class ProcessImplForWin32 extends Process { private static final int VERIFICATION_LEGACY = 3; // See Command shell overview for documentation of special characters. // https://docs.microsoft.com/en-us/previous-versions/windows/it-pro/windows-xp/bb490954(v=technet.10) - private static final char ESCAPE_VERIFICATION[][] = { + private static final char[][] ESCAPE_VERIFICATION = { // We guarantee the only command file execution for implicit [cmd.exe] run. // http://technet.microsoft.com/en-us/library/bb490954.aspx {' ', '\t', '<', '>', '&', '|', '^'}, @@ -214,7 +215,7 @@ public class ProcessImplForWin32 extends Process { private static String createCommandLine(int verificationType, final String executablePath, - final String cmd[]) + final String[] cmd) { StringBuilder cmdbuf = new StringBuilder(80); @@ -309,7 +310,7 @@ public class ProcessImplForWin32 extends Process { } if (!argIsQuoted) { - char testEscape[] = ESCAPE_VERIFICATION[verificationType]; + char[] testEscape = ESCAPE_VERIFICATION[verificationType]; for (int i = 0; i < testEscape.length; ++i) { if (arg.indexOf(testEscape[i]) >= 0) { return true; @@ -390,14 +391,14 @@ public class ProcessImplForWin32 extends Process { private static final char BACKSLASH = '\\'; private WinNT.HANDLE handle; - private OutputStream stdin_stream; - private InputStream stdout_stream; - private InputStream stderr_stream; + private OutputStream stdinStream; + private InputStream stdoutStream; + private InputStream stderrStream; private ProcessImplForWin32( String username, String password, - String cmd[], + String[] cmd, final String envblock, final String path, final long[] stdHandles, @@ -472,44 +473,44 @@ public class ProcessImplForWin32 extends Process { new PrivilegedAction() { public Void run() { if (stdHandles[0] == -1L) - stdin_stream = ProcessBuilderForWin32.NullOutputStream.INSTANCE; + stdinStream = ProcessBuilderForWin32.NullOutputStream.INSTANCE; else { - FileDescriptor stdin_fd = new FileDescriptor(); - setHandle(stdin_fd, stdHandles[0]); - stdin_stream = new BufferedOutputStream( - new FileOutputStream(stdin_fd)); + FileDescriptor stdinFd = new FileDescriptor(); + setHandle(stdinFd, stdHandles[0]); + stdinStream = new BufferedOutputStream( + new FileOutputStream(stdinFd)); } if (stdHandles[1] == -1L) - stdout_stream = ProcessBuilderForWin32.NullInputStream.INSTANCE; + stdoutStream = ProcessBuilderForWin32.NullInputStream.INSTANCE; else { - FileDescriptor stdout_fd = new FileDescriptor(); - setHandle(stdout_fd, stdHandles[1]); - stdout_stream = new BufferedInputStream( - new FileInputStream(stdout_fd)); + FileDescriptor stdoutFd = new FileDescriptor(); + setHandle(stdoutFd, stdHandles[1]); + stdoutStream = new BufferedInputStream( + new FileInputStream(stdoutFd)); } if (stdHandles[2] == -1L) - stderr_stream = ProcessBuilderForWin32.NullInputStream.INSTANCE; + stderrStream = ProcessBuilderForWin32.NullInputStream.INSTANCE; else { - FileDescriptor stderr_fd = new FileDescriptor(); - setHandle(stderr_fd, stdHandles[2]); - stderr_stream = new FileInputStream(stderr_fd); + FileDescriptor stderrFd = new FileDescriptor(); + setHandle(stderrFd, stdHandles[2]); + stderrStream = new FileInputStream(stderrFd); } return null; }}); } public OutputStream getOutputStream() { - return stdin_stream; + return stdinStream; } public InputStream getInputStream() { - return stdout_stream; + return stdoutStream; } public InputStream getErrorStream() { - return stderr_stream; + return stderrStream; } protected void finalize() { @@ -557,11 +558,12 @@ public class ProcessImplForWin32 extends Process { public void destroy() { terminateProcess(handle); } + @Override public Process destroyForcibly() { destroy(); return this; } - + @Override public boolean isAlive() { return isProcessAlive(handle); } @@ -582,7 +584,7 @@ public class ProcessImplForWin32 extends Process { pjhandles.setValue(thisProcessEnd); } } - Kernel32.INSTANCE.SetHandleInformation(phStd.getValue(), Kernel32.HANDLE_FLAG_INHERIT, Kernel32.HANDLE_FLAG_INHERIT); + Kernel32.INSTANCE.SetHandleInformation(phStd.getValue(), WinBase.HANDLE_FLAG_INHERIT, WinBase.HANDLE_FLAG_INHERIT); return true; } @@ -596,17 +598,17 @@ public class ProcessImplForWin32 extends Process { private static void prepareIOEHandleState(WinNT.HANDLE[] stdIOE, Boolean[] inherit) { for(int i = 0; i < HANDLE_STORAGE_SIZE; ++i) { WinNT.HANDLE hstd = stdIOE[i]; - if (!Kernel32.INVALID_HANDLE_VALUE.equals(hstd)) { + if (!WinBase.INVALID_HANDLE_VALUE.equals(hstd)) { inherit[i] = Boolean.TRUE; - Kernel32.INSTANCE.SetHandleInformation(hstd, Kernel32.HANDLE_FLAG_INHERIT, 0); + Kernel32.INSTANCE.SetHandleInformation(hstd, WinBase.HANDLE_FLAG_INHERIT, 0); } } } private static void restoreIOEHandleState(WinNT.HANDLE[] stdIOE, Boolean[] inherit) { for (int i = HANDLE_STORAGE_SIZE - 1; i >= 0; --i) { - if (!Kernel32.INVALID_HANDLE_VALUE.equals(stdIOE[i])) { - Kernel32.INSTANCE.SetHandleInformation(stdIOE[i], Kernel32.HANDLE_FLAG_INHERIT, inherit[i] ? Kernel32.HANDLE_FLAG_INHERIT : 0); + if (!WinBase.INVALID_HANDLE_VALUE.equals(stdIOE[i])) { + Kernel32.INSTANCE.SetHandleInformation(stdIOE[i], WinBase.HANDLE_FLAG_INHERIT, Boolean.TRUE.equals(inherit[i]) ? WinBase.HANDLE_FLAG_INHERIT : 0); } } } @@ -621,12 +623,12 @@ public class ProcessImplForWin32 extends Process { WinNT.HANDLE ret = new WinNT.HANDLE(Pointer.createConstant(0)); WinNT.HANDLE[] stdIOE = new WinNT.HANDLE[] { - Kernel32.INVALID_HANDLE_VALUE, Kernel32.INVALID_HANDLE_VALUE, Kernel32.INVALID_HANDLE_VALUE, + WinBase.INVALID_HANDLE_VALUE, WinBase.INVALID_HANDLE_VALUE, WinBase.INVALID_HANDLE_VALUE, stdHandles[0].getValue(), stdHandles[1].getValue(), stdHandles[2].getValue() }; - stdIOE[0] = Kernel32.INSTANCE.GetStdHandle(Kernel32.STD_INPUT_HANDLE); - stdIOE[1] = Kernel32.INSTANCE.GetStdHandle(Kernel32.STD_OUTPUT_HANDLE); - stdIOE[2] = Kernel32.INSTANCE.GetStdHandle(Kernel32.STD_ERROR_HANDLE); + stdIOE[0] = Kernel32.INSTANCE.GetStdHandle(Wincon.STD_INPUT_HANDLE); + stdIOE[1] = Kernel32.INSTANCE.GetStdHandle(Wincon.STD_OUTPUT_HANDLE); + stdIOE[2] = Kernel32.INSTANCE.GetStdHandle(Wincon.STD_ERROR_HANDLE); Boolean[] inherit = new Boolean[] { Boolean.FALSE, Boolean.FALSE, Boolean.FALSE, @@ -638,17 +640,17 @@ public class ProcessImplForWin32 extends Process { // input WinNT.HANDLEByReference hStdInput = new WinNT.HANDLEByReference(); WinNT.HANDLEByReference[] pipeIn = new WinNT.HANDLEByReference[] { - new WinNT.HANDLEByReference(Kernel32.INVALID_HANDLE_VALUE), new WinNT.HANDLEByReference(Kernel32.INVALID_HANDLE_VALUE) }; + new WinNT.HANDLEByReference(WinBase.INVALID_HANDLE_VALUE), new WinNT.HANDLEByReference(WinBase.INVALID_HANDLE_VALUE) }; // output WinNT.HANDLEByReference hStdOutput = new WinNT.HANDLEByReference(); WinNT.HANDLEByReference[] pipeOut = new WinNT.HANDLEByReference[] { - new WinNT.HANDLEByReference(Kernel32.INVALID_HANDLE_VALUE), new WinNT.HANDLEByReference(Kernel32.INVALID_HANDLE_VALUE) }; + new WinNT.HANDLEByReference(WinBase.INVALID_HANDLE_VALUE), new WinNT.HANDLEByReference(WinBase.INVALID_HANDLE_VALUE) }; // error WinNT.HANDLEByReference hStdError = new WinNT.HANDLEByReference(); WinNT.HANDLEByReference[] pipeError = new WinNT.HANDLEByReference[] { - new WinNT.HANDLEByReference(Kernel32.INVALID_HANDLE_VALUE), new WinNT.HANDLEByReference(Kernel32.INVALID_HANDLE_VALUE) }; + new WinNT.HANDLEByReference(WinBase.INVALID_HANDLE_VALUE), new WinNT.HANDLEByReference(WinBase.INVALID_HANDLE_VALUE) }; boolean success; if (initHolder(stdHandles[0], pipeIn, OFFSET_READ, hStdInput)) { @@ -668,8 +670,8 @@ public class ProcessImplForWin32 extends Process { if (success) { WTypes.LPSTR lpEnvironment = envblock == null ? new WTypes.LPSTR() : new WTypes.LPSTR(envblock); - Kernel32.PROCESS_INFORMATION pi = new WinBase.PROCESS_INFORMATION(); - si.dwFlags = Kernel32.STARTF_USESTDHANDLES; + WinBase.PROCESS_INFORMATION pi = new WinBase.PROCESS_INFORMATION(); + si.dwFlags = WinBase.STARTF_USESTDHANDLES; if (!Advapi32.INSTANCE.CreateProcessWithLogonW( username , null @@ -677,7 +679,7 @@ public class ProcessImplForWin32 extends Process { , Advapi32.LOGON_WITH_PROFILE , null , cmd - , Kernel32.CREATE_NO_WINDOW + , WinBase.CREATE_NO_WINDOW , lpEnvironment.getPointer() , path , si @@ -709,13 +711,11 @@ public class ProcessImplForWin32 extends Process { for (int i = 0; i < stdHandles.length; i++) { handles[i] = new WinNT.HANDLEByReference(new WinNT.HANDLE(Pointer.createConstant(stdHandles[i]))); } - - if (cmd != null) { - if (username != null && password != null) { - ret = processCreate(username, password, cmd, envblock, path, handles, redirectErrorStream); - } + + if (cmd != null && username != null && password != null) { + ret = processCreate(username, password, cmd, envblock, path, handles, redirectErrorStream); } - + for (int i = 0; i < stdHandles.length; i++) { stdHandles[i] = handles[i].getPointer().getLong(0); } @@ -742,7 +742,9 @@ public class ProcessImplForWin32 extends Process { } private static void closeHandle(WinNT.HANDLE handle) { - Kernel32Util.closeHandle(handle); + if (!handle.equals(INVALID_HANDLE_VALUE)) { + Kernel32Util.closeHandle(handle); + } } /** @@ -753,15 +755,15 @@ public class ProcessImplForWin32 extends Process { * @return the native HANDLE */ private static long openForAtomicAppend(String path) throws IOException { - int access = Kernel32.GENERIC_READ | Kernel32.GENERIC_WRITE; - int sharing = Kernel32.FILE_SHARE_READ | Kernel32.FILE_SHARE_WRITE; - int disposition = Kernel32.OPEN_ALWAYS; - int flagsAndAttributes = Kernel32.FILE_ATTRIBUTE_NORMAL; + int access = WinNT.GENERIC_READ | WinNT.GENERIC_WRITE; + int sharing = WinNT.FILE_SHARE_READ | WinNT.FILE_SHARE_WRITE; + int disposition = WinNT.OPEN_ALWAYS; + int flagsAndAttributes = WinNT.FILE_ATTRIBUTE_NORMAL; if (path == null || path.isEmpty()) { return -1; } else { WinNT.HANDLE handle = Kernel32.INSTANCE.CreateFile(path, access, sharing, null, disposition, flagsAndAttributes, null); - if (handle == Kernel32.INVALID_HANDLE_VALUE) { + if (handle == WinBase.INVALID_HANDLE_VALUE) { throw new Win32Exception(Kernel32.INSTANCE.GetLastError()); } return handle.getPointer().getLong(0); @@ -769,15 +771,15 @@ public class ProcessImplForWin32 extends Process { } private static void waitForInterruptibly(WinNT.HANDLE handle) { - int result = Kernel32.INSTANCE.WaitForMultipleObjects(1, new WinNT.HANDLE[]{handle}, false, Kernel32.INFINITE); - if (result == Kernel32.WAIT_FAILED) { + int result = Kernel32.INSTANCE.WaitForMultipleObjects(1, new WinNT.HANDLE[]{handle}, false, WinBase.INFINITE); + if (result == WinBase.WAIT_FAILED) { throw new Win32Exception(Kernel32.INSTANCE.GetLastError()); } } private static void waitForTimeoutInterruptibly(WinNT.HANDLE handle, long timeout) { int result = Kernel32.INSTANCE.WaitForMultipleObjects(1, new WinNT.HANDLE[]{handle}, false, (int) timeout); - if (result == Kernel32.WAIT_FAILED) { + if (result == WinBase.WAIT_FAILED) { throw new Win32Exception(Kernel32.INSTANCE.GetLastError()); } } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java index 2670eebc20..1815e48f84 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java @@ -67,7 +67,7 @@ public class OSUtilsTest { @Test public void cpuUsage() throws Exception { logger.info("cpuUsage : {}", OSUtils.cpuUsage()); - Thread.sleep(1000l); + Thread.sleep(1000L); logger.info("cpuUsage : {}", OSUtils.cpuUsage()); double cpuUsage = OSUtils.cpuUsage(); diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/FlinkParametersTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/FlinkParametersTest.java index 7ce00e875a..cd7b4f2200 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/FlinkParametersTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/FlinkParametersTest.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.common.task; import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.flink.FlinkParameters; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.junit.Assert; import org.junit.Test; @@ -28,8 +29,7 @@ public class FlinkParametersTest { @Test public void getResourceFilesList() { FlinkParameters flinkParameters = new FlinkParameters(); - Assert.assertNotNull(flinkParameters.getResourceFilesList()); - Assert.assertTrue(flinkParameters.getResourceFilesList().isEmpty()); + Assert.assertTrue(CollectionUtils.isEmpty(flinkParameters.getResourceFilesList())); ResourceInfo mainResource = new ResourceInfo(); mainResource.setRes("testFlinkMain-1.0.0-SNAPSHOT.jar"); @@ -41,15 +41,17 @@ public class FlinkParametersTest { resourceInfos.add(resourceInfo1); flinkParameters.setResourceList(resourceInfos); - Assert.assertNotNull(flinkParameters.getResourceFilesList()); - Assert.assertEquals(2, flinkParameters.getResourceFilesList().size()); + List resourceFilesList = flinkParameters.getResourceFilesList(); + Assert.assertNotNull(resourceFilesList); + Assert.assertEquals(2, resourceFilesList.size()); ResourceInfo resourceInfo2 = new ResourceInfo(); resourceInfo2.setRes("testFlinkParameters2.jar"); resourceInfos.add(resourceInfo2); flinkParameters.setResourceList(resourceInfos); - Assert.assertNotNull(flinkParameters.getResourceFilesList()); - Assert.assertEquals(3, flinkParameters.getResourceFilesList().size()); + resourceFilesList = flinkParameters.getResourceFilesList(); + Assert.assertNotNull(resourceFilesList); + Assert.assertEquals(3, resourceFilesList.size()); } } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringTest.java index 99a2cf05bc..b14be21e60 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringTest.java @@ -24,12 +24,6 @@ import java.util.List; public class StringTest { - - @Test - public void test1(){ - System.out.println(String.format("%s_%010d_%010d", String.valueOf(1), Long.valueOf(3), Integer.valueOf(4))); - } - @Test public void stringCompareTest(){ diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java index 2ba93d7607..d61a181564 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java @@ -99,13 +99,7 @@ public class AlertDao extends AbstractBaseDao { String content = String.format("[{'type':'%s','host':'%s','event':'server down','warning level':'serious'}]", serverType, host); alert.setTitle("Fault tolerance warning"); - alert.setShowType(ShowType.TABLE); - alert.setContent(content); - alert.setAlertType(AlertType.EMAIL); - alert.setAlertGroupId(alertgroupId); - alert.setCreateTime(new Date()); - alert.setUpdateTime(new Date()); - alertMapper.insert(alert); + saveTaskTimeoutAlert(alert, content, alertgroupId, null, null); } /** @@ -121,6 +115,11 @@ public class AlertDao extends AbstractBaseDao { String content = String.format("[{'id':'%d','name':'%s','event':'timeout','warnLevel':'middle'}]", processInstance.getId(), processInstance.getName()); alert.setTitle("Process Timeout Warn"); + saveTaskTimeoutAlert(alert, content, alertgroupId, receivers, receiversCc); + } + + private void saveTaskTimeoutAlert(Alert alert, String content, int alertgroupId, + String receivers, String receiversCc){ alert.setShowType(ShowType.TABLE); alert.setContent(content); alert.setAlertType(AlertType.EMAIL); @@ -150,19 +149,7 @@ public class AlertDao extends AbstractBaseDao { String content = String.format("[{'process instance id':'%d','task name':'%s','task id':'%d','task name':'%s'," + "'event':'timeout','warnLevel':'middle'}]", processInstanceId, processInstanceName, taskId, taskName); alert.setTitle("Task Timeout Warn"); - alert.setShowType(ShowType.TABLE); - alert.setContent(content); - alert.setAlertType(AlertType.EMAIL); - alert.setAlertGroupId(alertgroupId); - if (StringUtils.isNotEmpty(receivers)) { - alert.setReceivers(receivers); - } - if (StringUtils.isNotEmpty(receiversCc)) { - alert.setReceiversCc(receiversCc); - } - alert.setCreateTime(new Date()); - alert.setUpdateTime(new Date()); - alertMapper.insert(alert); + saveTaskTimeoutAlert(alert, content, alertgroupId, receivers, receiversCc); } /** @@ -182,5 +169,11 @@ public class AlertDao extends AbstractBaseDao { return userAlertGroupMapper.listUserByAlertgroupId(alertgroupId); } - + /** + * for test + * @return + */ + public AlertMapper getAlertMapper() { + return alertMapper; + } } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/MonitorDBDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/MonitorDBDao.java index 5ea5966238..53366777f7 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/MonitorDBDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/MonitorDBDao.java @@ -18,10 +18,10 @@ package org.apache.dolphinscheduler.dao; import com.alibaba.druid.pool.DruidDataSource; import java.sql.Connection; -import java.sql.SQLException; import java.util.ArrayList; import java.util.List; import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.common.utils.ConnectionUtils; import org.apache.dolphinscheduler.dao.entity.MonitorRecord; import org.apache.dolphinscheduler.dao.utils.MysqlPerformance; import org.apache.dolphinscheduler.dao.utils.PostgrePerformance; @@ -63,13 +63,7 @@ public class MonitorDBDao { }catch (Exception e) { logger.error("SQLException: {}", e.getMessage(), e); }finally { - try { - if (conn != null) { - conn.close(); - } - } catch (SQLException e) { - logger.error("SQLException ", e); - } + ConnectionUtils.releaseResource(conn); } return monitorRecord; } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/TaskRecordDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/TaskRecordDao.java index f89002ebdc..7cf9159e98 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/TaskRecordDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/TaskRecordDao.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.dao; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.TaskRecordStatus; import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import org.apache.dolphinscheduler.common.utils.ConnectionUtils; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.dao.entity.TaskRecord; @@ -84,9 +85,9 @@ public class TaskRecordDao { Class.forName(driver); conn = DriverManager.getConnection(url, username, password); } catch (ClassNotFoundException e) { - logger.error("Exception ", e); + logger.error("Class not found Exception ", e); } catch (SQLException e) { - logger.error("Exception ", e); + logger.error("SQL Exception ", e); } return conn; } @@ -163,14 +164,14 @@ public class TaskRecordDao { sql += getWhereString(filterMap); pstmt = conn.prepareStatement(sql); rs = pstmt.executeQuery(); - while (rs.next()) { + while (rs.next()){ count = rs.getInt("count"); break; } } catch (SQLException e) { logger.error("Exception ", e); - } finally { - closeResource(rs, pstmt, conn); + }finally { + ConnectionUtils.releaseResource(rs, pstmt, conn); } return count; } @@ -254,8 +255,8 @@ public class TaskRecordDao { } } catch (SQLException e) { logger.error("Exception ", e); - } finally { - closeResource(rs, pstmt, conn); + }finally { + ConnectionUtils.releaseResource(rs, pstmt, conn); } return recordList; } @@ -292,28 +293,4 @@ public class TaskRecordDao { } } - - private static void closeResource(ResultSet rs, PreparedStatement pstmt, Connection conn) { - if (rs != null) { - try { - rs.close(); - } catch (SQLException e) { - logger.error("Exception ", e); - } - } - if (pstmt != null) { - try { - pstmt.close(); - } catch (SQLException e) { - logger.error("Exception ", e); - } - } - if (conn != null) { - try { - conn.close(); - } catch (SQLException e) { - logger.error("Exception ", e); - } - } - } } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java index a46e5aabcc..6f95026759 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java @@ -16,10 +16,21 @@ */ package org.apache.dolphinscheduler.dao.datasource; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * data source base class */ public abstract class BaseDataSource { + + private static final Logger logger = LoggerFactory.getLogger(BaseDataSource.class); + /** * user name */ @@ -57,17 +68,105 @@ public abstract class BaseDataSource { public void setPrincipal(String principal) { this.principal = principal; } + /** - * test whether the data source can be connected successfully - * @throws Exception + * @return driver class + */ + public abstract String driverClassSelector(); + + /** + * @return db type */ - public abstract void isConnectable() throws Exception; + public abstract DbType dbTypeSelector(); /** * gets the JDBC url for the data source connection - * @return */ - public abstract String getJdbcUrl(); + public String getJdbcUrl() { + StringBuilder jdbcUrl = new StringBuilder(getAddress()); + + appendDatabase(jdbcUrl); + appendPrincipal(jdbcUrl); + appendOther(jdbcUrl); + + return jdbcUrl.toString(); + } + + /** + * append database + * @param jdbcUrl jdbc url + */ + private void appendDatabase(StringBuilder jdbcUrl) { + if (dbTypeSelector() == DbType.SQLSERVER) { + jdbcUrl.append(";databaseName=").append(getDatabase()); + } else { + if (getAddress().lastIndexOf('/') != (jdbcUrl.length() - 1)) { + jdbcUrl.append("/"); + } + jdbcUrl.append(getDatabase()); + } + } + + /** + * append principal + * @param jdbcUrl jdbc url + */ + private void appendPrincipal(StringBuilder jdbcUrl) { + boolean tag = dbTypeSelector() == DbType.HIVE || dbTypeSelector() == DbType.SPARK; + if (tag && StringUtils.isNotEmpty(getPrincipal())) { + jdbcUrl.append(";principal=").append(getPrincipal()); + } + } + + /** + * append other + * @param jdbcUrl jdbc url + */ + private void appendOther(StringBuilder jdbcUrl) { + if (StringUtils.isNotEmpty(getOther())) { + String separator = ""; + switch (dbTypeSelector()) { + case CLICKHOUSE: + case MYSQL: + case ORACLE: + case POSTGRESQL: + separator = "?"; + break; + case DB2: + separator = ":"; + break; + case HIVE: + case SPARK: + case SQLSERVER: + separator = ";"; + break; + default: + logger.error("Db type mismatch!"); + } + jdbcUrl.append(separator).append(getOther()); + } + } + + /** + * test whether the data source can be connected successfully + */ + public void isConnectable() { + Connection con = null; + try { + Class.forName(driverClassSelector()); + con = DriverManager.getConnection(getJdbcUrl(), getUser(), getPassword()); + } catch (ClassNotFoundException | SQLException e) { + logger.error("Get connection error: {}", e.getMessage()); + } finally { + if (con != null) { + try { + con.close(); + } catch (SQLException e) { + logger.error(e.getMessage(), e); + } + } + } + } public String getUser() { return user; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/ClickHouseDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/ClickHouseDataSource.java index e159f81d2e..ba34ff82d6 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/ClickHouseDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/ClickHouseDataSource.java @@ -17,59 +17,26 @@ package org.apache.dolphinscheduler.dao.datasource; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.utils.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; +import org.apache.dolphinscheduler.common.enums.DbType; /** * data source of ClickHouse */ public class ClickHouseDataSource extends BaseDataSource { - private static final Logger logger = LoggerFactory.getLogger(ClickHouseDataSource.class); /** - * gets the JDBC url for the data source connection - * @return + * @return driver class */ @Override - public String getJdbcUrl() { - String jdbcUrl = getAddress(); - if (jdbcUrl.lastIndexOf('/') != (jdbcUrl.length() - 1)) { - jdbcUrl += "/"; - } - - jdbcUrl += getDatabase(); - - if (StringUtils.isNotEmpty(getOther())) { - jdbcUrl += "?" + getOther(); - } - - return jdbcUrl; + public String driverClassSelector() { + return Constants.COM_CLICKHOUSE_JDBC_DRIVER; } /** - * test whether the data source can be connected successfully - * @throws Exception + * @return db type */ @Override - public void isConnectable() throws Exception { - Connection con = null; - try { - Class.forName(Constants.COM_CLICKHOUSE_JDBC_DRIVER); - con = DriverManager.getConnection(getJdbcUrl(), getUser(), getPassword()); - } finally { - if (con != null) { - try { - con.close(); - } catch (SQLException e) { - logger.error("ClickHouse datasource try conn close conn error", e); - } - } - } - + public DbType dbTypeSelector() { + return DbType.CLICKHOUSE; } } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/ConnectionFactory.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/ConnectionFactory.java index 199fed0ce6..51c56fd29e 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/ConnectionFactory.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/ConnectionFactory.java @@ -51,6 +51,7 @@ public class ConnectionFactory extends SpringConnectionFactory { private ConnectionFactory() { try { + dataSource = buildDataSource(); sqlSessionFactory = getSqlSessionFactory(); sqlSessionTemplate = getSqlSessionTemplate(); } catch (Exception e) { @@ -69,12 +70,18 @@ public class ConnectionFactory extends SpringConnectionFactory { */ private SqlSessionTemplate sqlSessionTemplate; + private DataSource dataSource; + + public DataSource getDataSource() { + return dataSource; + } + /** * get the data source * * @return druid dataSource */ - public DruidDataSource getDataSource() { + private DataSource buildDataSource() { DruidDataSource druidDataSource = new DruidDataSource(); @@ -112,10 +119,9 @@ public class ConnectionFactory extends SpringConnectionFactory { * @throws Exception sqlSessionFactory exception */ private SqlSessionFactory getSqlSessionFactory() throws Exception { - DataSource dataSource = getDataSource(); TransactionFactory transactionFactory = new JdbcTransactionFactory(); - Environment environment = new Environment("development", transactionFactory, dataSource); + Environment environment = new Environment("development", transactionFactory, getDataSource()); MybatisConfiguration configuration = new MybatisConfiguration(); configuration.setEnvironment(environment); @@ -125,7 +131,7 @@ public class ConnectionFactory extends SpringConnectionFactory { MybatisSqlSessionFactoryBean sqlSessionFactoryBean = new MybatisSqlSessionFactoryBean(); sqlSessionFactoryBean.setConfiguration(configuration); - sqlSessionFactoryBean.setDataSource(dataSource); + sqlSessionFactoryBean.setDataSource(getDataSource()); sqlSessionFactoryBean.setTypeEnumsPackage("org.apache.dolphinscheduler.*.enums"); sqlSessionFactory = sqlSessionFactoryBean.getObject(); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/DB2ServerDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/DB2ServerDataSource.java index 3c2366b5b0..4b524b8b81 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/DB2ServerDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/DB2ServerDataSource.java @@ -17,58 +17,26 @@ package org.apache.dolphinscheduler.dao.datasource; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.utils.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; +import org.apache.dolphinscheduler.common.enums.DbType; /** * data source of DB2 Server */ public class DB2ServerDataSource extends BaseDataSource { - private static final Logger logger = LoggerFactory.getLogger(DB2ServerDataSource.class); /** - * gets the JDBC url for the data source connection - * @return + * @return driver class */ @Override - public String getJdbcUrl() { - String jdbcUrl = getAddress(); - if (jdbcUrl.lastIndexOf("/") != (jdbcUrl.length() - 1)) { - jdbcUrl += "/"; - } - - jdbcUrl += getDatabase(); - - if (StringUtils.isNotEmpty(getOther())) { - jdbcUrl += ":" + getOther(); - } - return jdbcUrl; + public String driverClassSelector() { + return Constants.COM_DB2_JDBC_DRIVER; } /** - * test whether the data source can be connected successfully - * @throws Exception + * @return db type */ @Override - public void isConnectable() throws Exception { - Connection con = null; - try { - Class.forName(Constants.COM_DB2_JDBC_DRIVER); - con = DriverManager.getConnection(getJdbcUrl(), getUser(), getPassword()); - } finally { - if (con != null) { - try { - con.close(); - } catch (SQLException e) { - logger.error("DB2 Server datasource try conn close conn error", e); - } - } - } - + public DbType dbTypeSelector() { + return DbType.DB2; } } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java index 4dfb8817a8..2616c5add6 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java @@ -17,63 +17,26 @@ package org.apache.dolphinscheduler.dao.datasource; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.utils.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; +import org.apache.dolphinscheduler.common.enums.DbType; /** * data source of hive */ public class HiveDataSource extends BaseDataSource { - private static final Logger logger = LoggerFactory.getLogger(HiveDataSource.class); - /** - * gets the JDBC url for the data source connection - * @return + * @return driver class */ @Override - public String getJdbcUrl() { - String jdbcUrl = getAddress(); - if (jdbcUrl.lastIndexOf('/') != (jdbcUrl.length() - 1)) { - jdbcUrl += "/"; - } - - jdbcUrl += getDatabase(); - - if (StringUtils.isNotEmpty(getPrincipal())){ - jdbcUrl += ";principal=" + getPrincipal(); - } - - if (StringUtils.isNotEmpty(getOther())) { - jdbcUrl += ";" + getOther(); - } - - return jdbcUrl; + public String driverClassSelector() { + return Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER; } /** - * test whether the data source can be connected successfully - * @throws Exception + * @return db type */ @Override - public void isConnectable() throws Exception { - Connection con = null; - try { - Class.forName(Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER); - con = DriverManager.getConnection(getJdbcUrl(), getUser(), ""); - } finally { - if (con != null) { - try { - con.close(); - } catch (SQLException e) { - logger.error("hive datasource try conn close conn error", e); - } - } - } + public DbType dbTypeSelector() { + return DbType.HIVE; } } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSource.java index 969c17b54d..459cf946f1 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSource.java @@ -17,57 +17,27 @@ package org.apache.dolphinscheduler.dao.datasource; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.utils.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; +import org.apache.dolphinscheduler.common.enums.DbType; /** * data source of mySQL */ public class MySQLDataSource extends BaseDataSource { - private static final Logger logger = LoggerFactory.getLogger(MySQLDataSource.class); - /** - * gets the JDBC url for the data source connection - * @return + * @return driver class */ @Override - public String getJdbcUrl() { - String address = getAddress(); - if (address.lastIndexOf('/') != (address.length() - 1)) { - address += "/"; - } - String jdbcUrl = address + getDatabase(); - if (StringUtils.isNotEmpty(getOther())) { - jdbcUrl += "?" + getOther(); - } - return jdbcUrl; + public String driverClassSelector() { + return Constants.COM_MYSQL_JDBC_DRIVER; } /** - * test whether the data source can be connected successfully - * @throws Exception + * @return db type */ @Override - public void isConnectable() throws Exception { - Connection con = null; - try { - Class.forName(Constants.COM_MYSQL_JDBC_DRIVER); - con = DriverManager.getConnection(getJdbcUrl(), getUser(), getPassword()); - } finally { - if (con != null) { - try { - con.close(); - } catch (SQLException e) { - logger.error("Mysql datasource try conn close conn error", e); - } - } - } + public DbType dbTypeSelector() { + return DbType.MYSQL; } } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/OracleDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/OracleDataSource.java index cefaf879b5..879219c386 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/OracleDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/OracleDataSource.java @@ -17,59 +17,41 @@ package org.apache.dolphinscheduler.dao.datasource; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.apache.dolphinscheduler.common.enums.DbConnectType; +import org.apache.dolphinscheduler.common.enums.DbType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; - /** * data source of Oracle */ public class OracleDataSource extends BaseDataSource { - private static final Logger logger = LoggerFactory.getLogger(OracleDataSource.class); - /** - * gets the JDBC url for the data source connection - * @return - */ - @Override - public String getJdbcUrl() { - String jdbcUrl = getAddress(); - if (jdbcUrl.lastIndexOf("/") != (jdbcUrl.length() - 1)) { - jdbcUrl += "/"; - } + private static final Logger logger = LoggerFactory.getLogger(OracleDataSource.class); - jdbcUrl += getDatabase(); + private DbConnectType type; - if (StringUtils.isNotEmpty(getOther())) { - jdbcUrl += "?" + getOther(); - } + public DbConnectType getType() { + return type; + } - return jdbcUrl; + public void setType(DbConnectType type) { + this.type = type; } /** - * test whether the data source can be connected successfully - * @throws Exception + * @return driver class */ @Override - public void isConnectable() throws Exception { - Connection con = null; - try { - Class.forName(Constants.COM_ORACLE_JDBC_DRIVER); - con = DriverManager.getConnection(getJdbcUrl(), getUser(), getPassword()); - } finally { - if (con != null) { - try { - con.close(); - } catch (SQLException e) { - logger.error("Oracle datasource try conn close conn error", e); - } - } - } + public String driverClassSelector() { + return Constants.COM_ORACLE_JDBC_DRIVER; + } + /** + * @return db type + */ + @Override + public DbType dbTypeSelector() { + return DbType.ORACLE; } } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/PostgreDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/PostgreDataSource.java index c01dfa4dc0..03a2b5c7b6 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/PostgreDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/PostgreDataSource.java @@ -17,61 +17,26 @@ package org.apache.dolphinscheduler.dao.datasource; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.utils.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; +import org.apache.dolphinscheduler.common.enums.DbType; /** * data source of postgreSQL */ public class PostgreDataSource extends BaseDataSource { - private static final Logger logger = LoggerFactory.getLogger(PostgreDataSource.class); - - /** - * gets the JDBC url for the data source connection - * @return + * @return driver class */ @Override - public String getJdbcUrl() { - String jdbcUrl = getAddress(); - if (jdbcUrl.lastIndexOf('/') != (jdbcUrl.length() - 1)) { - jdbcUrl += "/"; - } - - jdbcUrl += getDatabase(); - - if (StringUtils.isNotEmpty(getOther())) { - jdbcUrl += "?" + getOther(); - } - - return jdbcUrl; + public String driverClassSelector() { + return Constants.ORG_POSTGRESQL_DRIVER; } /** - * test whether the data source can be connected successfully - * @throws Exception + * @return db type */ @Override - public void isConnectable() throws Exception { - Connection con = null; - try { - Class.forName(Constants.ORG_POSTGRESQL_DRIVER); - con = DriverManager.getConnection(getJdbcUrl(), getUser(), getPassword()); - } finally { - if (con != null) { - try { - con.close(); - } catch (SQLException e) { - logger.error("Postgre datasource try conn close conn error", e); - } - } - } - + public DbType dbTypeSelector() { + return DbType.POSTGRESQL; } } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SQLServerDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SQLServerDataSource.java index 07770c06a7..c692de8dcc 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SQLServerDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SQLServerDataSource.java @@ -17,55 +17,26 @@ package org.apache.dolphinscheduler.dao.datasource; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.utils.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; +import org.apache.dolphinscheduler.common.enums.DbType; /** * data source of SQL Server */ public class SQLServerDataSource extends BaseDataSource { - private static final Logger logger = LoggerFactory.getLogger(SQLServerDataSource.class); - - /** - * gets the JDBC url for the data source connection - * @return - */ - @Override - public String getJdbcUrl() { - String jdbcUrl = getAddress(); - jdbcUrl += ";databaseName=" + getDatabase(); - - if (StringUtils.isNotEmpty(getOther())) { - jdbcUrl += ";" + getOther(); - } - - return jdbcUrl; - } - - /** - * test whether the data source can be connected successfully - * @throws Exception - */ - @Override - public void isConnectable() throws Exception { - Connection con = null; - try { - Class.forName(Constants.COM_SQLSERVER_JDBC_DRIVER); - con = DriverManager.getConnection(getJdbcUrl(), getUser(), getPassword()); - } finally { - if (con != null) { - try { - con.close(); - } catch (SQLException e) { - logger.error("SQL Server datasource try conn close conn error", e); - } - } - } - } + /** + * @return driver class + */ + @Override + public String driverClassSelector() { + return Constants.COM_SQLSERVER_JDBC_DRIVER; + } + + /** + * @return db type + */ + @Override + public DbType dbTypeSelector() { + return DbType.SQLSERVER; + } } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SparkDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SparkDataSource.java index 81a5ac6f04..d8794651d8 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SparkDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SparkDataSource.java @@ -17,64 +17,26 @@ package org.apache.dolphinscheduler.dao.datasource; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.utils.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; +import org.apache.dolphinscheduler.common.enums.DbType; /** * data source of spark */ public class SparkDataSource extends BaseDataSource { - private static final Logger logger = LoggerFactory.getLogger(SparkDataSource.class); - /** - * gets the JDBC url for the data source connection - * @return + * @return driver class */ @Override - public String getJdbcUrl() { - String jdbcUrl = getAddress(); - if (jdbcUrl.lastIndexOf("/") != (jdbcUrl.length() - 1)) { - jdbcUrl += "/"; - } - - jdbcUrl += getDatabase(); - - if (StringUtils.isNotEmpty(getPrincipal())){ - jdbcUrl += ";principal=" + getPrincipal(); - } - - if (StringUtils.isNotEmpty(getOther())) { - jdbcUrl += ";" + getOther(); - } - - return jdbcUrl; + public String driverClassSelector() { + return Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER; } /** - * test whether the data source can be connected successfully - * @throws Exception + * @return db type */ @Override - public void isConnectable() throws Exception { - Connection con = null; - try { - Class.forName(Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER); - con = DriverManager.getConnection(getJdbcUrl(), getUser(), ""); - } finally { - if (con != null) { - try { - con.close(); - } catch (SQLException e) { - logger.error("Spark datasource try conn close conn error", e); - } - } - } - + public DbType dbTypeSelector() { + return DbType.SPARK; } } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java index f59d11f3fe..e29de897ef 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java @@ -163,6 +163,11 @@ public class ProcessDefinition { */ private String modifyBy; + /** + * resource ids + */ + private String resourceIds; + public String getName() { return name; @@ -334,6 +339,14 @@ public class ProcessDefinition { this.scheduleReleaseState = scheduleReleaseState; } + public String getResourceIds() { + return resourceIds; + } + + public void setResourceIds(String resourceIds) { + this.resourceIds = resourceIds; + } + public int getTimeout() { return timeout; } @@ -393,6 +406,8 @@ public class ProcessDefinition { ", timeout=" + timeout + ", tenantId=" + tenantId + ", modifyBy='" + modifyBy + '\'' + + ", resourceIds='" + resourceIds + '\'' + '}'; } + } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java index fd9653a36c..097bf11238 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java @@ -366,7 +366,7 @@ public class ProcessInstance { } - public boolean IsProcessInstanceStop(){ + public boolean isProcessInstanceStop(){ return this.state.typeIsFinished(); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Resource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Resource.java index 934be4ba3d..16d94914fd 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Resource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Resource.java @@ -32,11 +32,26 @@ public class Resource { @TableId(value="id", type=IdType.AUTO) private int id; + /** + * parent id + */ + private int pid; + /** * resource alias */ private String alias; + /** + * full name + */ + private String fullName; + + /** + * is directory + */ + private boolean isDirectory=false; + /** * description */ @@ -89,7 +104,15 @@ public class Resource { this.updateTime = updateTime; } - public Resource(String alias, String fileName, String description, int userId, ResourceType type, long size, Date createTime, Date updateTime) { + public Resource(int id, int pid, String alias, String fullName, boolean isDirectory) { + this.id = id; + this.pid = pid; + this.alias = alias; + this.fullName = fullName; + this.isDirectory = isDirectory; + } + + /*public Resource(String alias, String fileName, String description, int userId, ResourceType type, long size, Date createTime, Date updateTime) { this.alias = alias; this.fileName = fileName; this.description = description; @@ -98,6 +121,20 @@ public class Resource { this.size = size; this.createTime = createTime; this.updateTime = updateTime; + }*/ + + public Resource(int pid, String alias, String fullName, boolean isDirectory, String description, String fileName, int userId, ResourceType type, long size, Date createTime, Date updateTime) { + this.pid = pid; + this.alias = alias; + this.fullName = fullName; + this.isDirectory = isDirectory; + this.description = description; + this.fileName = fileName; + this.userId = userId; + this.type = type; + this.size = size; + this.createTime = createTime; + this.updateTime = updateTime; } public int getId() { @@ -116,6 +153,30 @@ public class Resource { this.alias = alias; } + public int getPid() { + return pid; + } + + public void setPid(int pid) { + this.pid = pid; + } + + public String getFullName() { + return fullName; + } + + public void setFullName(String fullName) { + this.fullName = fullName; + } + + public boolean isDirectory() { + return isDirectory; + } + + public void setDirectory(boolean directory) { + isDirectory = directory; + } + public String getFileName() { return fileName; } @@ -177,9 +238,12 @@ public class Resource { public String toString() { return "Resource{" + "id=" + id + + ", pid=" + pid + ", alias='" + alias + '\'' + - ", fileName='" + fileName + '\'' + + ", fullName='" + fullName + '\'' + + ", isDirectory=" + isDirectory + ", description='" + description + '\'' + + ", fileName='" + fileName + '\'' + ", userId=" + userId + ", type=" + type + ", size=" + size + diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.java index 9f9225cb04..b75bb58b7d 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.java @@ -20,9 +20,11 @@ import org.apache.dolphinscheduler.dao.entity.DefinitionGroupByUser; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import com.baomidou.mybatisplus.core.mapper.BaseMapper; import com.baomidou.mybatisplus.core.metadata.IPage; +import org.apache.ibatis.annotations.MapKey; import org.apache.ibatis.annotations.Param; import java.util.List; +import java.util.Map; /** * process definition mapper interface @@ -83,7 +85,7 @@ public interface ProcessDefinitionMapper extends BaseMapper { List queryDefinitionListByTenant(@Param("tenantId") int tenantId); /** - * count process definition group by user + * count process definition group by user * @param userId userId * @param projectIds projectIds * @param isAdmin isAdmin @@ -93,4 +95,11 @@ public interface ProcessDefinitionMapper extends BaseMapper { @Param("userId") Integer userId, @Param("projectIds") Integer[] projectIds, @Param("isAdmin") boolean isAdmin); + + /** + * list all resource ids + * @return resource ids list + */ + @MapKey("id") + List> listResources(); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java index cf65e5d08a..f07a92c0a2 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java @@ -30,12 +30,12 @@ public interface ResourceMapper extends BaseMapper { /** * query resource list - * @param alias alias + * @param fullName full name * @param userId userId * @param type type * @return resource list */ - List queryResourceList(@Param("alias") String alias, + List queryResourceList(@Param("fullName") String fullName, @Param("userId") int userId, @Param("type") int type); @@ -59,6 +59,7 @@ public interface ResourceMapper extends BaseMapper { */ IPage queryResourcePaging(IPage page, @Param("userId") int userId, + @Param("id") int id, @Param("type") int type, @Param("searchVal") String searchVal); @@ -76,13 +77,13 @@ public interface ResourceMapper extends BaseMapper { */ List queryResourceExceptUserId(@Param("userId") int userId); - /** * query tenant code by name * @param resName resource name + * @param resType resource type * @return tenant code */ - String queryTenantCodeByResourceName(@Param("resName") String resName); + String queryTenantCodeByResourceName(@Param("resName") String resName,@Param("resType") int resType); /** * list authorized resource @@ -91,4 +92,48 @@ public interface ResourceMapper extends BaseMapper { * @return resource list */ List listAuthorizedResource(@Param("userId") int userId,@Param("resNames")T[] resNames); + + /** + * list authorized resource + * @param userId userId + * @param resIds resource ids + * @return resource list + */ + List listAuthorizedResourceById(@Param("userId") int userId,@Param("resIds")T[] resIds); + + /** + * delete resource by id array + * @param resIds resource id array + * @return delete num + */ + int deleteIds(@Param("resIds")Integer[] resIds); + + /** + * list children + * @param direcotyId directory id + * @return resource id array + */ + List listChildren(@Param("direcotyId") int direcotyId); + + /** + * query resource by full name or pid + * @param fullName full name + * @param type resource type + * @return resource + */ + List queryResource(@Param("fullName") String fullName,@Param("type") int type); + + /** + * list resource by id array + * @param resIds resource id array + * @return resource list + */ + List listResourceByIds(@Param("resIds")Integer[] resIds); + + /** + * update resource + * @param resourceList resource list + * @return update num + */ + int batchUpdateResource(@Param("resourceList") List resourceList); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java index 5a8734233c..2411c9b178 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java @@ -86,4 +86,19 @@ public interface UdfFuncMapper extends BaseMapper { */ List listAuthorizedUdfFunc (@Param("userId") int userId,@Param("udfIds")T[] udfIds); + /** + * list UDF by resource id + * @param resourceIds resource id array + * @return UDF function list + */ + List listUdfByResourceId(@Param("resourceIds") int[] resourceIds); + + /** + * list authorized UDF by resource id + * @param resourceIds resource id array + * @return UDF function list + */ + List listAuthorizedUdfByResourceId(@Param("userId") int userId,@Param("resourceIds") int[] resourceIds); + + } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/MysqlUpgradeDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/MysqlUpgradeDao.java index a20a3acb95..255f1cf081 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/MysqlUpgradeDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/MysqlUpgradeDao.java @@ -66,7 +66,7 @@ public class MysqlUpgradeDao extends UpgradeDao { logger.error(e.getMessage(),e); throw new RuntimeException(e.getMessage(),e); } finally { - ConnectionUtils.releaseResource(rs, null, conn); + ConnectionUtils.releaseResource(rs, conn); } } @@ -89,7 +89,7 @@ public class MysqlUpgradeDao extends UpgradeDao { logger.error(e.getMessage(),e); throw new RuntimeException(e.getMessage(),e); } finally { - ConnectionUtils.releaseResource(null, null, conn); + ConnectionUtils.releaseResource(conn); } } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/PostgresqlUpgradeDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/PostgresqlUpgradeDao.java index c58521de1d..b4049450ab 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/PostgresqlUpgradeDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/PostgresqlUpgradeDao.java @@ -96,7 +96,7 @@ public class PostgresqlUpgradeDao extends UpgradeDao { logger.error(e.getMessage(),e); throw new RuntimeException(e.getMessage(),e); } finally { - ConnectionUtils.releaseResource(rs, null, conn); + ConnectionUtils.releaseResource(rs, conn); } } @@ -119,7 +119,7 @@ public class PostgresqlUpgradeDao extends UpgradeDao { logger.error(e.getMessage(),e); throw new RuntimeException(e.getMessage(),e); } finally { - ConnectionUtils.releaseResource(rs, null, conn); + ConnectionUtils.releaseResource(rs, conn); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java index 384c3604c4..e708620f8a 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java @@ -27,6 +27,7 @@ import org.apache.dolphinscheduler.dao.datasource.ConnectionFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.sql.DataSource; import java.io.*; import java.sql.Connection; import java.sql.PreparedStatement; @@ -40,7 +41,7 @@ public abstract class UpgradeDao extends AbstractBaseDao { private static final String T_VERSION_NAME = "t_escheduler_version"; private static final String T_NEW_VERSION_NAME = "t_ds_version"; private static final String rootDir = System.getProperty("user.dir"); - protected static final DruidDataSource dataSource = getDataSource(); + protected static final DataSource dataSource = getDataSource(); private static final DbType dbType = getCurrentDbType(); @Override @@ -52,13 +53,8 @@ public abstract class UpgradeDao extends AbstractBaseDao { * get datasource * @return DruidDataSource */ - public static DruidDataSource getDataSource(){ - DruidDataSource dataSource = ConnectionFactory.getInstance().getDataSource(); - dataSource.setInitialSize(2); - dataSource.setMinIdle(2); - dataSource.setMaxActive(2); - - return dataSource; + public static DataSource getDataSource(){ + return ConnectionFactory.getInstance().getDataSource(); } /** @@ -83,7 +79,7 @@ public abstract class UpgradeDao extends AbstractBaseDao { logger.error(e.getMessage(),e); return null; }finally { - ConnectionUtils.releaseResource(null, null, conn); + ConnectionUtils.releaseResource(conn); } } @@ -164,7 +160,7 @@ public abstract class UpgradeDao extends AbstractBaseDao { logger.error(e.getMessage(),e); throw new RuntimeException(e.getMessage(),e); } finally { - ConnectionUtils.releaseResource(null, null, conn); + ConnectionUtils.releaseResource(conn); } @@ -197,7 +193,7 @@ public abstract class UpgradeDao extends AbstractBaseDao { logger.error(e.getMessage(),e); throw new RuntimeException(e.getMessage(),e); } finally { - ConnectionUtils.releaseResource(null, null, conn); + ConnectionUtils.releaseResource(conn); } @@ -333,7 +329,7 @@ public abstract class UpgradeDao extends AbstractBaseDao { logger.error(e.getMessage(),e); throw new RuntimeException(e.getMessage(),e); } finally { - ConnectionUtils.releaseResource(null, pstmt, conn); + ConnectionUtils.releaseResource(pstmt, conn); } } @@ -376,7 +372,7 @@ public abstract class UpgradeDao extends AbstractBaseDao { logger.error(e.getMessage(),e); throw new RuntimeException(e.getMessage(),e); } finally { - ConnectionUtils.releaseResource(null, pstmt, conn); + ConnectionUtils.releaseResource(pstmt, conn); } } @@ -405,7 +401,7 @@ public abstract class UpgradeDao extends AbstractBaseDao { logger.error(e.getMessage(),e); throw new RuntimeException("sql: " + upgradeSQL, e); } finally { - ConnectionUtils.releaseResource(null, pstmt, conn); + ConnectionUtils.releaseResource(pstmt, conn); } } diff --git a/dolphinscheduler-dao/src/main/resources/application.properties b/dolphinscheduler-dao/src/main/resources/application.properties index 06b0ee94d5..c928c72df6 100644 --- a/dolphinscheduler-dao/src/main/resources/application.properties +++ b/dolphinscheduler-dao/src/main/resources/application.properties @@ -22,7 +22,7 @@ spring.datasource.driver-class-name=org.postgresql.Driver spring.datasource.url=jdbc:postgresql://localhost:5432/dolphinscheduler # mysql #spring.datasource.driver-class-name=com.mysql.jdbc.Driver -#spring.datasource.url=jdbc:mysql://192.168.xx.xx:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 +#spring.datasource.url=jdbc:mysql://localhost:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 # h2 #spring.datasource.driver-class-name=org.h2.Driver #spring.datasource.url=jdbc:h2:file:../sql/h2;AUTO_SERVER=TRUE diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.xml index f2157783e8..0cabf800cd 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.xml @@ -29,7 +29,9 @@ and pd.name = #{processDefinitionName} + + + \ No newline at end of file diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.xml index 3559ca9c85..e5697d1a60 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.xml @@ -66,7 +66,12 @@ select * from t_ds_resources - where type=#{type} - + where type=#{type} and pid=#{id} + and id in (select resources_id from t_ds_relation_resources_user where user_id=#{userId} union select id as resources_id from t_ds_resources where user_id=#{userId}) @@ -70,7 +70,74 @@ + + + + + delete from t_ds_resources where id in + + #{i} + + + + + + + + + + update t_ds_resources + + full_name=#{resource.fullName}, + update_time=#{resource.updateTime} + + + id=#{resource.id} + + + + + diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.xml index 0aa10607c4..e38d1637d6 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.xml @@ -87,4 +87,28 @@ + + \ No newline at end of file diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/AlertDaoTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/AlertDaoTest.java index b4f197adc9..b94162c563 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/AlertDaoTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/AlertDaoTest.java @@ -16,19 +16,42 @@ */ package org.apache.dolphinscheduler.dao; +import org.apache.dolphinscheduler.common.enums.AlertStatus; +import org.apache.dolphinscheduler.common.enums.AlertType; +import org.apache.dolphinscheduler.common.enums.ShowType; +import org.apache.dolphinscheduler.dao.entity.Alert; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class AlertDaoTest { - private static final Logger logger = LoggerFactory.getLogger(AlertDaoTest.class); +import java.util.Arrays; +import java.util.List; +public class AlertDaoTest { @Test - public void testGetAlertDao() { - logger.info("testGetAlertDao start"); + public void testAlertDao(){ AlertDao alertDao = DaoFactory.getDaoInstance(AlertDao.class); - Assert.assertNotNull(alertDao); - logger.info("testGetAlertDao end"); + Alert alert = new Alert(); + alert.setTitle("Mysql Exception"); + alert.setShowType(ShowType.TEXT); + alert.setContent("[\"alarm time：2018-02-05\", \"service name：MYSQL_ALTER\", \"alarm name：MYSQL_ALTER_DUMP\", " + + "\"get the alarm exception.！，interface error，exception information：timed out\", \"request address：http://blog.csdn.net/dreamInTheWorld/article/details/78539286\"]"); + alert.setAlertType(AlertType.EMAIL); + alert.setAlertGroupId(1); + alert.setAlertStatus(AlertStatus.WAIT_EXECUTION); + alertDao.addAlert(alert); + + + List alerts = alertDao.listWaitExecutionAlert(); + Assert.assertNotNull(alerts); + Assert.assertNotEquals(0, alerts.size()); + int id = alerts.get(0).getId(); + AlertStatus alertStatus = alerts.get(0).getAlertStatus(); + alertDao.updateAlert(AlertStatus.EXECUTION_SUCCESS, "", id); + + alerts = alertDao.listWaitExecutionAlert(); + Assert.assertEquals(0, alerts.size()); + alertDao.getAlertMapper().deleteById(id); } } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSourceTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSourceTest.java new file mode 100644 index 0000000000..6c44c3e329 --- /dev/null +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSourceTest.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.dao.datasource; + +import org.apache.dolphinscheduler.common.Constants; +import org.junit.Assert; +import org.junit.Test; + +public class BaseDataSourceTest { + + @Test + public void testDriverClassSelector() { + String mysqlDriverClass = new MySQLDataSource().driverClassSelector(); + Assert.assertEquals(Constants.COM_MYSQL_JDBC_DRIVER, mysqlDriverClass); + + String clickHouseDriverClass = new ClickHouseDataSource().driverClassSelector(); + Assert.assertEquals(Constants.COM_CLICKHOUSE_JDBC_DRIVER, clickHouseDriverClass); + + String db2ServerDriverClass = new DB2ServerDataSource().driverClassSelector(); + Assert.assertEquals(Constants.COM_DB2_JDBC_DRIVER, db2ServerDriverClass); + + String oracleDriverClass = new OracleDataSource().driverClassSelector(); + Assert.assertEquals(Constants.COM_ORACLE_JDBC_DRIVER, oracleDriverClass); + + String postgreDriverClass = new PostgreDataSource().driverClassSelector(); + Assert.assertEquals(Constants.ORG_POSTGRESQL_DRIVER, postgreDriverClass); + + String sqlServerDriverClass = new SQLServerDataSource().driverClassSelector(); + Assert.assertEquals(Constants.COM_SQLSERVER_JDBC_DRIVER, sqlServerDriverClass); + + String hiveDriverClass = new HiveDataSource().driverClassSelector(); + Assert.assertEquals(Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER, hiveDriverClass); + + String sparkDriverClass = new SparkDataSource().driverClassSelector(); + Assert.assertEquals(Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER, sparkDriverClass); + } + + @Test + public void testGetJdbcUrl() { + BaseDataSource hiveDataSource = new HiveDataSource(); + hiveDataSource.setAddress("jdbc:hive2://127.0.0.1:10000"); + hiveDataSource.setDatabase("test"); + hiveDataSource.setPassword("123456"); + hiveDataSource.setUser("test"); + Assert.assertEquals("jdbc:hive2://127.0.0.1:10000/test", hiveDataSource.getJdbcUrl()); + //set principal + hiveDataSource.setPrincipal("hive/test.com@TEST.COM"); + Assert.assertEquals("jdbc:hive2://127.0.0.1:10000/test;principal=hive/test.com@TEST.COM", + hiveDataSource.getJdbcUrl()); + //set fake other + hiveDataSource.setOther("charset=UTF-8"); + Assert.assertEquals( + "jdbc:hive2://127.0.0.1:10000/test;principal=hive/test.com@TEST.COM;charset=UTF-8", + hiveDataSource.getJdbcUrl()); + + BaseDataSource clickHouseDataSource = new ClickHouseDataSource(); + clickHouseDataSource.setAddress("jdbc:clickhouse://127.0.0.1:8123"); + clickHouseDataSource.setDatabase("test"); + clickHouseDataSource.setPassword("123456"); + clickHouseDataSource.setUser("test"); + Assert.assertEquals("jdbc:clickhouse://127.0.0.1:8123/test", clickHouseDataSource.getJdbcUrl()); + //set fake principal + clickHouseDataSource.setPrincipal("fake principal"); + Assert.assertEquals("jdbc:clickhouse://127.0.0.1:8123/test", clickHouseDataSource.getJdbcUrl()); + //set fake other + clickHouseDataSource.setOther("charset=UTF-8"); + Assert.assertEquals("jdbc:clickhouse://127.0.0.1:8123/test?charset=UTF-8", + clickHouseDataSource.getJdbcUrl()); + + BaseDataSource sqlServerDataSource = new SQLServerDataSource(); + sqlServerDataSource.setAddress("jdbc:sqlserver://127.0.0.1:1433"); + sqlServerDataSource.setDatabase("test"); + sqlServerDataSource.setPassword("123456"); + sqlServerDataSource.setUser("test"); + Assert.assertEquals("jdbc:sqlserver://127.0.0.1:1433;databaseName=test", + sqlServerDataSource.getJdbcUrl()); + //set fake principal + sqlServerDataSource.setPrincipal("fake principal"); + Assert.assertEquals("jdbc:sqlserver://127.0.0.1:1433;databaseName=test", + sqlServerDataSource.getJdbcUrl()); + //set fake other + sqlServerDataSource.setOther("charset=UTF-8"); + Assert.assertEquals("jdbc:sqlserver://127.0.0.1:1433;databaseName=test;charset=UTF-8", + sqlServerDataSource.getJdbcUrl()); + + BaseDataSource db2DataSource = new DB2ServerDataSource(); + db2DataSource.setAddress("jdbc:db2://127.0.0.1:50000"); + db2DataSource.setDatabase("test"); + db2DataSource.setPassword("123456"); + db2DataSource.setUser("test"); + Assert.assertEquals("jdbc:db2://127.0.0.1:50000/test", db2DataSource.getJdbcUrl()); + //set fake principal + db2DataSource.setPrincipal("fake principal"); + Assert.assertEquals("jdbc:db2://127.0.0.1:50000/test", db2DataSource.getJdbcUrl()); + //set fake other + db2DataSource.setOther("charset=UTF-8"); + Assert.assertEquals("jdbc:db2://127.0.0.1:50000/test:charset=UTF-8", db2DataSource.getJdbcUrl()); + + + } +} diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ConnectionFactoryTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ConnectionFactoryTest.java index f4139444f5..1d419a83d8 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ConnectionFactoryTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ConnectionFactoryTest.java @@ -31,7 +31,7 @@ public class ConnectionFactoryTest { */ @Test public void testConnection()throws Exception{ - Connection connection = ConnectionFactory.getInstance().getDataSource().getPooledConnection().getConnection(); + Connection connection = ConnectionFactory.getInstance().getDataSource().getConnection(); Assert.assertTrue(connection != null); } } \ No newline at end of file diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapMapperTest.java index 175dd57948..cbc056e63d 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapMapperTest.java @@ -60,7 +60,7 @@ public class ProcessInstanceMapMapperTest { //update processInstanceMap.setParentProcessInstanceId(1); int update = processInstanceMapMapper.updateById(processInstanceMap); - Assert.assertEquals(update, 1); + Assert.assertEquals(1, update); processInstanceMapMapper.deleteById(processInstanceMap.getId()); } @@ -71,7 +71,7 @@ public class ProcessInstanceMapMapperTest { public void testDelete(){ ProcessInstanceMap processInstanceMap = insertOne(); int delete = processInstanceMapMapper.deleteById(processInstanceMap.getId()); - Assert.assertEquals(delete, 1); + Assert.assertEquals(1, delete); } /** @@ -132,7 +132,7 @@ public class ProcessInstanceMapMapperTest { int delete = processInstanceMapMapper.deleteByParentProcessId( processInstanceMap.getParentProcessInstanceId() ); - Assert.assertEquals(delete, 1); + Assert.assertEquals(1, delete); } /** diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapperTest.java index 3b307cc2ad..fd4749c4bf 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapperTest.java @@ -74,7 +74,7 @@ public class ProcessInstanceMapperTest { ProcessInstance processInstanceMap = insertOne(); //update int update = processInstanceMapper.updateById(processInstanceMap); - Assert.assertEquals(update, 1); + Assert.assertEquals(1, update); processInstanceMapper.deleteById(processInstanceMap.getId()); } @@ -85,7 +85,7 @@ public class ProcessInstanceMapperTest { public void testDelete(){ ProcessInstance processInstanceMap = insertOne(); int delete = processInstanceMapper.deleteById(processInstanceMap.getId()); - Assert.assertEquals(delete, 1); + Assert.assertEquals(1, delete); } /** @@ -197,7 +197,7 @@ public class ProcessInstanceMapperTest { Assert.assertNotEquals(update, 0); processInstance = processInstanceMapper.selectById(processInstance.getId()); - Assert.assertEquals(processInstance.getHost(), null); + Assert.assertNull(processInstance.getHost()); processInstanceMapper.deleteById(processInstance.getId()); } @@ -217,7 +217,7 @@ public class ProcessInstanceMapperTest { ProcessInstance processInstance1 = processInstanceMapper.selectById(processInstance.getId()); processInstanceMapper.deleteById(processInstance.getId()); - Assert.assertEquals(processInstance1.getState(), ExecutionStatus.SUCCESS); + Assert.assertEquals(ExecutionStatus.SUCCESS, processInstance1.getState()); } @@ -261,10 +261,10 @@ public class ProcessInstanceMapperTest { List processInstances = processInstanceMapper.queryByProcessDefineId(processInstance.getProcessDefinitionId(), 1); - Assert.assertEquals(processInstances.size(), 1); + Assert.assertEquals(1, processInstances.size()); processInstances = processInstanceMapper.queryByProcessDefineId(processInstance.getProcessDefinitionId(), 2); - Assert.assertEquals(processInstances.size(), 2); + Assert.assertEquals(2, processInstances.size()); processInstanceMapper.deleteById(processInstance.getId()); processInstanceMapper.deleteById(processInstance1.getId()); @@ -320,7 +320,7 @@ public class ProcessInstanceMapperTest { start = new Date(2019-1900, 1-1, 01, 1, 0, 0); processInstance1 = processInstanceMapper.queryLastManualProcess(processInstance.getProcessDefinitionId(),start, end ); - Assert.assertEquals(processInstance1, null); + Assert.assertNull(processInstance1); processInstanceMapper.deleteById(processInstance.getId()); diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/QueueMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/QueueMapperTest.java index 30d2be03e0..ad3b05a048 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/QueueMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/QueueMapperTest.java @@ -64,7 +64,7 @@ public class QueueMapperTest { queue.setCreateTime(new Date()); //update int update = queueMapper.updateById(queue); - Assert.assertEquals(update, 1); + Assert.assertEquals(1, update); queueMapper.deleteById(queue.getId()); } @@ -75,7 +75,7 @@ public class QueueMapperTest { public void testDelete(){ Queue queue = insertOne(); int delete = queueMapper.deleteById(queue.getId()); - Assert.assertEquals(delete, 1); + Assert.assertEquals(1, delete); } /** diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapperTest.java index aaf5129c02..6a2aea5ad2 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapperTest.java @@ -34,6 +34,7 @@ import org.springframework.test.annotation.Rollback; import org.springframework.test.context.junit4.SpringRunner; import org.springframework.transaction.annotation.Transactional; +import java.util.ArrayList; import java.util.Arrays; import java.util.Date; import java.util.List; @@ -68,7 +69,10 @@ public class ResourceMapperTest { private Resource insertOne(){ //insertOne Resource resource = new Resource(); - resource.setAlias("ut resource"); + resource.setAlias("ut-resource"); + resource.setFullName("/ut-resource"); + resource.setPid(-1); + resource.setDirectory(false); resource.setType(ResourceType.FILE); resource.setUserId(111); resourceMapper.insert(resource); @@ -80,16 +84,32 @@ public class ResourceMapperTest { * @param user user * @return Resource */ - private Resource createResource(User user){ + private Resource createResource(User user,boolean isDirectory,ResourceType resourceType,int pid,String alias,String fullName){ //insertOne Resource resource = new Resource(); - resource.setAlias(String.format("ut resource %s",user.getUserName())); - resource.setType(ResourceType.FILE); + resource.setDirectory(isDirectory); + resource.setType(resourceType); + resource.setAlias(alias); + resource.setFullName(fullName); resource.setUserId(user.getId()); resourceMapper.insert(resource); return resource; } + /** + * create resource by user + * @param user user + * @return Resource + */ + private Resource createResource(User user){ + //insertOne + String alias = String.format("ut-resource-%s",user.getUserName()); + String fullName = String.format("/%s",alias); + + Resource resource = createResource(user, false, ResourceType.FILE, -1, alias, fullName); + return resource; + } + /** * create user * @return User @@ -138,7 +158,7 @@ public class ResourceMapperTest { resource.setCreateTime(new Date()); //update int update = resourceMapper.updateById(resource); - Assert.assertEquals(update, 1); + Assert.assertEquals(1, update); resourceMapper.deleteById(resource.getId()); } @@ -149,7 +169,7 @@ public class ResourceMapperTest { public void testDelete(){ Resource resourceMap = insertOne(); int delete = resourceMapper.deleteById(resourceMap.getId()); - Assert.assertEquals(delete, 1); + Assert.assertEquals(1, delete); } /** @@ -200,13 +220,15 @@ public class ResourceMapperTest { IPage resourceIPage = resourceMapper.queryResourcePaging( page, - resource.getUserId(), + 0, + -1, resource.getType().ordinal(), "" ); IPage resourceIPage1 = resourceMapper.queryResourcePaging( page, 1110, + -1, resource.getType().ordinal(), "" ); @@ -236,8 +258,8 @@ public class ResourceMapperTest { resourceUserMapper.deleteById(resourcesUser.getId()); resourceMapper.deleteById(resource.getId()); - Assert.assertEquals(resources.size(), 0); - Assert.assertNotEquals(resources1.size(), 0); + Assert.assertEquals(0, resources.size()); + Assert.assertNotEquals(0, resources1.size()); } @@ -251,7 +273,7 @@ public class ResourceMapperTest { List resources = resourceMapper.queryAuthorizedResourceList(resource.getUserId()); resourceMapper.deleteById(resource.getId()); - Assert.assertEquals(resources.size(), 0); + Assert.assertEquals(0, resources.size()); } /** @@ -289,11 +311,11 @@ public class ResourceMapperTest { resourceMapper.updateById(resource); String resource1 = resourceMapper.queryTenantCodeByResourceName( - resource.getAlias() + resource.getFullName(),ResourceType.FILE.ordinal() ); - Assert.assertEquals(resource1, "ut tenant code for resource"); + Assert.assertEquals("ut tenant code for resource", resource1); resourceMapper.deleteById(resource.getId()); } @@ -305,22 +327,37 @@ public class ResourceMapperTest { User generalUser2 = createGeneralUser("user2"); // create one resource Resource resource = createResource(generalUser2); - Resource unauthorizedResource = createResource(generalUser2); + Resource unauthorizedResource = createResource(generalUser1); // need download resources - String[] resNames = new String[]{resource.getAlias(), unauthorizedResource.getAlias()}; + String[] resNames = new String[]{resource.getFullName(), unauthorizedResource.getFullName()}; List resources = resourceMapper.listAuthorizedResource(generalUser2.getId(), resNames); Assert.assertEquals(generalUser2.getId(),resource.getUserId()); - Assert.assertFalse(resources.stream().map(t -> t.getAlias()).collect(toList()).containsAll(Arrays.asList(resNames))); + Assert.assertFalse(resources.stream().map(t -> t.getFullName()).collect(toList()).containsAll(Arrays.asList(resNames))); // authorize object unauthorizedResource to generalUser createResourcesUser(unauthorizedResource,generalUser2); List authorizedResources = resourceMapper.listAuthorizedResource(generalUser2.getId(), resNames); - Assert.assertTrue(authorizedResources.stream().map(t -> t.getAlias()).collect(toList()).containsAll(Arrays.asList(resNames))); + Assert.assertTrue(authorizedResources.stream().map(t -> t.getFullName()).collect(toList()).containsAll(Arrays.asList(resNames))); + + } + + @Test + public void deleteIdsTest(){ + // create a general user + User generalUser1 = createGeneralUser("user1"); + + Resource resource = createResource(generalUser1); + Resource resource1 = createResource(generalUser1); + List resourceList = new ArrayList<>(); + resourceList.add(resource.getId()); + resourceList.add(resource1.getId()); + int result = resourceMapper.deleteIds(resourceList.toArray(new Integer[resourceList.size()])); + Assert.assertEquals(result,2); } } \ No newline at end of file diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapperTest.java index 233e88c5dd..2434a8c284 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapperTest.java @@ -62,7 +62,7 @@ public class ResourceUserMapperTest { queue.setCreateTime(new Date()); //update int update = resourceUserMapper.updateById(queue); - Assert.assertEquals(update, 1); + Assert.assertEquals(1, update); resourceUserMapper.deleteById(queue.getId()); } @@ -73,7 +73,7 @@ public class ResourceUserMapperTest { public void testDelete(){ ResourcesUser queue = insertOne(); int delete = resourceUserMapper.deleteById(queue.getId()); - Assert.assertEquals(delete, 1); + Assert.assertEquals(1, delete); } /** diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapperTest.java index 51a22b756e..ffd49b4f0d 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapperTest.java @@ -78,7 +78,7 @@ public class TaskInstanceMapperTest { TaskInstance taskInstance = insertOne(); //update int update = taskInstanceMapper.updateById(taskInstance); - Assert.assertEquals(update, 1); + Assert.assertEquals(1, update); taskInstanceMapper.deleteById(taskInstance.getId()); } @@ -89,7 +89,7 @@ public class TaskInstanceMapperTest { public void testDelete(){ TaskInstance taskInstance = insertOne(); int delete = taskInstanceMapper.deleteById(taskInstance.getId()); - Assert.assertEquals(delete, 1); + Assert.assertEquals(1, delete); } /** diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TenantMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TenantMapperTest.java index f5cb8fca12..dc6a7d1710 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TenantMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TenantMapperTest.java @@ -64,7 +64,7 @@ public class TenantMapperTest { tenant.setUpdateTime(new Date()); //update int update = tenantMapper.updateById(tenant); - Assert.assertEquals(update, 1); + Assert.assertEquals(1, update); tenantMapper.deleteById(tenant.getId()); } @@ -75,7 +75,7 @@ public class TenantMapperTest { public void testDelete(){ Tenant tenant = insertOne(); int delete = tenantMapper.deleteById(tenant.getId()); - Assert.assertEquals(delete, 1); + Assert.assertEquals(1, delete); } /** diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/WorkerGroupMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/WorkerGroupMapperTest.java index ea05f1bf11..dfa2a5de8e 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/WorkerGroupMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/WorkerGroupMapperTest.java @@ -65,7 +65,7 @@ public class WorkerGroupMapperTest { workerGroup.setName("workerGroup11"); int update = workerGroupMapper.updateById(workerGroup); workerGroupMapper.deleteById(workerGroup.getId()); - Assert.assertEquals(update, 1); + Assert.assertEquals(1, update); } /** @@ -77,7 +77,7 @@ public class WorkerGroupMapperTest { WorkerGroup workerGroup = insertOne(); //delete int delete = workerGroupMapper.deleteById(workerGroup.getId()); - Assert.assertEquals(delete, 1); + Assert.assertEquals(1, delete); } /** diff --git a/dolphinscheduler-dist/release-docs/LICENSE b/dolphinscheduler-dist/release-docs/LICENSE index 97946d1172..82e641ec72 100644 --- a/dolphinscheduler-dist/release-docs/LICENSE +++ b/dolphinscheduler-dist/release-docs/LICENSE @@ -518,6 +518,8 @@ MIT licenses js-cookie 2.2.1: https://github.com/js-cookie/js-cookie MIT jsplumb 2.8.6: https://github.com/jsplumb/jsplumb MIT and GPLv2 lodash 4.17.11: https://github.com/lodash/lodash MIT + normalize.css 8.0.1: https://github.com/necolas/normalize.css MIT + vue-treeselect 0.4.0: https://github.com/riophae/vue-treeselect MIT vue 2.5.17: https://github.com/vuejs/vue MIT vue-router 2.7.0: https://github.com/vuejs/vue-router MIT vuex 3.0.0: https://github.com/vuejs/vuex MIT diff --git a/dolphinscheduler-dist/release-docs/licenses/ui-licenses/LICENSE-normalize b/dolphinscheduler-dist/release-docs/licenses/ui-licenses/LICENSE-normalize new file mode 100644 index 0000000000..90e0c091a5 --- /dev/null +++ b/dolphinscheduler-dist/release-docs/licenses/ui-licenses/LICENSE-normalize @@ -0,0 +1,8 @@ +The MIT License (MIT) +Copyright © Nicolas Gallagher and Jonathan Neal + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/dolphinscheduler-dist/release-docs/licenses/ui-licenses/LICENSE-vue-treeselect b/dolphinscheduler-dist/release-docs/licenses/ui-licenses/LICENSE-vue-treeselect new file mode 100644 index 0000000000..f7d8cc3ebd --- /dev/null +++ b/dolphinscheduler-dist/release-docs/licenses/ui-licenses/LICENSE-vue-treeselect @@ -0,0 +1,20 @@ +Copyright (c) 2017-present Riophae Lee + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/dolphinscheduler-remote/pom.xml b/dolphinscheduler-remote/pom.xml index 39c7c6a7c0..0968e610bc 100644 --- a/dolphinscheduler-remote/pom.xml +++ b/dolphinscheduler-remote/pom.xml @@ -1,5 +1,20 @@ - + diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingClient.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingClient.java index 96258d752a..d3def39c02 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingClient.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingClient.java @@ -301,7 +301,7 @@ public class NettyRemotingClient { return channel; } } catch (Exception ex) { - logger.info("connect to {} error {}", address, ex); + logger.error("connect to {} error", address, ex); } return null; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/ResponseFuture.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/ResponseFuture.java index ca304646e4..2e3954f4bc 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/ResponseFuture.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/ResponseFuture.java @@ -32,9 +32,9 @@ import java.util.concurrent.*; */ public class ResponseFuture { - private final static Logger LOGGER = LoggerFactory.getLogger(ResponseFuture.class); + private static final Logger LOGGER = LoggerFactory.getLogger(ResponseFuture.class); - private final static ConcurrentHashMap FUTURE_TABLE = new ConcurrentHashMap<>(256); + private static final ConcurrentHashMap FUTURE_TABLE = new ConcurrentHashMap<>(256); /** * request unique identification @@ -63,11 +63,11 @@ public class ResponseFuture { /** * response command */ - private volatile Command responseCommand; + private Command responseCommand; private volatile boolean sendOk = true; - private volatile Throwable cause; + private Throwable cause; public ResponseFuture(long opaque, long timeoutMillis, InvokeCallback invokeCallback, ReleaseSemaphore releaseSemaphore) { this.opaque = opaque; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java index 88cd2d5518..c601aa9891 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java @@ -117,7 +117,7 @@ public class NettyServerHandler extends ChannelInboundHandlerAdapter { try { pair.getLeft().process(channel, msg); } catch (Throwable ex) { - logger.error("process msg {} error : {}", msg, ex); + logger.error("process msg {} error", msg, ex); } } }; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java index 26457a386a..d6ee6a3b1d 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java @@ -674,9 +674,9 @@ public class MasterExecThread implements Runnable { TaskNode taskNode = dag.getNode(taskName); List depNameList = taskNode.getDepList(); for(String depsNode : depNameList ){ - - if(forbiddenTaskList.containsKey(depsNode) || - skipTaskNodeList.containsKey(depsNode)){ + if(!dag.containsNode(depsNode) + || forbiddenTaskList.containsKey(depsNode) + || skipTaskNodeList.containsKey(depsNode)){ continue; } // dependencies must be fully completed @@ -937,7 +937,7 @@ public class MasterExecThread implements Runnable { // submit start node submitPostNode(null); boolean sendTimeWarning = false; - while(!processInstance.IsProcessInstanceStop()){ + while(!processInstance.isProcessInstanceStop()){ // send warning email if process time out. if( !sendTimeWarning && checkProcessTimeOut(processInstance) ){ diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/SubProcessTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/SubProcessTaskExecThread.java index fc16b5112b..13a59505bc 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/SubProcessTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/SubProcessTaskExecThread.java @@ -95,7 +95,7 @@ public class SubProcessTaskExecThread extends MasterBaseTaskExecThread { * set task instance state * @return */ - private Boolean setTaskInstanceState(){ + private boolean setTaskInstanceState(){ subProcessInstance = processService.findSubProcessInstance(processInstance.getId(), taskInstance.getId()); if(subProcessInstance == null || taskInstance.getState().typeIsFinished()){ return false; @@ -131,8 +131,8 @@ public class SubProcessTaskExecThread extends MasterBaseTaskExecThread { if (taskInstance.getState().typeIsFinished()) { logger.info("sub work flow task {} already complete. task state:{}, parent work flow instance state:{}", this.taskInstance.getName(), - this.taskInstance.getState().toString(), - this.processInstance.getState().toString()); + this.taskInstance.getState(), + this.processInstance.getState()); return; } while (Stopper.isRunning()) { diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/RemoveZKNode.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/RemoveZKNode.java index 5550e750b5..caec6e78a8 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/RemoveZKNode.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/RemoveZKNode.java @@ -28,7 +28,7 @@ import org.springframework.context.annotation.ComponentScan; @ComponentScan("org.apache.dolphinscheduler") public class RemoveZKNode implements CommandLineRunner { - private static Integer ARGS_LENGTH = 1; + private static final Integer ARGS_LENGTH = 1; private static final Logger logger = LoggerFactory.getLogger(RemoveZKNode.class); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java index 86bb7d3e07..868b80960f 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java @@ -52,7 +52,6 @@ import org.springframework.context.annotation.FilterType; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.util.Set; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -255,7 +254,7 @@ public class WorkerServer implements IStoppable { */ private Runnable heartBeatThread(){ logger.info("start worker heart beat thread..."); - Runnable heartBeatThread = new Runnable() { + return new Runnable() { @Override public void run() { // send heartbeat to zk @@ -266,7 +265,6 @@ public class WorkerServer implements IStoppable { zkWorkerClient.heartBeatForZk(zkWorkerClient.getWorkerZNode() , Constants.WORKER_PREFIX); } }; - return heartBeatThread; } @@ -276,7 +274,7 @@ public class WorkerServer implements IStoppable { * @return kill process thread */ private Runnable getKillProcessThread(){ - Runnable killProcessThread = new Runnable() { + return new Runnable() { @Override public void run() { logger.info("start listening kill process thread..."); @@ -297,7 +295,6 @@ public class WorkerServer implements IStoppable { } } }; - return killProcessThread; } /** @@ -307,17 +304,17 @@ public class WorkerServer implements IStoppable { * @param pd process dao */ private void killTask(String taskInfo, ProcessService pd) { - logger.info("get one kill command from tasks kill queue: " + taskInfo); + logger.info("get one kill command from tasks kill queue: {}" , taskInfo); String[] taskInfoArray = taskInfo.split("-"); if(taskInfoArray.length != 2){ - logger.error("error format kill info: " + taskInfo); + logger.error("error format kill info: {}", taskInfo); return ; } String host = taskInfoArray[0]; int taskInstanceId = Integer.parseInt(taskInfoArray[1]); TaskInstance taskInstance = pd.getTaskInstanceDetailByTaskId(taskInstanceId); if(taskInstance == null){ - logger.error("cannot find the kill task :" + taskInfo); + logger.error("cannot find the kill task : {}", taskInfo); return; } @@ -332,8 +329,7 @@ public class WorkerServer implements IStoppable { }else if(!taskInstance.getState().typeIsFinished()){ ProcessUtils.kill(taskInstance); }else{ - logger.info("the task aleady finish: task id: " + taskInstance.getId() - + " state: " + taskInstance.getState().toString()); + logger.info("the task aleady finish: task id: {} state: {}", taskInstance.getId(), taskInstance.getState()); } } } @@ -347,7 +343,7 @@ public class WorkerServer implements IStoppable { private void deleteTaskFromQueue(TaskInstance taskInstance, ProcessService pd){ // creating distributed locks, lock path /dolphinscheduler/lock/worker InterProcessMutex mutex = null; - logger.info("delete task from tasks queue: " + taskInstance.getId()); + logger.info("delete task from tasks queue: {}", taskInstance.getId()); try { mutex = zkWorkerClient.acquireZkLock(zkWorkerClient.getZkClient(), diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java index c4d4b61af5..aaaf5c7805 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java @@ -34,7 +34,7 @@ public class WorkerConfig { @Value("${worker.max.cpuload.avg}") private int workerMaxCpuloadAvg; - @Value("${master.reserved.memory}") + @Value("${worker.reserved.memory}") private double workerReservedMemory; public int getWorkerExecThreads() { diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskScheduleThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskScheduleThread.java index d36d4de5b4..c7806f1b66 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskScheduleThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskScheduleThread.java @@ -23,15 +23,18 @@ import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.AuthorizationType; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; +import org.apache.dolphinscheduler.common.enums.ResourceType; import org.apache.dolphinscheduler.common.enums.TaskType; import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.process.Property; +import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.TaskTimeoutParameter; import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.dolphinscheduler.common.utils.HadoopUtils; import org.apache.dolphinscheduler.common.utils.TaskParametersUtils; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; +import org.apache.dolphinscheduler.dao.entity.Resource; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.common.log.TaskLogDiscriminator; @@ -96,7 +99,7 @@ public class TaskScheduleThread implements Runnable { TaskNode taskNode = JSON.parseObject(taskInstance.getTaskJson(), TaskNode.class); // get resource files - List resourceFiles = createProjectResFiles(taskNode); + List resourceFiles = createProjectResFiles(taskNode); // copy hdfs/minio file to local downloadResource( taskInstance.getExecutePath(), @@ -165,6 +168,7 @@ public class TaskScheduleThread implements Runnable { new Date(), taskInstance.getId()); } + /** * get global paras map * @return @@ -230,7 +234,7 @@ public class TaskScheduleThread implements Runnable { taskInstance.getId() + ".log"; } }catch (Exception e){ - logger.error("logger" + e); + logger.error("logger {}", e.getMessage(), e); logPath = ""; } return logPath; @@ -289,14 +293,16 @@ public class TaskScheduleThread implements Runnable { /** * create project resource files */ - private List createProjectResFiles(TaskNode taskNode) throws Exception{ + private List createProjectResFiles(TaskNode taskNode) throws Exception{ - Set projectFiles = new HashSet<>(); + Set projectFiles = new HashSet<>(); AbstractParameters baseParam = TaskParametersUtils.getParameters(taskNode.getType(), taskNode.getParams()); if (baseParam != null) { - List projectResourceFiles = baseParam.getResourceFilesList(); - projectFiles.addAll(projectResourceFiles); + List projectResourceFiles = baseParam.getResourceFilesList(); + if (projectResourceFiles != null) { + projectFiles.addAll(projectResourceFiles); + } } return new ArrayList<>(projectFiles); @@ -309,18 +315,25 @@ public class TaskScheduleThread implements Runnable { * @param projectRes * @param logger */ - private void downloadResource(String execLocalPath, List projectRes, Logger logger) throws Exception { + private void downloadResource(String execLocalPath, List projectRes, Logger logger) throws Exception { checkDownloadPermission(projectRes); - for (String res : projectRes) { - File resFile = new File(execLocalPath, res); + String resourceName; + for (ResourceInfo res : projectRes) { + if (res.getId() != 0) { + Resource resource = processService.getResourceById(res.getId()); + resourceName = resource.getFullName(); + }else{ + resourceName = res.getRes(); + } + File resFile = new File(execLocalPath, resourceName); if (!resFile.exists()) { try { // query the tenant code of the resource according to the name of the resource - String tentnCode = processService.queryTenantCodeByResName(res); - String resHdfsPath = HadoopUtils.getHdfsFilename(tentnCode, res); + String tentnCode = processService.queryTenantCodeByResName(resourceName, ResourceType.FILE); + String resHdfsPath = HadoopUtils.getHdfsResourceFileName(tentnCode, resourceName); logger.info("get resource file from hdfs :{}", resHdfsPath); - HadoopUtils.getInstance().copyHdfsToLocal(resHdfsPath, execLocalPath + File.separator + res, false, true); + HadoopUtils.getInstance().copyHdfsToLocal(resHdfsPath, execLocalPath + File.separator + resourceName, false, true); }catch (Exception e){ logger.error(e.getMessage(),e); throw new RuntimeException(e.getMessage()); @@ -336,10 +349,17 @@ public class TaskScheduleThread implements Runnable { * @param projectRes resource name list * @throws Exception exception */ - private void checkDownloadPermission(List projectRes) throws Exception { + private void checkDownloadPermission(List projectRes) throws Exception { + int userId = taskInstance.getProcessInstance().getExecutorId(); - String[] resNames = projectRes.toArray(new String[projectRes.size()]); - PermissionCheck permissionCheck = new PermissionCheck<>(AuthorizationType.RESOURCE_FILE, processService,resNames,userId,logger); - permissionCheck.checkPermission(); + if (projectRes.stream().allMatch(t->t.getId() == 0)) { + String[] resNames = projectRes.stream().map(t -> t.getRes()).collect(Collectors.toList()).toArray(new String[projectRes.size()]); + PermissionCheck permissionCheck = new PermissionCheck(AuthorizationType.RESOURCE_FILE_NAME,processService,resNames,userId,logger); + permissionCheck.checkPermission(); + }else{ + Integer[] resIds = projectRes.stream().map(t -> t.getId()).collect(Collectors.toList()).toArray(new Integer[projectRes.size()]); + PermissionCheck permissionCheck = new PermissionCheck(AuthorizationType.RESOURCE_FILE_ID,processService,resIds,userId,logger); + permissionCheck.checkPermission(); + } } } \ No newline at end of file diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractYarnTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractYarnTask.java index 39f4dfbb97..cda12ca525 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractYarnTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractYarnTask.java @@ -94,4 +94,9 @@ public abstract class AbstractYarnTask extends AbstractTask { * @throws Exception exception */ protected abstract String buildCommand() throws Exception; + + /** + * set main jar name + */ + protected abstract void setMainJarName(); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentExecute.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentExecute.java index b08cabc2e9..087bb80ccb 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentExecute.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentExecute.java @@ -22,6 +22,7 @@ import org.apache.dolphinscheduler.common.enums.DependentRelation; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.model.DateInterval; import org.apache.dolphinscheduler.common.model.DependentItem; +import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.utils.DependentUtils; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance; @@ -82,7 +83,7 @@ public class DependentExecute { * @param currentTime current time * @return DependResult */ - public DependResult getDependentResultForItem(DependentItem dependentItem, Date currentTime){ + private DependResult getDependentResultForItem(DependentItem dependentItem, Date currentTime){ List dateIntervals = DependentUtils.getDateIntervalList(currentTime, dependentItem.getDateValue()); return calculateResultForTasks(dependentItem, dateIntervals ); } @@ -94,7 +95,8 @@ public class DependentExecute { * @return dateIntervals */ private DependResult calculateResultForTasks(DependentItem dependentItem, - List dateIntervals) { + List dateIntervals) { + DependResult result = DependResult.FAILED; for(DateInterval dateInterval : dateIntervals){ ProcessInstance processInstance = findLastProcessInterval(dependentItem.getDefinitionId(), @@ -104,25 +106,35 @@ public class DependentExecute { dependentItem.getDefinitionId(), dateInterval.getStartTime(), dateInterval.getEndTime() ); return DependResult.FAILED; } + // need to check workflow for updates, so get all task and check the task state if(dependentItem.getDepTasks().equals(Constants.DEPENDENT_ALL)){ - result = getDependResultByState(processInstance.getState()); - }else{ - TaskInstance taskInstance = null; - List taskInstanceList = processService.findValidTaskListByProcessId(processInstance.getId()); + List taskNodes = + processService.getTaskNodeListByDefinitionId(dependentItem.getDefinitionId()); - for(TaskInstance task : taskInstanceList){ - if(task.getName().equals(dependentItem.getDepTasks())){ - taskInstance = task; - break; + if(taskNodes != null && taskNodes.size() > 0){ + List results = new ArrayList<>(); + DependResult tmpResult = DependResult.FAILED; + for(TaskNode taskNode:taskNodes){ + tmpResult = getDependTaskResult(taskNode.getName(),processInstance); + if(DependResult.FAILED == tmpResult){ + break; + }else{ + results.add(getDependTaskResult(taskNode.getName(),processInstance)); + } + } + + if(DependResult.FAILED == tmpResult){ + result = DependResult.FAILED; + }else if(results.contains(DependResult.WAITING)){ + result = DependResult.WAITING; + }else{ + result = DependResult.SUCCESS; } - } - if(taskInstance == null){ - // cannot find task in the process instance - // maybe because process instance is running or failed. - result = getDependResultByState(processInstance.getState()); }else{ - result = getDependResultByState(taskInstance.getState()); + result = DependResult.FAILED; } + }else{ + result = getDependTaskResult(dependentItem.getDepTasks(),processInstance); } if(result != DependResult.SUCCESS){ break; @@ -131,6 +143,35 @@ public class DependentExecute { return result; } + /** + * get depend task result + * @param taskName + * @param processInstance + * @return + */ + private DependResult getDependTaskResult(String taskName, ProcessInstance processInstance) { + DependResult result = DependResult.FAILED; + TaskInstance taskInstance = null; + List taskInstanceList = processService.findValidTaskListByProcessId(processInstance.getId()); + + for(TaskInstance task : taskInstanceList){ + if(task.getName().equals(taskName)){ + taskInstance = task; + break; + } + } + + if(taskInstance == null){ + // cannot find task in the process instance + // maybe because process instance is running or failed. + result = getDependResultByProcessStateWhenTaskNull(processInstance.getState()); + }else{ + result = getDependResultByState(taskInstance.getState()); + } + + return result; + } + /** * find the last one process instance that : * 1. manual run and finish between the interval @@ -172,7 +213,9 @@ public class DependentExecute { */ private DependResult getDependResultByState(ExecutionStatus state) { - if(state.typeIsRunning() || state == ExecutionStatus.SUBMITTED_SUCCESS || state == ExecutionStatus.WAITTING_THREAD){ + if(state.typeIsRunning() + || state == ExecutionStatus.SUBMITTED_SUCCESS + || state == ExecutionStatus.WAITTING_THREAD){ return DependResult.WAITING; }else if(state.typeIsSuccess()){ return DependResult.SUCCESS; @@ -181,6 +224,22 @@ public class DependentExecute { } } + /** + * get dependent result by task instance state when task instance is null + * @param state state + * @return DependResult + */ + private DependResult getDependResultByProcessStateWhenTaskNull(ExecutionStatus state) { + + if(state.typeIsRunning() + || state == ExecutionStatus.SUBMITTED_SUCCESS + || state == ExecutionStatus.WAITTING_THREAD){ + return DependResult.WAITING; + }else{ + return DependResult.FAILED; + } + } + /** * judge depend item finished * @param currentTime current time @@ -222,7 +281,7 @@ public class DependentExecute { * @param currentTime current time * @return DependResult */ - public DependResult getDependResultForItem(DependentItem item, Date currentTime){ + private DependResult getDependResultForItem(DependentItem item, Date currentTime){ String key = item.getKey(); if(dependResultMap.containsKey(key)){ return dependResultMap.get(key); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentTask.java index f074d57e6c..b426d32502 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentTask.java @@ -82,10 +82,11 @@ public class DependentTask extends AbstractTask { this.dependentParameters = JSONUtils.parseObject(this.taskProps.getDependence(), DependentParameters.class); - - for(DependentTaskModel taskModel : dependentParameters.getDependTaskList()){ - this.dependentTaskList.add(new DependentExecute( - taskModel.getDependItemList(), taskModel.getRelation())); + if(dependentParameters != null){ + for(DependentTaskModel taskModel : dependentParameters.getDependTaskList()){ + this.dependentTaskList.add(new DependentExecute( + taskModel.getDependItemList(), taskModel.getRelation())); + } } this.processService = SpringApplicationContext.getBean(ProcessService.class); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/flink/FlinkTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/flink/FlinkTask.java index c562fbe4dd..0dc7c6a638 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/flink/FlinkTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/flink/FlinkTask.java @@ -17,12 +17,14 @@ package org.apache.dolphinscheduler.server.worker.task.flink; import org.apache.dolphinscheduler.common.process.Property; +import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.flink.FlinkParameters; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; +import org.apache.dolphinscheduler.dao.entity.Resource; import org.apache.dolphinscheduler.server.utils.FlinkArgsUtils; import org.apache.dolphinscheduler.server.utils.ParamUtils; import org.apache.dolphinscheduler.server.worker.task.AbstractYarnTask; @@ -63,6 +65,7 @@ public class FlinkTask extends AbstractYarnTask { if (!flinkParameters.checkParameters()) { throw new RuntimeException("flink task params is not valid"); } + setMainJarName(); flinkParameters.setQueue(taskProps.getQueue()); if (StringUtils.isNotEmpty(flinkParameters.getMainArgs())) { @@ -111,6 +114,28 @@ public class FlinkTask extends AbstractYarnTask { return command; } + @Override + protected void setMainJarName() { + // main jar + ResourceInfo mainJar = flinkParameters.getMainJar(); + if (mainJar != null) { + int resourceId = mainJar.getId(); + String resourceName; + if (resourceId == 0) { + resourceName = mainJar.getRes(); + } else { + Resource resource = processService.getResourceById(flinkParameters.getMainJar().getId()); + if (resource == null) { + logger.error("resource id: {} not exist", resourceId); + throw new RuntimeException(String.format("resource id: %d not exist", resourceId)); + } + resourceName = resource.getFullName().replaceFirst("/", ""); + } + mainJar.setRes(resourceName); + flinkParameters.setMainJar(mainJar); + } + } + @Override public AbstractParameters getParameters() { return flinkParameters; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/mr/MapReduceTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/mr/MapReduceTask.java index 7f6baad427..0909fbd06e 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/mr/MapReduceTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/mr/MapReduceTask.java @@ -19,11 +19,13 @@ package org.apache.dolphinscheduler.server.worker.task.mr; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ProgramType; import org.apache.dolphinscheduler.common.process.Property; +import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.mr.MapreduceParameters; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.apache.dolphinscheduler.dao.entity.Resource; import org.apache.dolphinscheduler.server.utils.ParamUtils; import org.apache.dolphinscheduler.server.worker.task.AbstractYarnTask; import org.apache.dolphinscheduler.server.worker.task.TaskProps; @@ -64,7 +66,7 @@ public class MapReduceTask extends AbstractYarnTask { if (!mapreduceParameters.checkParameters()) { throw new RuntimeException("mapreduce task params is not valid"); } - + setMainJarName(); mapreduceParameters.setQueue(taskProps.getQueue()); // replace placeholder @@ -99,6 +101,28 @@ public class MapReduceTask extends AbstractYarnTask { return command; } + @Override + protected void setMainJarName() { + // main jar + ResourceInfo mainJar = mapreduceParameters.getMainJar(); + if (mainJar != null) { + int resourceId = mainJar.getId(); + String resourceName; + if (resourceId == 0) { + resourceName = mainJar.getRes(); + } else { + Resource resource = processService.getResourceById(mapreduceParameters.getMainJar().getId()); + if (resource == null) { + logger.error("resource id: {} not exist", resourceId); + throw new RuntimeException(String.format("resource id: %d not exist", resourceId)); + } + resourceName = resource.getFullName().replaceFirst("/", ""); + } + mainJar.setRes(resourceName); + mapreduceParameters.setMainJar(mainJar); + } + } + @Override public AbstractParameters getParameters() { return mapreduceParameters; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java index 2c8433aa01..fae514c03d 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java @@ -134,7 +134,6 @@ public class ShellTask extends AbstractTask { String script = shellParameters.getRawScript().replaceAll("\\r\\n", "\n"); - /** * combining local and global parameters */ @@ -143,18 +142,17 @@ public class ShellTask extends AbstractTask { shellParameters.getLocalParametersMap(), taskProps.getCmdTypeIfComplement(), taskProps.getScheduleTime()); -// if (paramsMap != null){ -// script = ParameterUtils.convertParameterPlaceholders(script, ParamUtils.convert(paramsMap)); -// } - //new -// replace variable TIME with $[YYYYmmddd...] in shell file when history run job and batch complement job + // new + // replace variable TIME with $[YYYYmmddd...] in shell file when history run job and batch complement job if (paramsMap != null) { - String dateTime = DateUtils.format(taskProps.getScheduleTime(), Constants.PARAMETER_FORMAT_TIME); - Property p = new Property(); - p.setValue(dateTime); - p.setProp(Constants.PARAMETER_SHECDULE_TIME); - paramsMap.put(Constants.PARAMETER_SHECDULE_TIME, p); + if (taskProps.getScheduleTime() != null) { + String dateTime = DateUtils.format(taskProps.getScheduleTime(), Constants.PARAMETER_FORMAT_TIME); + Property p = new Property(); + p.setValue(dateTime); + p.setProp(Constants.PARAMETER_SHECDULE_TIME); + paramsMap.put(Constants.PARAMETER_SHECDULE_TIME, p); + } script = ParameterUtils.convertParameterPlaceholders2(script, ParamUtils.convert(paramsMap)); } @@ -182,6 +180,4 @@ public class ShellTask extends AbstractTask { return shellParameters; } - - } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/spark/SparkTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/spark/SparkTask.java index 203c0fe146..d2a8674146 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/spark/SparkTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/spark/SparkTask.java @@ -18,11 +18,13 @@ package org.apache.dolphinscheduler.server.worker.task.spark; import org.apache.dolphinscheduler.common.enums.SparkVersion; import org.apache.dolphinscheduler.common.process.Property; +import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.spark.SparkParameters; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.apache.dolphinscheduler.dao.entity.Resource; import org.apache.dolphinscheduler.server.utils.ParamUtils; import org.apache.dolphinscheduler.server.utils.SparkArgsUtils; import org.apache.dolphinscheduler.server.worker.task.AbstractYarnTask; @@ -67,8 +69,8 @@ public class SparkTask extends AbstractYarnTask { if (!sparkParameters.checkParameters()) { throw new RuntimeException("spark task params is not valid"); } + setMainJarName(); sparkParameters.setQueue(taskProps.getQueue()); - if (StringUtils.isNotEmpty(sparkParameters.getMainArgs())) { String args = sparkParameters.getMainArgs(); @@ -115,6 +117,28 @@ public class SparkTask extends AbstractYarnTask { return command; } + @Override + protected void setMainJarName() { + // main jar + ResourceInfo mainJar = sparkParameters.getMainJar(); + if (mainJar != null) { + int resourceId = mainJar.getId(); + String resourceName; + if (resourceId == 0) { + resourceName = mainJar.getRes(); + } else { + Resource resource = processService.getResourceById(sparkParameters.getMainJar().getId()); + if (resource == null) { + logger.error("resource id: {} not exist", resourceId); + throw new RuntimeException(String.format("resource id: %d not exist", resourceId)); + } + resourceName = resource.getFullName().replaceFirst("/", ""); + } + mainJar.setRes(resourceName); + sparkParameters.setMainJar(mainJar); + } + } + @Override public AbstractParameters getParameters() { return sparkParameters; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java index 71ab56b528..7da6bd9115 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java @@ -25,6 +25,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.dolphinscheduler.alert.utils.MailUtils; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.AuthorizationType; +import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.enums.ShowType; import org.apache.dolphinscheduler.common.enums.TaskTimeoutStrategy; import org.apache.dolphinscheduler.common.enums.UdfType; @@ -140,7 +141,6 @@ public class SqlTask extends AbstractTask { dataSource.getUserId(), dataSource.getConnectionParams()); - Connection con = null; List createFuncs = null; try { // load class @@ -178,18 +178,10 @@ public class SqlTask extends AbstractTask { } // execute sql task - con = executeFuncAndSql(mainSqlBinds, preStatementSqlBinds, postStatementSqlBinds, createFuncs); + executeFuncAndSql(mainSqlBinds, preStatementSqlBinds, postStatementSqlBinds, createFuncs); } catch (Exception e) { logger.error(e.getMessage(), e); throw e; - } finally { - if (con != null) { - try { - con.close(); - } catch (SQLException e) { - logger.error(e.getMessage(),e); - } - } } } @@ -249,19 +241,19 @@ public class SqlTask extends AbstractTask { * @param preStatementsBinds pre statements binds * @param postStatementsBinds post statements binds * @param createFuncs create functions - * @return Connection */ - public Connection executeFuncAndSql(SqlBinds mainSqlBinds, + public void executeFuncAndSql(SqlBinds mainSqlBinds, List preStatementsBinds, List postStatementsBinds, List createFuncs){ Connection connection = null; try { - // if upload resource is HDFS and kerberos startup - CommonUtils.loadKerberosConf(); // if hive , load connection params if exists - if (HIVE == dataSource.getType()) { + if (DbType.HIVE == dataSource.getType() || DbType.SPARK == dataSource.getType()) { + // if upload resource is HDFS and kerberos startup + CommonUtils.loadKerberosConf(); + Properties paramProp = new Properties(); paramProp.setProperty(USER, baseDataSource.getUser()); paramProp.setProperty(PASSWORD, baseDataSource.getPassword()); @@ -307,7 +299,11 @@ public class SqlTask extends AbstractTask { while (resultSet.next()) { JSONObject mapOfColValues = new JSONObject(true); for (int i = 1; i <= num; i++) { - mapOfColValues.put(md.getColumnName(i), resultSet.getObject(i)); + if (StringUtils.isNotEmpty(md.getColumnLabel(i))) { + mapOfColValues.put(md.getColumnLabel(i), resultSet.getObject(i)); + } else { + mapOfColValues.put(md.getColumnName(i), resultSet.getObject(i)); + } } resultJSONArray.add(mapOfColValues); } @@ -343,13 +339,9 @@ public class SqlTask extends AbstractTask { logger.error(e.getMessage(),e); throw new RuntimeException(e.getMessage()); } finally { - try { - connection.close(); - } catch (Exception e) { - logger.error(e.getMessage(), e); - } + ConnectionUtils.releaseResource(connection); } - return connection; + } /** @@ -363,20 +355,20 @@ public class SqlTask extends AbstractTask { // is the timeout set boolean timeoutFlag = taskProps.getTaskTimeoutStrategy() == TaskTimeoutStrategy.FAILED || taskProps.getTaskTimeoutStrategy() == TaskTimeoutStrategy.WARNFAILED; - try (PreparedStatement stmt = connection.prepareStatement(sqlBinds.getSql())) { - if(timeoutFlag){ - stmt.setQueryTimeout(taskProps.getTaskTimeout()); - } - Map params = sqlBinds.getParamsMap(); - if(params != null) { - for (Map.Entry entry : params.entrySet()) { - Property prop = entry.getValue(); - ParameterUtils.setInParameter(entry.getKey(), stmt, prop.getType(), prop.getValue()); - } + // prepare statement + PreparedStatement stmt = connection.prepareStatement(sqlBinds.getSql()); + if(timeoutFlag){ + stmt.setQueryTimeout(taskProps.getTaskTimeout()); + } + Map params = sqlBinds.getParamsMap(); + if(params != null) { + for (Map.Entry entry : params.entrySet()) { + Property prop = entry.getValue(); + ParameterUtils.setInParameter(entry.getKey(), stmt, prop.getType(), prop.getValue()); } - logger.info("prepare statement replace sql : {} ", stmt); - return stmt; } + logger.info("prepare statement replace sql : {} ", stmt); + return stmt; } /** @@ -392,7 +384,7 @@ public class SqlTask extends AbstractTask { List users = alertDao.queryUserByAlertGroupId(instance.getWarningGroupId()); // receiving group list - List receviersList = new ArrayList(); + List receviersList = new ArrayList<>(); for(User user:users){ receviersList.add(user.getEmail().trim()); } @@ -406,7 +398,7 @@ public class SqlTask extends AbstractTask { } // copy list - List receviersCcList = new ArrayList(); + List receviersCcList = new ArrayList<>(); // Custom Copier String receiversCc = sqlParameters.getReceiversCc(); if (StringUtils.isNotEmpty(receiversCc)){ @@ -420,7 +412,7 @@ public class SqlTask extends AbstractTask { if(EnumUtils.isValidEnum(ShowType.class,showTypeName)){ Map mailResult = MailUtils.sendMails(receviersList, receviersCcList, title, content, ShowType.valueOf(showTypeName)); - if(!(Boolean) mailResult.get(STATUS)){ + if(!(boolean) mailResult.get(STATUS)){ throw new RuntimeException("send mail failed!"); } }else{ @@ -477,22 +469,7 @@ public class SqlTask extends AbstractTask { ProcessInstance processInstance = processService.findProcessInstanceByTaskId(taskProps.getTaskInstId()); int userId = processInstance.getExecutorId(); - PermissionCheck permissionCheckUdf = new PermissionCheck(AuthorizationType.UDF, processService,udfFunIds,userId,logger); + PermissionCheck permissionCheckUdf = new PermissionCheck<>(AuthorizationType.UDF, processService,udfFunIds,userId,logger); permissionCheckUdf.checkPermission(); } - - /** - * check data source permission - * @param dataSourceId data source id - * @return if has download permission return true else false - */ - private void checkDataSourcePermission(int dataSourceId) throws Exception{ - // process instance - ProcessInstance processInstance = processService.findProcessInstanceByTaskId(taskProps.getTaskInstId()); - int userId = processInstance.getExecutorId(); - - PermissionCheck permissionCheckDataSource = new PermissionCheck(AuthorizationType.DATASOURCE, processService,new Integer[]{dataSourceId},userId,logger); - permissionCheckDataSource.checkPermission(); - } - } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTask.java index 64bc7924d2..5c701dcd52 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTask.java @@ -71,6 +71,10 @@ public class SqoopTask extends AbstractYarnTask { return null; } + @Override + protected void setMainJarName() { + } + @Override public AbstractParameters getParameters() { return sqoopParameters; diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentTaskTest.java index 272fb546da..c13a7647fe 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentTaskTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentTaskTest.java @@ -17,47 +17,106 @@ package org.apache.dolphinscheduler.server.worker.task.dependent; import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.ExecutionStatus; +import org.apache.dolphinscheduler.common.model.DateInterval; +import org.apache.dolphinscheduler.common.model.TaskNode; +import org.apache.dolphinscheduler.common.utils.dependent.DependentDateUtils; +import org.apache.dolphinscheduler.dao.entity.ProcessInstance; +import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.server.worker.task.TaskProps; +import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; +import org.apache.dolphinscheduler.service.process.ProcessService; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.context.ApplicationContext; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +@RunWith(MockitoJUnitRunner.Silent.class) public class DependentTaskTest { private static final Logger logger = LoggerFactory.getLogger(DependentTaskTest.class); + private ProcessService processService; + private ApplicationContext applicationContext; - @Test - public void testDependInit() throws Exception{ - TaskProps taskProps = new TaskProps(); + @Before + public void before() throws Exception{ + processService = Mockito.mock(ProcessService.class); + Mockito.when(processService + .findLastRunningProcess(4,DependentDateUtils.getTodayInterval(new Date()).get(0))) + .thenReturn(findLastProcessInterval()); + Mockito.when(processService + .getTaskNodeListByDefinitionId(4)) + .thenReturn(getTaskNodes()); + Mockito.when(processService + .findValidTaskListByProcessId(11)) + .thenReturn(getTaskInstances()); - String dependString = "{\n" + - "\"dependTaskList\":[\n" + - " {\n" + - " \"dependItemList\":[\n" + - " {\n" + - " \"definitionId\": 101,\n" + - " \"depTasks\": \"ALL\",\n" + - " \"cycle\": \"day\",\n" + - " \"dateValue\": \"last1Day\"\n" + - " }\n" + - " ],\n" + - " \"relation\": \"AND\"\n" + - " }\n" + - " ],\n" + - "\"relation\":\"OR\"\n" + - "}"; + Mockito.when(processService + .findTaskInstanceById(252612)) + .thenReturn(getTaskInstance()); + applicationContext = Mockito.mock(ApplicationContext.class); + SpringApplicationContext springApplicationContext = new SpringApplicationContext(); + springApplicationContext.setApplicationContext(applicationContext); + Mockito.when(applicationContext.getBean(ProcessService.class)).thenReturn(processService); + } + @Test + public void test() throws Exception{ + + TaskProps taskProps = new TaskProps(); + String dependString = "{\"dependTaskList\":[{\"dependItemList\":[{\"dateValue\":\"today\",\"depTasks\":\"ALL\",\"projectId\":1,\"definitionList\":[{\"label\":\"C\",\"value\":4},{\"label\":\"B\",\"value\":3},{\"label\":\"A\",\"value\":2}],\"cycle\":\"day\",\"definitionId\":4}],\"relation\":\"AND\"}],\"relation\":\"AND\"}"; taskProps.setTaskInstId(252612); taskProps.setDependence(dependString); + taskProps.setTaskStartTime(new Date()); DependentTask dependentTask = new DependentTask(taskProps, logger); dependentTask.init(); dependentTask.handle(); - Assert.assertEquals(dependentTask.getExitStatusCode(), Constants.EXIT_CODE_FAILURE ); + Assert.assertEquals(dependentTask.getExitStatusCode(), Constants.EXIT_CODE_SUCCESS ); } + private ProcessInstance findLastProcessInterval(){ + ProcessInstance processInstance = new ProcessInstance(); + processInstance.setId(11); + processInstance.setState(ExecutionStatus.SUCCESS); + return processInstance; + } + + private List getTaskNodes(){ + List list = new ArrayList<>(); + TaskNode taskNode = new TaskNode(); + taskNode.setName("C"); + taskNode.setType("SQL"); + list.add(taskNode); + return list; + } + private List getTaskInstances(){ + List list = new ArrayList<>(); + TaskInstance taskInstance = new TaskInstance(); + taskInstance.setName("C"); + taskInstance.setState(ExecutionStatus.SUCCESS); + taskInstance.setDependency("1231"); + list.add(taskInstance); + return list; + } + + private TaskInstance getTaskInstance(){ + TaskInstance taskInstance = new TaskInstance(); + taskInstance.setId(252612); + taskInstance.setName("C"); + taskInstance.setState(ExecutionStatus.SUCCESS); + return taskInstance; + } } \ No newline at end of file diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTaskTest.java index ebe90147d1..387c7c5e53 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTaskTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTaskTest.java @@ -25,13 +25,10 @@ import org.apache.dolphinscheduler.server.worker.task.ShellCommandExecutor; import org.apache.dolphinscheduler.server.worker.task.TaskProps; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.process.ProcessService; -import org.junit.After; -import org.junit.Assert; -import org.junit.Assume; -import org.junit.Before; -import org.junit.Test; +import org.junit.*; import org.junit.runner.RunWith; import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; import org.slf4j.Logger; @@ -45,6 +42,7 @@ import java.util.Date; */ @RunWith(PowerMockRunner.class) @PrepareForTest(OSUtils.class) +@PowerMockIgnore({"javax.management.*"}) public class ShellTaskTest { private static final Logger logger = LoggerFactory.getLogger(ShellTaskTest.class); @@ -136,6 +134,28 @@ public class ShellTaskTest { } } + @Test + public void testInitException() { + TaskProps props = new TaskProps(); + props.setTaskDir("/tmp"); + props.setTaskAppId(String.valueOf(System.currentTimeMillis())); + props.setTaskInstId(1); + props.setTenantCode("1"); + props.setEnvFile(".dolphinscheduler_env.sh"); + props.setTaskStartTime(new Date()); + props.setTaskTimeout(0); + props.setTaskParams("{\"rawScript\": \"\"}"); + ShellTask shellTask = new ShellTask(props, logger); + try { + shellTask.init(); + } catch (Exception e) { + logger.info(e.getMessage(), e); + if (e.getMessage().contains("shell task params is not valid")) { + Assert.assertTrue(true); + } + } + } + /** * Method: init for Windows */ @@ -157,7 +177,20 @@ public class ShellTaskTest { public void testHandleForUnix() throws Exception { try { PowerMockito.when(OSUtils.isWindows()).thenReturn(false); - shellTask.handle(); + TaskProps props = new TaskProps(); + props.setTaskDir("/tmp"); + props.setTaskAppId(String.valueOf(System.currentTimeMillis())); + props.setTaskInstId(1); + props.setTenantCode("1"); + props.setEnvFile(".dolphinscheduler_env.sh"); + props.setTaskStartTime(new Date()); + props.setTaskTimeout(0); + props.setScheduleTime(new Date()); + props.setCmdTypeIfComplement(CommandType.START_PROCESS); + props.setTaskParams("{\"rawScript\": \" echo ${test}\", \"localParams\": [{\"prop\":\"test\", \"direct\":\"IN\", \"type\":\"VARCHAR\", \"value\":\"123\"}]}"); + ShellTask shellTask1 = new ShellTask(props, logger); + shellTask1.init(); + shellTask1.handle(); Assert.assertTrue(true); } catch (Error | Exception e) { if (!e.getMessage().contains("process error . exitCode is : -1") @@ -174,7 +207,20 @@ public class ShellTaskTest { public void testHandleForWindows() throws Exception { try { Assume.assumeTrue(OSUtils.isWindows()); - shellTask.handle(); + TaskProps props = new TaskProps(); + props.setTaskDir("/tmp"); + props.setTaskAppId(String.valueOf(System.currentTimeMillis())); + props.setTaskInstId(1); + props.setTenantCode("1"); + props.setEnvFile(".dolphinscheduler_env.sh"); + props.setTaskStartTime(new Date()); + props.setTaskTimeout(0); + props.setScheduleTime(new Date()); + props.setCmdTypeIfComplement(CommandType.START_PROCESS); + props.setTaskParams("{\"rawScript\": \" echo ${test}\", \"localParams\": [{\"prop\":\"test\", \"direct\":\"IN\", \"type\":\"VARCHAR\", \"value\":\"123\"}]}"); + ShellTask shellTask1 = new ShellTask(props, logger); + shellTask1.init(); + shellTask1.handle(); Assert.assertTrue(true); } catch (Error | Exception e) { if (!e.getMessage().contains("process error . exitCode is : -1")) { diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/spark/SparkTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/spark/SparkTaskTest.java index a18e0b2a9d..f0bcd9ec27 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/spark/SparkTaskTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/spark/SparkTaskTest.java @@ -135,7 +135,7 @@ public class SparkTaskTest { logger.info("spark task command : {}", sparkArgs); - Assert.assertEquals(sparkArgs.split(" ")[0], SPARK2_COMMAND ); + Assert.assertEquals(SPARK2_COMMAND, sparkArgs.split(" ")[0]); } } diff --git a/dolphinscheduler-service/pom.xml b/dolphinscheduler-service/pom.xml index 03bb94dc29..c150e834b9 100644 --- a/dolphinscheduler-service/pom.xml +++ b/dolphinscheduler-service/pom.xml @@ -1,5 +1,20 @@ - + diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/permission/PermissionCheck.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/permission/PermissionCheck.java index e53fae6e86..9f93f4ce3e 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/permission/PermissionCheck.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/permission/PermissionCheck.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.service.permission; import org.apache.dolphinscheduler.common.enums.AuthorizationType; import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.service.process.ProcessService; @@ -45,6 +46,11 @@ public class PermissionCheck { */ private T[] needChecks; + /** + * resoruce info + */ + private List resourceList; + /** * user id */ @@ -90,6 +96,22 @@ public class PermissionCheck { this.logger = logger; } + /** + * permission check + * @param logger + * @param authorizationType + * @param processService + * @param resourceList + * @param userId + */ + public PermissionCheck(AuthorizationType authorizationType, ProcessService processService, List resourceList, int userId,Logger logger) { + this.authorizationType = authorizationType; + this.processService = processService; + this.resourceList = resourceList; + this.userId = userId; + this.logger = logger; + } + public AuthorizationType getAuthorizationType() { return authorizationType; } @@ -122,6 +144,14 @@ public class PermissionCheck { this.userId = userId; } + public List getResourceList() { + return resourceList; + } + + public void setResourceList(List resourceList) { + this.resourceList = resourceList; + } + /** * has permission * @return true if has permission @@ -141,6 +171,7 @@ public class PermissionCheck { */ public void checkPermission() throws Exception{ if(this.needChecks.length > 0){ + // get user type in order to judge whether the user is admin User user = processService.getUserById(userId); if (user.getUserType() != UserType.ADMIN_USER){ diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java index c848ec5197..ce4424e24a 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java @@ -236,6 +236,30 @@ public class ProcessService { return processInstanceMapper.queryDetailById(processId); } + /** + * get task node list by definitionId + * @param defineId + * @return + */ + public List getTaskNodeListByDefinitionId(Integer defineId){ + ProcessDefinition processDefinition = processDefineMapper.selectById(defineId); + if (processDefinition == null) { + logger.info("process define not exists"); + return null; + } + + String processDefinitionJson = processDefinition.getProcessDefinitionJson(); + ProcessData processData = JSONUtils.parseObject(processDefinitionJson, ProcessData.class); + + //process data check + if (null == processData) { + logger.error("process data is null"); + return null; + } + + return processData.getTasks(); + } + /** * find process instance by id * @param processId processId @@ -1556,10 +1580,11 @@ public class ProcessService { /** * find tenant code by resource name * @param resName resource name + * @param resourceType resource type * @return tenant code */ - public String queryTenantCodeByResName(String resName){ - return resourceMapper.queryTenantCodeByResourceName(resName); + public String queryTenantCodeByResName(String resName,ResourceType resourceType){ + return resourceMapper.queryTenantCodeByResourceName(resName,resourceType.ordinal()); } /** @@ -1791,10 +1816,18 @@ public class ProcessService { Set originResSet = new HashSet(Arrays.asList(needChecks)); switch (authorizationType){ - case RESOURCE_FILE: - Set authorizedResources = resourceMapper.listAuthorizedResource(userId, needChecks).stream().map(t -> t.getAlias()).collect(toSet()); + case RESOURCE_FILE_ID: + Set authorizedResourceFiles = resourceMapper.listAuthorizedResourceById(userId, needChecks).stream().map(t -> t.getId()).collect(toSet()); + originResSet.removeAll(authorizedResourceFiles); + break; + case RESOURCE_FILE_NAME: + Set authorizedResources = resourceMapper.listAuthorizedResource(userId, needChecks).stream().map(t -> t.getFullName()).collect(toSet()); originResSet.removeAll(authorizedResources); break; + case UDF_FILE: + Set authorizedUdfFiles = resourceMapper.listAuthorizedResourceById(userId, needChecks).stream().map(t -> t.getId()).collect(toSet()); + originResSet.removeAll(authorizedUdfFiles); + break; case DATASOURCE: Set authorizedDatasources = dataSourceMapper.listAuthorizedDataSource(userId,needChecks).stream().map(t -> t.getId()).collect(toSet()); originResSet.removeAll(authorizedDatasources); @@ -1820,5 +1853,14 @@ public class ProcessService { return userMapper.queryDetailsById(userId); } + /** + * get resource by resoruce id + * @param resoruceId resource id + * @return Resource + */ + public Resource getResourceById(int resoruceId){ + return resourceMapper.selectById(resoruceId); + } + } diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java index 60cdb1dd97..30e7c52b19 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java @@ -70,9 +70,10 @@ public class QuartzExecutors { synchronized (QuartzExecutors.class) { // when more than two threads run into the first null check same time, to avoid instanced more than one time, it needs to be checked again. if (INSTANCE == null) { - INSTANCE = new QuartzExecutors(); + QuartzExecutors quartzExecutors = new QuartzExecutors(); //finish QuartzExecutors init - INSTANCE.init(); + quartzExecutors.init(); + INSTANCE = quartzExecutors; } } } diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskQueueZkImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskQueueZkImpl.java index 9c1d318ea5..5ac3ece663 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskQueueZkImpl.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskQueueZkImpl.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.service.queue; import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.IpUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.service.zk.ZookeeperOperator; @@ -67,8 +68,7 @@ public class TaskQueueZkImpl implements ITaskQueue { @Override public List getAllTasks(String key) { try { - List list = zookeeperOperator.getChildrenKeys(getTasksPath(key)); - return list; + return zookeeperOperator.getChildrenKeys(getTasksPath(key)); } catch (Exception e) { logger.error("get all tasks from tasks queue exception",e); } @@ -141,7 +141,7 @@ public class TaskQueueZkImpl implements ITaskQueue { try{ List list = zookeeperOperator.getChildrenKeys(getTasksPath(key)); - if(list != null && list.size() > 0){ + if(CollectionUtils.isNotEmpty(list)){ String workerIp = OSUtils.getHost(); String workerIpLongStr = String.valueOf(IpUtils.ipToLong(workerIp)); diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/TaskQueueZKImplTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/TaskQueueZKImplTest.java index 5d464ac3c9..a630c494c5 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/TaskQueueZKImplTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/TaskQueueZKImplTest.java @@ -58,11 +58,11 @@ public class TaskQueueZKImplTest extends BaseTaskQueueTest { init(); // get all List allTasks = tasksQueue.getAllTasks(Constants.DOLPHINSCHEDULER_TASKS_QUEUE); - assertEquals(allTasks.size(),2); + assertEquals(2, allTasks.size()); //delete all tasksQueue.delete(); allTasks = tasksQueue.getAllTasks(Constants.DOLPHINSCHEDULER_TASKS_QUEUE); - assertEquals(allTasks.size(),0); + assertEquals(0, allTasks.size()); } @Test public void hasTask(){ @@ -126,10 +126,10 @@ public class TaskQueueZKImplTest extends BaseTaskQueueTest { //add init(); List taskList = tasksQueue.poll(Constants.DOLPHINSCHEDULER_TASKS_QUEUE, 2); - assertEquals(taskList.size(),2); + assertEquals(2, taskList.size()); - assertEquals(taskList.get(0),"0_1_1_1_-1"); - assertEquals(taskList.get(1),"1_0_1_1_-1"); + assertEquals("0_1_1_1_-1", taskList.get(0)); + assertEquals("1_0_1_1_-1", taskList.get(1)); } /** @@ -153,7 +153,7 @@ public class TaskQueueZKImplTest extends BaseTaskQueueTest { String task = "1_0_1_1_-1"; tasksQueue.sadd(Constants.DOLPHINSCHEDULER_TASKS_QUEUE,task); //check size - assertEquals(tasksQueue.smembers(Constants.DOLPHINSCHEDULER_TASKS_QUEUE).size(),1); + assertEquals(1, tasksQueue.smembers(Constants.DOLPHINSCHEDULER_TASKS_QUEUE).size()); } @@ -166,10 +166,10 @@ public class TaskQueueZKImplTest extends BaseTaskQueueTest { String task = "1_0_1_1_-1"; tasksQueue.sadd(Constants.DOLPHINSCHEDULER_TASKS_QUEUE,task); //check size - assertEquals(tasksQueue.smembers(Constants.DOLPHINSCHEDULER_TASKS_QUEUE).size(),1); + assertEquals(1, tasksQueue.smembers(Constants.DOLPHINSCHEDULER_TASKS_QUEUE).size()); //remove and get size tasksQueue.srem(Constants.DOLPHINSCHEDULER_TASKS_QUEUE,task); - assertEquals(tasksQueue.smembers(Constants.DOLPHINSCHEDULER_TASKS_QUEUE).size(),0); + assertEquals(0, tasksQueue.smembers(Constants.DOLPHINSCHEDULER_TASKS_QUEUE).size()); } /** @@ -179,17 +179,17 @@ public class TaskQueueZKImplTest extends BaseTaskQueueTest { public void smembers(){ //first init - assertEquals(tasksQueue.smembers(Constants.DOLPHINSCHEDULER_TASKS_QUEUE).size(),0); + assertEquals(0, tasksQueue.smembers(Constants.DOLPHINSCHEDULER_TASKS_QUEUE).size()); //add String task = "1_0_1_1_-1"; tasksQueue.sadd(Constants.DOLPHINSCHEDULER_TASKS_QUEUE,task); //check size - assertEquals(tasksQueue.smembers(Constants.DOLPHINSCHEDULER_TASKS_QUEUE).size(),1); + assertEquals(1, tasksQueue.smembers(Constants.DOLPHINSCHEDULER_TASKS_QUEUE).size()); //add task = "0_1_1_1_"; tasksQueue.sadd(Constants.DOLPHINSCHEDULER_TASKS_QUEUE,task); //check size - assertEquals(tasksQueue.smembers(Constants.DOLPHINSCHEDULER_TASKS_QUEUE).size(),2); + assertEquals(2, tasksQueue.smembers(Constants.DOLPHINSCHEDULER_TASKS_QUEUE).size()); } @@ -222,7 +222,7 @@ public class TaskQueueZKImplTest extends BaseTaskQueueTest { } String node1 = tasksQueue.poll(Constants.DOLPHINSCHEDULER_TASKS_QUEUE, 1).get(0); - assertEquals(node1,"0"); + assertEquals("0", node1); } diff --git a/dolphinscheduler-ui/package.json b/dolphinscheduler-ui/package.json index da15b722fc..b23969803b 100644 --- a/dolphinscheduler-ui/package.json +++ b/dolphinscheduler-ui/package.json @@ -11,7 +11,8 @@ "build:release": "npm run clean && cross-env NODE_ENV=production PUBLIC_PATH=/dolphinscheduler/ui webpack --config ./build/webpack.config.release.js" }, "dependencies": { - "ans-ui": "1.1.7", + "@riophae/vue-treeselect": "^0.4.0", + "ans-ui": "1.1.9", "axios": "^0.16.2", "bootstrap": "3.3.7", "canvg": "1.5.1", @@ -26,6 +27,7 @@ "js-cookie": "^2.2.1", "jsplumb": "^2.8.6", "lodash": "^4.17.11", + "normalize.css": "^8.0.1", "vue": "^2.5.17", "vue-router": "2.7.0", "vuex": "^3.0.0", diff --git a/dolphinscheduler-ui/src/js/conf/home/index.js b/dolphinscheduler-ui/src/js/conf/home/index.js index 33fc63d8b0..1913088eca 100644 --- a/dolphinscheduler-ui/src/js/conf/home/index.js +++ b/dolphinscheduler-ui/src/js/conf/home/index.js @@ -31,6 +31,7 @@ import Permissions from '@/module/permissions' import 'ans-ui/lib/ans-ui.min.css' import ans from 'ans-ui/lib/ans-ui.min' import en_US from 'ans-ui/lib/locale/en' // eslint-disable-line +import'normalize.css/normalize.css' import 'sass/conf/home/index.scss' import'bootstrap/dist/css/bootstrap.min.css' diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss index 6d97856960..9973750d98 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss @@ -135,7 +135,7 @@ width: 36px; height: 36px; float: left; - margin-bottom: 11px; + margin-bottom: 8px; border-radius: 3px; .disabled { .icos { diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue index a1ccd39260..8628fdb8ef 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue @@ -326,45 +326,62 @@ * Storage interface */ _save (sourceType) { - return new Promise((resolve, reject) => { - this.spinnerLoading = true - // Storage store - Dag.saveStore().then(res => { - if (this.urlParam.id) { - /** - * Edit - * @param saveInstanceEditDAGChart => Process instance editing - * @param saveEditDAGChart => Process definition editing - */ - this[this.type === 'instance' ? 'updateInstance' : 'updateDefinition'](this.urlParam.id).then(res => { - this.$message.success(res.msg) - this.spinnerLoading = false - resolve() - }).catch(e => { - this.$message.error(e.msg || '') - this.spinnerLoading = false - reject(e) - }) - } else { - // New - this.saveDAGchart().then(res => { - this.$message.success(res.msg) - this.spinnerLoading = false - // source @/conf/home/pages/dag/_source/editAffirmModel/index.js - if (sourceType !== 'affirm') { - // Jump process definition - this.$router.push({ name: 'projects-definition-list' }) - } - resolve() - }).catch(e => { - this.$message.error(e.msg || '') - this.setName('') - this.spinnerLoading = false - reject(e) - }) - } + if(this._verifConditions()) { + return new Promise((resolve, reject) => { + this.spinnerLoading = true + // Storage store + Dag.saveStore().then(res => { + if (this.urlParam.id) { + /** + * Edit + * @param saveInstanceEditDAGChart => Process instance editing + * @param saveEditDAGChart => Process definition editing + */ + this[this.type === 'instance' ? 'updateInstance' : 'updateDefinition'](this.urlParam.id).then(res => { + this.$message.success(res.msg) + this.spinnerLoading = false + resolve() + }).catch(e => { + this.$message.error(e.msg || '') + this.spinnerLoading = false + reject(e) + }) + } else { + // New + this.saveDAGchart().then(res => { + this.$message.success(res.msg) + this.spinnerLoading = false + // source @/conf/home/pages/dag/_source/editAffirmModel/index.js + if (sourceType !== 'affirm') { + // Jump process definition + this.$router.push({ name: 'projects-definition-list' }) + } + resolve() + }).catch(e => { + this.$message.error(e.msg || '') + this.setName('') + this.spinnerLoading = false + reject(e) + }) + } + }) }) + } + }, + _verifConditions () { + let tasks = this.$store.state.dag.tasks + let bool = true + tasks.map(v=>{ + if(v.type == 'CONDITIONS' && (v.conditionResult.successNode[0] =='' || v.conditionResult.successNode[0] == null || v.conditionResult.failedNode[0] =='' || v.conditionResult.failedNode[0] == null)) { + bool = false + return false + } }) + if(!bool) { + this.$message.warning(`${i18n.$t('Successful branch flow and failed branch flow are required')}`) + return false + } + return true }, /** * Global parameter diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue index cc1c8b6d6c..017f154b44 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue @@ -430,7 +430,7 @@ * return params */ _onParams (o) { - this.params = Object.assign(this.params, {}, o) + this.params = Object.assign({}, o) }, _onCacheParams (o) { @@ -470,7 +470,7 @@ this.$message.warning(`${i18n.$t('Please enter name (required)')}`) return false } - if (this.successBranch !='' && this.successBranch == this.failedBranch) { + if (this.successBranch !='' && this.successBranch !=null && this.successBranch == this.failedBranch) { this.$message.warning(`${i18n.$t('Cannot select the same node for successful branch flow and failed branch flow')}`) return false } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/nodeStatus.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/nodeStatus.vue index fa7ee89e98..0c3f7433a3 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/nodeStatus.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/nodeStatus.vue @@ -128,10 +128,6 @@ this.store.dispatch('dag/getProcessTasksList', { processDefinitionId: ids }).then(res => { resolve(['ALL'].concat(_.map(res, v => v.name))) }) - } else { - this.store.dispatch('dag/getTaskListDefIdAll', { processDefinitionIdList: ids }).then(res => { - resolve(res) - }) } }) }, diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/datax.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/datax.vue index 959610f95a..f1c9b757bd 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/datax.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/datax.vue @@ -17,90 +17,127 @@ diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue index 706a35f4fe..121147d7e5 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue @@ -44,19 +44,9 @@
{{$t('Main jar package')}}
- - - - + +
{{ node.raw.fullName }}
+
@@ -88,12 +78,9 @@
{{$t('Resources')}}
- - + +
{{ node.raw.fullName }}
+
@@ -115,6 +102,8 @@ import mListBox from './_source/listBox' import mResources from './_source/resources' import mLocalParams from './_source/localParams' + import Treeselect from '@riophae/vue-treeselect' + import '@riophae/vue-treeselect/dist/vue-treeselect.css' import disabledState from '@/module/mixin/disabledState' export default { name: 'mr', @@ -125,6 +114,7 @@ // Master jar package mainJar: null, // Main jar package (List) + mainJarLists: [], mainJarList: [], // Resource(list) resourceList: [], @@ -139,7 +129,12 @@ // Program type programType: 'JAVA', // Program type(List) - programTypeList: [{ code: 'JAVA' }, { code: 'PYTHON' }] + programTypeList: [{ code: 'JAVA' }, { code: 'PYTHON' }], + normalizer(node) { + return { + label: node.name + } + } } }, props: { @@ -147,6 +142,19 @@ }, mixins: [disabledState], methods: { + /** + * getResourceId + */ + marjarId(name) { + this.store.dispatch('dag/getResourceId',{ + type: 'FILE', + fullName: '/'+name + }).then(res => { + this.mainJar = res.id + }).catch(e => { + this.$message.error(e.msg || '') + }) + }, /** * return localParams */ @@ -165,6 +173,12 @@ _onCacheResourcesData (a) { this.cacheResourceList = a }, + diGuiTree(item) { // Recursive convenience tree structure + item.forEach(item => { + item.children === '' || item.children === undefined || item.children === null || item.children.length === 0?　　　　　　　　 + delete item.children : this.diGuiTree(item.children); + }) + }, /** * verification */ @@ -179,22 +193,19 @@ return false } - if (!this.$refs.refResources._verifResources()) { - return false - } - // localParams Subcomponent verification if (!this.$refs.refLocalParams._verifProp()) { return false } - // storage this.$emit('on-params', { mainClass: this.mainClass, mainJar: { - res: this.mainJar + id: this.mainJar }, - resourceList: this.resourceList, + resourceList: _.map(this.resourceList, v => { + return {id: v} + }), localParams: this.localParams, mainArgs: this.mainArgs, others: this.others, @@ -202,24 +213,7 @@ }) return true }, - /** - * Get resource data - */ - _getResourcesList () { - return new Promise((resolve, reject) => { - let isJar = (alias) => { - return alias.substring(alias.lastIndexOf('.') + 1, alias.length) !== 'jar' - } - this.mainJarList = _.map(_.cloneDeep(this.store.state.dag.resourcesListS), v => { - return { - id: v.id, - code: v.alias, - disabled: isJar(v.alias) - } - }) - resolve() - }) - } + }, watch: { /** @@ -240,9 +234,11 @@ return { mainClass: this.mainClass, mainJar: { - res: this.mainJar + id: this.mainJar }, - resourceList: this.cacheResourceList, + resourceList: _.map(this.resourceList, v => { + return {id: v} + }), localParams: this.localParams, mainArgs: this.mainArgs, others: this.others, @@ -251,13 +247,24 @@ } }, created () { - this._getResourcesList().then(() => { + let item = this.store.state.dag.resourcesListS + let items = this.store.state.dag.resourcesListJar + this.diGuiTree(item) + this.diGuiTree(items) + this.mainJarList = item + this.mainJarLists = items let o = this.backfillItem // Non-null objects represent backfill if (!_.isEmpty(o)) { this.mainClass = o.params.mainClass || '' - this.mainJar = o.params.mainJar.res || '' + if(o.params.mainJar.res) { + this.marjarId(o.params.mainJar.res) + } else if(o.params.mainJar.res=='') { + this.mainJar = '' + } else { + this.mainJar = o.params.mainJar.id || '' + } this.mainArgs = o.params.mainArgs || '' this.others = o.params.others this.programType = o.params.programType || 'JAVA' @@ -265,7 +272,20 @@ // backfill resourceList let resourceList = o.params.resourceList || [] if (resourceList.length) { - this.resourceList = resourceList + _.map(resourceList, v => { + if(v.res) { + this.store.dispatch('dag/getResourceId',{ + type: 'FILE', + fullName: '/'+v.res + }).then(res => { + this.resourceList.push(res.id) + }).catch(e => { + this.$message.error(e.msg || '') + }) + } else { + this.resourceList.push(v.id) + } + }) this.cacheResourceList = resourceList } @@ -275,12 +295,11 @@ this.localParams = localParams } } - }) }, mounted () { }, - components: { mLocalParams, mListBox, mResources } + components: { mLocalParams, mListBox, mResources, Treeselect } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue index 6f495d22a0..28fded41d3 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue @@ -108,10 +108,6 @@ return false } - if (!this.$refs.refResources._verifResources()) { - return false - } - // localParams Subcomponent verification if (!this.$refs.refLocalParams._verifProp()) { return false @@ -119,7 +115,9 @@ // storage this.$emit('on-params', { - resourceList: this.resourceList, + resourceList: _.map(this.resourceList, v => { + return {id: v} + }), localParams: this.localParams, rawScript: editor.getValue() }) @@ -129,8 +127,6 @@ * Processing code highlighting */ _handlerEditor () { - this._destroyEditor() - // editor editor = codemirror('code-python-mirror', { mode: 'python', @@ -145,44 +141,26 @@ } } - this.changes = () => { - this._cacheParams() - } - // Monitor keyboard editor.on('keypress', this.keypress) - editor.on('changes', this.changes) - editor.setValue(this.rawScript) return editor - }, - _cacheParams () { - this.$emit('on-cache-params', { - resourceList: this.cacheResourceList, - localParams: this.localParams, - rawScript: editor ? editor.getValue() : '' - }); - }, - _destroyEditor () { - if (editor) { - editor.toTextArea() // Uninstall - editor.off($('.code-python-mirror'), 'keypress', this.keypress) - editor.off($('.code-python-mirror'), 'changes', this.changes) - } } }, watch: { //Watch the cacheParams cacheParams (val) { - this._cacheParams() + this.$emit('on-cache-params', val); } }, computed: { cacheParams () { return { - resourceList: this.cacheResourceList, + resourceList: _.map(this.resourceList, v => { + return {id: v} + }), localParams: this.localParams } } @@ -197,7 +175,20 @@ // backfill resourceList let resourceList = o.params.resourceList || [] if (resourceList.length) { - this.resourceList = resourceList + _.map(resourceList, v => { + if(v.res) { + this.store.dispatch('dag/getResourceId',{ + type: 'FILE', + fullName: '/'+v.res + }).then(res => { + this.resourceList.push(res.id) + }).catch(e => { + this.$message.error(e.msg || '') + }) + } else { + this.resourceList.push(v.id) + } + }) this.cacheResourceList = resourceList } @@ -214,11 +205,8 @@ }, 200) }, destroyed () { - if (editor) { - editor.toTextArea() // Uninstall - editor.off($('.code-python-mirror'), 'keypress', this.keypress) - editor.off($('.code-python-mirror'), 'changes', this.changes) - } + editor.toTextArea() // Uninstall + editor.off($('.code-python-mirror'), 'keypress', this.keypress) }, components: { mLocalParams, mListBox, mResources } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue index a4b20f3310..bee095acd5 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue @@ -32,6 +32,14 @@ +
{{$t('Resources')}}
+
+ +
{{ node.raw.fullName }}
+ +
+ +
{{$t('Custom Parameters')}}
@@ -63,6 +71,8 @@ import mResources from './_source/resources' import mLocalParams from './_source/localParams' import disabledState from '@/module/mixin/disabledState' + import Treeselect from '@riophae/vue-treeselect' + import '@riophae/vue-treeselect/dist/vue-treeselect.css' import codemirror from '@/conf/home/pages/resource/pages/file/pages/_source/codemirror' let editor @@ -78,7 +88,14 @@ // resource(list) resourceList: [], // Cache ResourceList - cacheResourceList: [] + cacheResourceList: [], + // define options + options: [], + normalizer(node) { + return { + label: node.name + } + } } }, mixins: [disabledState], @@ -143,17 +160,19 @@ return false } - if (!this.$refs.refResources._verifResources()) { - return false - } - // localParams Subcomponent verification if (!this.$refs.refLocalParams._verifProp()) { return false } + // Process resourcelist + let dataProcessing= _.map(this.resourceList, v => { + return { + id: v + } + }) // storage this.$emit('on-params', { - resourceList: this.resourceList, + resourceList: dataProcessing, localParams: this.localParams, rawScript: editor.getValue() }) @@ -163,8 +182,6 @@ * Processing code highlighting */ _handlerEditor () { - this._destroyEditor() - // editor editor = codemirror('code-shell-mirror', { mode: 'shell', @@ -179,51 +196,40 @@ } } - this.changes = () => { - this._cacheParams() - } - // Monitor keyboard editor.on('keypress', this.keypress) - - editor.on('changes', this.changes) - editor.setValue(this.rawScript) return editor }, - _cacheParams () { - this.$emit('on-cache-params', { - resourceList: this.cacheResourceList, - localParams: this.localParams, - rawScript: editor ? editor.getValue() : '' - }); - }, - _destroyEditor () { - if (editor) { - editor.toTextArea() // Uninstall - editor.off($('.code-sql-mirror'), 'keypress', this.keypress) - editor.off($('.code-sql-mirror'), 'changes', this.changes) - } + diGuiTree(item) { // Recursive convenience tree structure + item.forEach(item => { + item.children === '' || item.children === undefined || item.children === null || item.children.length === 0?　　　　　　　　 + delete item.children : this.diGuiTree(item.children); + }) } }, watch: { //Watch the cacheParams cacheParams (val) { - this._cacheParams() + this.$emit('on-cache-params', val); } }, computed: { cacheParams () { return { - resourceList: this.cacheResourceList, + resourceList: _.map(this.resourceList, v => { + return {id: v} + }), localParams: this.localParams } } }, created () { + let item = this.store.state.dag.resourcesListS + this.diGuiTree(item) + this.options = item let o = this.backfillItem - // Non-null objects represent backfill if (!_.isEmpty(o)) { this.rawScript = o.params.rawScript || '' @@ -231,10 +237,22 @@ // backfill resourceList let resourceList = o.params.resourceList || [] if (resourceList.length) { - this.resourceList = resourceList + _.map(resourceList, v => { + if(v.res) { + this.store.dispatch('dag/getResourceId',{ + type: 'FILE', + fullName: '/'+v.res + }).then(res => { + this.resourceList.push(res.id) + }).catch(e => { + this.$message.error(e.msg || '') + }) + } else { + this.resourceList.push(v.id) + } + }) this.cacheResourceList = resourceList } - // backfill localParams let localParams = o.params.localParams || [] if (localParams.length) { @@ -251,10 +269,9 @@ if (editor) { editor.toTextArea() // Uninstall editor.off($('.code-shell-mirror'), 'keypress', this.keypress) - editor.off($('.code-shell-mirror'), 'changes', this.changes) } }, - components: { mLocalParams, mListBox, mResources, mScriptBox } + components: { mLocalParams, mListBox, mResources, mScriptBox, Treeselect } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue index 57ae6bd685..513b8ec6dd 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue @@ -22,13 +22,13 @@ - + {{$t('#')}} - + {{$t('Process Name')}} - + {{$t('Executor')}} @@ -71,7 +71,7 @@ {{item.name}} - {{item.executorName}} + {{item.executorName}} - {{_rtRunningType(item.commandType)}} diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/_source/common.js b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/_source/common.js old mode 100644 new mode 100755 diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/create/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/create/index.vue index df84f0f292..fac37a46fb 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/create/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/create/index.vue @@ -96,6 +96,8 @@ description: '', fileTypeList: filtTypeArr, content: '', + pid: -1, + currentDir: '/', spinnerLoading: false } }, @@ -107,6 +109,8 @@ this.spinnerLoading = true this.createResourceFile({ type: 'FILE', + pid: this.pid, + currentDir: this.currentDir, fileName: this.fileName, suffix: this.suffix, description: this.description, @@ -136,6 +140,7 @@ this.$message.warning(`${i18n.$t('Resource content cannot exceed 3000 lines')}`) return false } + return true }, /** diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/createFolder/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/createFolder/index.vue new file mode 100644 index 0000000000..be53033642 --- /dev/null +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/createFolder/index.vue @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + + diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/createUdfFolder/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/createUdfFolder/index.vue new file mode 100755 index 0000000000..2511452269 --- /dev/null +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/createUdfFolder/index.vue @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + + diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/details/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/details/index.vue index e961d8b1ee..6875cd4b2e 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/details/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/details/index.vue @@ -21,7 +21,7 @@

{{name}}
- + {{size}}

diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/edit/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/edit/index.vue index a0d1d7d187..0290af0988 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/edit/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/edit/index.vue @@ -44,8 +44,8 @@ + \ No newline at end of file diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/list/_source/rename.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/list/_source/rename.vue index b082f883fb..f7639bb959 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/list/_source/rename.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/list/_source/rename.vue @@ -47,9 +47,9 @@ + + diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/subFileFolder/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/subFileFolder/index.vue new file mode 100755 index 0000000000..9f903a127b --- /dev/null +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/subFileFolder/index.vue @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + + diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/subdirectory/_source/list.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/subdirectory/_source/list.vue new file mode 100755 index 0000000000..f5e801a205 --- /dev/null +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/subdirectory/_source/list.vue @@ -0,0 +1,251 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/subdirectory/_source/rename.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/subdirectory/_source/rename.vue new file mode 100755 index 0000000000..6f7dacae89 --- /dev/null +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/subdirectory/_source/rename.vue @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + \ No newline at end of file diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/subdirectory/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/subdirectory/index.vue new file mode 100755 index 0000000000..12be6b0bc8 --- /dev/null +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/subdirectory/index.vue @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/createUdfFolder/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/createUdfFolder/index.vue new file mode 100755 index 0000000000..c707ce8c90 --- /dev/null +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/createUdfFolder/index.vue @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + + diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/function/_source/createUdf.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/function/_source/createUdf.vue index 01d8d22650..1408c552db 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/function/_source/createUdf.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/function/_source/createUdf.vue @@ -15,7 +15,7 @@ * limitations under the License. */