Browse Source
* [cherry-pick][python] Make it work to 2.0.2 * Remove unused ProcessExecutionTypeEnum * Add queryByName to project * Add checkTenantExists to tenant * Add queryByTenantCode to tenant * Add queryQueueName to queue * Add all content from dev branch * Add gitignore * Add pydolphinscheduler content * Add ds-py to bin test * Py merge to 202 * Fix version * Fix missing variable * Add py4j as known deps * Fix core database bug2.0.2-release
Jiajie Zhong
2 years ago
committed by
GitHub
92 changed files with 7153 additions and 39 deletions
@ -0,0 +1,60 @@ |
|||||||
|
<?xml version="1.0" encoding="UTF-8"?> |
||||||
|
<!-- |
||||||
|
~ Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
~ contributor license agreements. See the NOTICE file distributed with |
||||||
|
~ this work for additional information regarding copyright ownership. |
||||||
|
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
~ (the "License"); you may not use this file except in compliance with |
||||||
|
~ the License. You may obtain a copy of the License at |
||||||
|
~ |
||||||
|
~ http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
~ |
||||||
|
~ Unless required by applicable law or agreed to in writing, software |
||||||
|
~ distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
~ See the License for the specific language governing permissions and |
||||||
|
~ limitations under the License. |
||||||
|
--> |
||||||
|
|
||||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" |
||||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> |
||||||
|
<modelVersion>4.0.0</modelVersion> |
||||||
|
<parent> |
||||||
|
<groupId>org.apache.dolphinscheduler</groupId> |
||||||
|
<artifactId>dolphinscheduler</artifactId> |
||||||
|
<version>2.0.2-SNAPSHOT</version> |
||||||
|
</parent> |
||||||
|
<artifactId>dolphinscheduler-python</artifactId> |
||||||
|
<name>${project.artifactId}</name> |
||||||
|
<packaging>jar</packaging> |
||||||
|
|
||||||
|
<dependencies> |
||||||
|
<!-- dolphinscheduler --> |
||||||
|
<dependency> |
||||||
|
<groupId>org.apache.dolphinscheduler</groupId> |
||||||
|
<artifactId>dolphinscheduler-api</artifactId> |
||||||
|
</dependency> |
||||||
|
|
||||||
|
<!--springboot--> |
||||||
|
<dependency> |
||||||
|
<groupId>org.springframework.boot</groupId> |
||||||
|
<artifactId>spring-boot-starter-web</artifactId> |
||||||
|
<exclusions> |
||||||
|
<exclusion> |
||||||
|
<groupId>org.springframework.boot</groupId> |
||||||
|
<artifactId>spring-boot-starter-tomcat</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<artifactId>log4j-to-slf4j</artifactId> |
||||||
|
<groupId>org.apache.logging.log4j</groupId> |
||||||
|
</exclusion> |
||||||
|
</exclusions> |
||||||
|
</dependency> |
||||||
|
|
||||||
|
<dependency> |
||||||
|
<groupId>net.sf.py4j</groupId> |
||||||
|
<artifactId>py4j</artifactId> |
||||||
|
</dependency> |
||||||
|
|
||||||
|
</dependencies> |
||||||
|
</project> |
@ -0,0 +1,32 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
[run] |
||||||
|
command_line = -m pytest |
||||||
|
omit = |
||||||
|
# Ignore all test cases in tests/ |
||||||
|
tests/* |
||||||
|
# TODO. Temporary ignore java_gateway file, because we could not find good way to test it. |
||||||
|
src/pydolphinscheduler/java_gateway.py |
||||||
|
|
||||||
|
[report] |
||||||
|
# Don’t report files that are 100% covered |
||||||
|
skip_covered = True |
||||||
|
show_missing = True |
||||||
|
precision = 2 |
||||||
|
# Report will fail when coverage under 90.00% |
||||||
|
fail_under = 85 |
@ -0,0 +1,37 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
[flake8] |
||||||
|
max-line-length = 110 |
||||||
|
exclude = |
||||||
|
.git, |
||||||
|
__pycache__, |
||||||
|
.pytest_cache, |
||||||
|
*.egg-info, |
||||||
|
docs/source/conf.py |
||||||
|
old, |
||||||
|
build, |
||||||
|
dist, |
||||||
|
htmlcov |
||||||
|
ignore = |
||||||
|
# It's clear and not need to add docstring |
||||||
|
D107, # D107: Don't require docstrings on __init__ |
||||||
|
D105, # D105: Missing docstring in magic method |
||||||
|
# Conflict to Black |
||||||
|
W503 # W503: Line breaks before binary operators |
||||||
|
per-file-ignores = |
||||||
|
src/pydolphinscheduler/side/__init__.py:F401 |
@ -0,0 +1,19 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
[settings] |
||||||
|
profile=black |
@ -0,0 +1,173 @@ |
|||||||
|
<!-- |
||||||
|
Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
or more contributor license agreements. See the NOTICE file |
||||||
|
distributed with this work for additional information |
||||||
|
regarding copyright ownership. The ASF licenses this file |
||||||
|
to you under the Apache License, Version 2.0 (the |
||||||
|
"License"); you may not use this file except in compliance |
||||||
|
with the License. You may obtain a copy of the License at |
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, |
||||||
|
software distributed under the License is distributed on an |
||||||
|
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
KIND, either express or implied. See the License for the |
||||||
|
specific language governing permissions and limitations |
||||||
|
under the License. |
||||||
|
--> |
||||||
|
|
||||||
|
# pydolphinscheduler |
||||||
|
|
||||||
|
[![GitHub Build][ga-py-test]][ga] |
||||||
|
[![Code style: black][black-shield]][black-gh] |
||||||
|
[![Imports: isort][isort-shield]][isort-gh] |
||||||
|
|
||||||
|
pydolphinscheduler is python API for Apache DolphinScheduler, which allow you definition |
||||||
|
your workflow by python code, aka workflow-as-codes. |
||||||
|
|
||||||
|
## Quick Start |
||||||
|
|
||||||
|
> **_Notice:_** For now, due to pydolphinscheduler without release to any binary tarball or [PyPI][pypi], you |
||||||
|
> have to clone Apache DolphinScheduler code from GitHub to ensure quick start setup |
||||||
|
|
||||||
|
Here we show you how to install and run a simple example of pydolphinscheduler |
||||||
|
|
||||||
|
### Prepare |
||||||
|
|
||||||
|
```shell |
||||||
|
# Clone code from github |
||||||
|
git clone git@github.com:apache/dolphinscheduler.git |
||||||
|
|
||||||
|
# Install pydolphinscheduler from source |
||||||
|
cd dolphinscheduler-python/pydolphinscheduler |
||||||
|
pip install -e . |
||||||
|
``` |
||||||
|
|
||||||
|
### Start Server And Run Example |
||||||
|
|
||||||
|
Before you run an example, you have to start backend server. You could follow [development setup][dev-setup] |
||||||
|
section "DolphinScheduler Standalone Quick Start" to set up developer environment. You have to start backend |
||||||
|
and frontend server in this step, which mean that you could view DolphinScheduler UI in your browser with URL |
||||||
|
http://localhost:12345/dolphinscheduler |
||||||
|
|
||||||
|
After backend server is being start, all requests from `pydolphinscheduler` would be sent to backend server. |
||||||
|
And for now we could run a simple example by: |
||||||
|
|
||||||
|
```shell |
||||||
|
cd dolphinscheduler-python/pydolphinscheduler |
||||||
|
python example/tutorial.py |
||||||
|
``` |
||||||
|
|
||||||
|
> **_NOTICE:_** Since Apache DolphinScheduler's tenant is requests while running command, you might need to change |
||||||
|
> tenant value in `example/tutorial.py`. For now the value is `tenant_exists`, please change it to username exists |
||||||
|
> in you environment. |
||||||
|
|
||||||
|
After command execute, you could see a new project with single process definition named *tutorial* in the [UI][ui-project]. |
||||||
|
|
||||||
|
Until now, we finish quick start by an example of pydolphinscheduler and run it. If you want to inspect or join |
||||||
|
pydolphinscheduler develop, you could take a look at [develop](#develop) |
||||||
|
|
||||||
|
## Develop |
||||||
|
|
||||||
|
pydolphinscheduler is python API for Apache DolphinScheduler, it just defines what workflow look like instead of |
||||||
|
store or execute it. We here use [py4j][py4j] to dynamically access Java Virtual Machine. |
||||||
|
|
||||||
|
### Setup Develop Environment |
||||||
|
|
||||||
|
We already clone the code in [quick start](#quick-start), so next step we have to open pydolphinscheduler project |
||||||
|
in you editor. We recommend you use [pycharm][pycharm] instead of [IntelliJ IDEA][idea] to open it. And you could |
||||||
|
just open directory `dolphinscheduler-python/pydolphinscheduler` instead of `dolphinscheduler-python`. |
||||||
|
|
||||||
|
Then you should add developer dependence to make sure you could run test and check code style locally |
||||||
|
|
||||||
|
```shell |
||||||
|
pip install -r requirements_dev.txt |
||||||
|
``` |
||||||
|
|
||||||
|
### Brief Concept |
||||||
|
|
||||||
|
Apache DolphinScheduler is design to define workflow by UI, and pydolphinscheduler try to define it by code. When |
||||||
|
define by code, user usually do not care user, tenant, or queue exists or not. All user care about is created |
||||||
|
a new workflow by the code his/her definition. So we have some **side object** in `pydolphinscheduler/side` |
||||||
|
directory, their only check object exists or not, and create them if not exists. |
||||||
|
|
||||||
|
#### Process Definition |
||||||
|
|
||||||
|
pydolphinscheduler workflow object name, process definition is also same name as Java object(maybe would be change to |
||||||
|
other word for more simple). |
||||||
|
|
||||||
|
#### Tasks |
||||||
|
|
||||||
|
pydolphinscheduler tasks object, we use tasks to define exact job we want DolphinScheduler do for us. For now, |
||||||
|
we only support `shell` task to execute shell task. [This link][all-task] list all tasks support in DolphinScheduler |
||||||
|
and would be implemented in the further. |
||||||
|
|
||||||
|
### Code Style |
||||||
|
|
||||||
|
We use [isort][isort] to automatically keep Python imports alphabetically, and use [Black][black] for code |
||||||
|
formatter and [Flake8][flake8] for pep8 checker. If you use [pycharm][pycharm]or [IntelliJ IDEA][idea], |
||||||
|
maybe you could follow [Black-integration][black-editor] to configure them in your environment. |
||||||
|
|
||||||
|
Our Python API CI would automatically run code style checker and unittest when you submit pull request in |
||||||
|
GitHub, you could also run static check locally. |
||||||
|
|
||||||
|
```shell |
||||||
|
# We recommend you run isort and Black before Flake8, because Black could auto fix some code style issue |
||||||
|
# but Flake8 just hint when code style not match pep8 |
||||||
|
|
||||||
|
# Run Isort |
||||||
|
isort . |
||||||
|
|
||||||
|
# Run Black |
||||||
|
black . |
||||||
|
|
||||||
|
# Run Flake8 |
||||||
|
flake8 |
||||||
|
``` |
||||||
|
|
||||||
|
### Testing |
||||||
|
|
||||||
|
pydolphinscheduler using [pytest][pytest] to test our codebase. GitHub Action will run our test when you create |
||||||
|
pull request or commit to dev branch, with python version `3.6|3.7|3.8|3.9` and operating system `linux|macOS|windows`. |
||||||
|
|
||||||
|
To test locally, you could directly run pytest after set `PYTHONPATH` |
||||||
|
|
||||||
|
```shell |
||||||
|
PYTHONPATH=src/ pytest |
||||||
|
``` |
||||||
|
|
||||||
|
We try to keep pydolphinscheduler usable through unit test coverage. 90% test coverage is our target, but for |
||||||
|
now, we require test coverage up to 85%, and each pull request leas than 85% would fail our CI step |
||||||
|
`Tests coverage`. We use [coverage][coverage] to check our test coverage, and you could check it locally by |
||||||
|
run command. |
||||||
|
|
||||||
|
```shell |
||||||
|
coverage run && coverage report |
||||||
|
``` |
||||||
|
|
||||||
|
It would not only run unit test but also show each file coverage which cover rate less than 100%, and `TOTAL` |
||||||
|
line show you total coverage of you code. If your CI failed with coverage you could go and find some reason by |
||||||
|
this command output. |
||||||
|
|
||||||
|
<!-- content --> |
||||||
|
[pypi]: https://pypi.org/ |
||||||
|
[dev-setup]: https://dolphinscheduler.apache.org/en-us/development/development-environment-setup.html |
||||||
|
[ui-project]: http://8.142.34.29:12345/dolphinscheduler/ui/#/projects/list |
||||||
|
[py4j]: https://www.py4j.org/index.html |
||||||
|
[pycharm]: https://www.jetbrains.com/pycharm |
||||||
|
[idea]: https://www.jetbrains.com/idea/ |
||||||
|
[all-task]: https://dolphinscheduler.apache.org/en-us/docs/dev/user_doc/guide/task/shell.html |
||||||
|
[pytest]: https://docs.pytest.org/en/latest/ |
||||||
|
[black]: https://black.readthedocs.io/en/stable/index.html |
||||||
|
[flake8]: https://flake8.pycqa.org/en/latest/index.html |
||||||
|
[black-editor]: https://black.readthedocs.io/en/stable/integrations/editors.html#pycharm-intellij-idea |
||||||
|
[coverage]: https://coverage.readthedocs.io/en/stable/ |
||||||
|
[isort]: https://pycqa.github.io/isort/index.html |
||||||
|
<!-- badge --> |
||||||
|
[ga-py-test]: https://github.com/apache/dolphinscheduler/actions/workflows/py-ci.yml/badge.svg?branch=dev |
||||||
|
[ga]: https://github.com/apache/dolphinscheduler/actions |
||||||
|
[black-shield]: https://img.shields.io/badge/code%20style-black-000000.svg |
||||||
|
[black-gh]: https://github.com/psf/black |
||||||
|
[isort-shield]: https://img.shields.io/badge/%20imports-isort-%231674b1?style=flat&labelColor=ef8336 |
||||||
|
[isort-gh]: https://pycqa.github.io/isort/ |
@ -0,0 +1,34 @@ |
|||||||
|
<!-- |
||||||
|
Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
or more contributor license agreements. See the NOTICE file |
||||||
|
distributed with this work for additional information |
||||||
|
regarding copyright ownership. The ASF licenses this file |
||||||
|
to you under the Apache License, Version 2.0 (the |
||||||
|
"License"); you may not use this file except in compliance |
||||||
|
with the License. You may obtain a copy of the License at |
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, |
||||||
|
software distributed under the License is distributed on an |
||||||
|
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
KIND, either express or implied. See the License for the |
||||||
|
specific language governing permissions and limitations |
||||||
|
under the License. |
||||||
|
--> |
||||||
|
|
||||||
|
## Roadmap |
||||||
|
|
||||||
|
### v0.0.3 |
||||||
|
|
||||||
|
Add other features, tasks, parameters in DS, keep code coverage up to 90% |
||||||
|
|
||||||
|
### v0.0.2 |
||||||
|
|
||||||
|
Add docs about how to use and develop package, code coverage up to 90%, add CI/CD |
||||||
|
for package |
||||||
|
|
||||||
|
### v0.0.1(current) |
||||||
|
|
||||||
|
Setup up POC, for defining DAG with python code, running DAG manually, |
||||||
|
releasing to pypi |
@ -0,0 +1,55 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
""" |
||||||
|
This example show you how to create workflows in batch mode. |
||||||
|
|
||||||
|
After this example run, we will create 10 workflows named `workflow:<workflow_num>`, and with 3 tasks |
||||||
|
named `task:<task_num>-workflow:<workflow_num>` in each workflow. Task shape as below |
||||||
|
|
||||||
|
task:1-workflow:1 -> task:2-workflow:1 -> task:3-workflow:1 |
||||||
|
|
||||||
|
Each workflow is linear since we set `IS_CHAIN=True`, you could change task to parallel by set it to `False`. |
||||||
|
""" |
||||||
|
|
||||||
|
from pydolphinscheduler.core.process_definition import ProcessDefinition |
||||||
|
from pydolphinscheduler.tasks.shell import Shell |
||||||
|
|
||||||
|
NUM_WORKFLOWS = 10 |
||||||
|
NUM_TASKS = 5 |
||||||
|
# Make sure your tenant exists in your operator system |
||||||
|
TENANT = "exists_tenant" |
||||||
|
# Whether task should dependent on pre one or not |
||||||
|
# False will create workflow with independent task, while True task will dependent on pre-task and dependence |
||||||
|
# link like `pre_task -> current_task -> next_task`, default True |
||||||
|
IS_CHAIN = True |
||||||
|
|
||||||
|
for wf in range(0, NUM_WORKFLOWS): |
||||||
|
workflow_name = f"workflow:{wf}" |
||||||
|
|
||||||
|
with ProcessDefinition(name=workflow_name, tenant=TENANT) as pd: |
||||||
|
for t in range(0, NUM_TASKS): |
||||||
|
task_name = f"task:{t}-{workflow_name}" |
||||||
|
command = f"echo This is task {task_name}" |
||||||
|
task = Shell(name=task_name, command=command) |
||||||
|
|
||||||
|
if IS_CHAIN and t > 0: |
||||||
|
pre_task_name = f"task:{t-1}-{workflow_name}" |
||||||
|
pd.get_one_task_by_name(pre_task_name) >> task |
||||||
|
|
||||||
|
# We just submit workflow and task definition without set schedule time or run it manually |
||||||
|
pd.submit() |
@ -0,0 +1,55 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
r""" |
||||||
|
A example workflow for task condition. |
||||||
|
|
||||||
|
This example will create five task in single workflow, with four shell task and one condition task. Task |
||||||
|
condition have one upstream which we declare explicit with syntax `parent >> condition`, and three downstream |
||||||
|
automatically set dependence by condition task by passing parameter `condition`. The graph of this workflow |
||||||
|
like: |
||||||
|
--> condition_success_1 |
||||||
|
/ |
||||||
|
parent -> conditions -> --> condition_success_2 |
||||||
|
\ |
||||||
|
--> condition_fail |
||||||
|
. |
||||||
|
""" |
||||||
|
|
||||||
|
from pydolphinscheduler.core.process_definition import ProcessDefinition |
||||||
|
from pydolphinscheduler.tasks.condition import FAILURE, SUCCESS, And, Conditions |
||||||
|
from pydolphinscheduler.tasks.shell import Shell |
||||||
|
|
||||||
|
with ProcessDefinition(name="task_conditions_example", tenant="tenant_exists") as pd: |
||||||
|
parent = Shell(name="parent", command="echo parent") |
||||||
|
condition_success_1 = Shell( |
||||||
|
name="condition_success_1", command="echo condition_success_1" |
||||||
|
) |
||||||
|
condition_success_2 = Shell( |
||||||
|
name="condition_success_2", command="echo condition_success_2" |
||||||
|
) |
||||||
|
condition_fail = Shell(name="condition_fail", command="echo condition_fail") |
||||||
|
cond_operator = And( |
||||||
|
And( |
||||||
|
SUCCESS(condition_success_1, condition_success_2), |
||||||
|
FAILURE(condition_fail), |
||||||
|
), |
||||||
|
) |
||||||
|
|
||||||
|
condition = Conditions(name="conditions", condition=cond_operator) |
||||||
|
parent >> condition |
||||||
|
pd.submit() |
@ -0,0 +1,50 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
""" |
||||||
|
A example workflow for task datax. |
||||||
|
|
||||||
|
This example will create a workflow named `task_datax`. |
||||||
|
`task_datax` is true workflow define and run task task_datax. |
||||||
|
You can create data sources `first_mysql` and `first_mysql` through UI. |
||||||
|
It creates a task to synchronize datax from the source database to the target database. |
||||||
|
""" |
||||||
|
|
||||||
|
|
||||||
|
from pydolphinscheduler.core.process_definition import ProcessDefinition |
||||||
|
from pydolphinscheduler.tasks.datax import CustomDataX, DataX |
||||||
|
|
||||||
|
# datax json template |
||||||
|
JSON_TEMPLATE = "" |
||||||
|
|
||||||
|
with ProcessDefinition( |
||||||
|
name="task_datax", |
||||||
|
tenant="tenant_exists", |
||||||
|
) as pd: |
||||||
|
# This task synchronizes the data in `t_ds_project` |
||||||
|
# of `first_mysql` database to `target_project` of `second_mysql` database. |
||||||
|
task1 = DataX( |
||||||
|
name="task_datax", |
||||||
|
datasource_name="first_mysql", |
||||||
|
datatarget_name="second_mysql", |
||||||
|
sql="select id, name, code, description from source_table", |
||||||
|
target_table="target_table", |
||||||
|
) |
||||||
|
|
||||||
|
# you can custom json_template of datax to sync data. |
||||||
|
task2 = CustomDataX(name="task_custom_datax", json=JSON_TEMPLATE) |
||||||
|
pd.run() |
@ -0,0 +1,72 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
r""" |
||||||
|
A example workflow for task dependent. |
||||||
|
|
||||||
|
This example will create two workflows named `task_dependent` and `task_dependent_external`. |
||||||
|
`task_dependent` is true workflow define and run task dependent, while `task_dependent_external` |
||||||
|
define outside workflow and task from dependent. |
||||||
|
|
||||||
|
After this script submit, we would get workflow as below: |
||||||
|
|
||||||
|
task_dependent_external: |
||||||
|
|
||||||
|
task_1 |
||||||
|
task_2 |
||||||
|
task_3 |
||||||
|
|
||||||
|
task_dependent: |
||||||
|
|
||||||
|
task_dependent(this task dependent on task_dependent_external.task_1 and task_dependent_external.task_2). |
||||||
|
""" |
||||||
|
from pydolphinscheduler.constants import ProcessDefinitionDefault |
||||||
|
from pydolphinscheduler.core.process_definition import ProcessDefinition |
||||||
|
from pydolphinscheduler.tasks.dependent import And, Dependent, DependentItem, Or |
||||||
|
from pydolphinscheduler.tasks.shell import Shell |
||||||
|
|
||||||
|
with ProcessDefinition( |
||||||
|
name="task_dependent_external", |
||||||
|
tenant="tenant_exists", |
||||||
|
) as pd: |
||||||
|
task_1 = Shell(name="task_1", command="echo task 1") |
||||||
|
task_2 = Shell(name="task_2", command="echo task 2") |
||||||
|
task_3 = Shell(name="task_3", command="echo task 3") |
||||||
|
pd.submit() |
||||||
|
|
||||||
|
with ProcessDefinition( |
||||||
|
name="task_dependent", |
||||||
|
tenant="tenant_exists", |
||||||
|
) as pd: |
||||||
|
task = Dependent( |
||||||
|
name="task_dependent", |
||||||
|
dependence=And( |
||||||
|
Or( |
||||||
|
DependentItem( |
||||||
|
project_name=ProcessDefinitionDefault.PROJECT, |
||||||
|
process_definition_name="task_dependent_external", |
||||||
|
dependent_task_name="task_1", |
||||||
|
), |
||||||
|
DependentItem( |
||||||
|
project_name=ProcessDefinitionDefault.PROJECT, |
||||||
|
process_definition_name="task_dependent_external", |
||||||
|
dependent_task_name="task_2", |
||||||
|
), |
||||||
|
) |
||||||
|
), |
||||||
|
) |
||||||
|
pd.submit() |
@ -0,0 +1,50 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
r""" |
||||||
|
A example workflow for task switch. |
||||||
|
|
||||||
|
This example will create four task in single workflow, with three shell task and one switch task. Task switch |
||||||
|
have one upstream which we declare explicit with syntax `parent >> switch`, and two downstream automatically |
||||||
|
set dependence by switch task by passing parameter `condition`. The graph of this workflow like: |
||||||
|
--> switch_child_1 |
||||||
|
/ |
||||||
|
parent -> switch -> |
||||||
|
\ |
||||||
|
--> switch_child_2 |
||||||
|
. |
||||||
|
""" |
||||||
|
|
||||||
|
from pydolphinscheduler.core.process_definition import ProcessDefinition |
||||||
|
from pydolphinscheduler.tasks.shell import Shell |
||||||
|
from pydolphinscheduler.tasks.switch import Branch, Default, Switch, SwitchCondition |
||||||
|
|
||||||
|
with ProcessDefinition( |
||||||
|
name="task_dependent_external", |
||||||
|
tenant="tenant_exists", |
||||||
|
) as pd: |
||||||
|
parent = Shell(name="parent", command="echo parent") |
||||||
|
switch_child_1 = Shell(name="switch_child_1", command="echo switch_child_1") |
||||||
|
switch_child_2 = Shell(name="switch_child_2", command="echo switch_child_2") |
||||||
|
switch_condition = SwitchCondition( |
||||||
|
Branch(condition="${var} > 1", task=switch_child_1), |
||||||
|
Default(task=switch_child_2), |
||||||
|
) |
||||||
|
|
||||||
|
switch = Switch(name="switch", condition=switch_condition) |
||||||
|
parent >> switch |
||||||
|
pd.submit() |
@ -0,0 +1,52 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
r""" |
||||||
|
A tutorial example take you to experience pydolphinscheduler. |
||||||
|
|
||||||
|
After tutorial.py file submit to Apache DolphinScheduler server a DAG would be create, |
||||||
|
and workflow DAG graph as below: |
||||||
|
|
||||||
|
--> task_child_one |
||||||
|
/ \ |
||||||
|
task_parent --> --> task_union |
||||||
|
\ / |
||||||
|
--> task_child_two |
||||||
|
|
||||||
|
it will instantiate and run all the task it have. |
||||||
|
""" |
||||||
|
|
||||||
|
from pydolphinscheduler.core.process_definition import ProcessDefinition |
||||||
|
from pydolphinscheduler.tasks.shell import Shell |
||||||
|
|
||||||
|
with ProcessDefinition( |
||||||
|
name="tutorial", |
||||||
|
schedule="0 0 0 * * ? *", |
||||||
|
start_time="2021-01-01", |
||||||
|
tenant="tenant_exists", |
||||||
|
) as pd: |
||||||
|
task_parent = Shell(name="task_parent", command="echo hello pydolphinscheduler") |
||||||
|
task_child_one = Shell(name="task_child_one", command="echo 'child one'") |
||||||
|
task_child_two = Shell(name="task_child_two", command="echo 'child two'") |
||||||
|
task_union = Shell(name="task_union", command="echo union") |
||||||
|
|
||||||
|
task_group = [task_child_one, task_child_two] |
||||||
|
task_parent.set_downstream(task_group) |
||||||
|
|
||||||
|
task_union << task_group |
||||||
|
|
||||||
|
pd.run() |
@ -0,0 +1,22 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
# contributor license agreements. See the NOTICE file distributed with |
||||||
|
# this work for additional information regarding copyright ownership. |
||||||
|
# The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
# (the "License"); you may not use this file except in compliance with |
||||||
|
# the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, software |
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
# See the License for the specific language governing permissions and |
||||||
|
# limitations under the License. |
||||||
|
|
||||||
|
[pytest] |
||||||
|
# Do not test test_java_gateway.py due to we can not mock java gateway for now |
||||||
|
addopts = --ignore=tests/test_java_gateway.py |
||||||
|
|
||||||
|
# add path here to skip pytest scan it |
||||||
|
norecursedirs = |
||||||
|
tests/testing |
@ -0,0 +1,18 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
py4j~=0.10.9.2 |
@ -0,0 +1,27 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
# testting |
||||||
|
pytest~=6.2.5 |
||||||
|
freezegun |
||||||
|
# Test coverage |
||||||
|
coverage |
||||||
|
# code linting and formatting |
||||||
|
flake8 |
||||||
|
flake8-docstrings |
||||||
|
flake8-black |
||||||
|
isort |
@ -0,0 +1,16 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
@ -0,0 +1,94 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""The script for setting up pydolphinscheduler.""" |
||||||
|
|
||||||
|
import sys |
||||||
|
from os.path import dirname, join |
||||||
|
|
||||||
|
from setuptools import find_packages, setup |
||||||
|
|
||||||
|
version = "0.0.1.dev0" |
||||||
|
|
||||||
|
if sys.version_info[0] < 3: |
||||||
|
raise Exception( |
||||||
|
"pydolphinscheduler does not support Python 2. Please upgrade to Python 3." |
||||||
|
) |
||||||
|
|
||||||
|
|
||||||
|
def read(*names, **kwargs): |
||||||
|
"""Read file content from given file path.""" |
||||||
|
return open( |
||||||
|
join(dirname(__file__), *names), encoding=kwargs.get("encoding", "utf8") |
||||||
|
).read() |
||||||
|
|
||||||
|
|
||||||
|
setup( |
||||||
|
name="pydolphinscheduler", |
||||||
|
version=version, |
||||||
|
license="Apache License 2.0", |
||||||
|
description="Apache DolphinScheduler python SDK", |
||||||
|
long_description=read("README.md"), |
||||||
|
# Make sure pypi is expecting markdown |
||||||
|
long_description_content_type="text/markdown", |
||||||
|
author="Apache Software Foundation", |
||||||
|
author_email="dev@dolphinscheduler.apache.org", |
||||||
|
url="https://dolphinscheduler.apache.org/", |
||||||
|
python_requires=">=3.6", |
||||||
|
keywords=[ |
||||||
|
"dolphinscheduler", |
||||||
|
"workflow", |
||||||
|
"scheduler", |
||||||
|
"taskflow", |
||||||
|
], |
||||||
|
project_urls={ |
||||||
|
"Homepage": "https://dolphinscheduler.apache.org", |
||||||
|
"Documentation": "https://dolphinscheduler.apache.org/en-us/docs/latest/user_doc/quick-start.html", |
||||||
|
"Source": "https://github.com/apache/dolphinscheduler", |
||||||
|
"Issue Tracker": "https://github.com/apache/dolphinscheduler/issues", |
||||||
|
"Discussion": "https://github.com/apache/dolphinscheduler/discussions", |
||||||
|
"Twitter": "https://twitter.com/dolphinschedule", |
||||||
|
}, |
||||||
|
packages=find_packages(where="src"), |
||||||
|
package_dir={"": "src"}, |
||||||
|
include_package_data=True, |
||||||
|
classifiers=[ |
||||||
|
# complete classifier list: http://pypi.python.org/pypi?%3Aaction=list_classifiers |
||||||
|
"Development Status :: 1 - Planning", |
||||||
|
"Environment :: Console", |
||||||
|
"Intended Audience :: Developers", |
||||||
|
"License :: OSI Approved :: Apache Software License", |
||||||
|
"Operating System :: Unix", |
||||||
|
"Operating System :: POSIX", |
||||||
|
"Operating System :: Microsoft :: Windows", |
||||||
|
"Programming Language :: Python", |
||||||
|
"Programming Language :: Python :: 3", |
||||||
|
"Programming Language :: Python :: 3.6", |
||||||
|
"Programming Language :: Python :: 3.7", |
||||||
|
"Programming Language :: Python :: 3.8", |
||||||
|
"Programming Language :: Python :: 3.9", |
||||||
|
"Programming Language :: Python :: Implementation :: CPython", |
||||||
|
"Programming Language :: Python :: Implementation :: PyPy", |
||||||
|
"Topic :: Software Development :: User Interfaces", |
||||||
|
], |
||||||
|
install_requires=[ |
||||||
|
# Core |
||||||
|
"py4j~=0.10", |
||||||
|
# Dev |
||||||
|
"pytest~=6.2", |
||||||
|
], |
||||||
|
) |
@ -0,0 +1,18 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""Init root of pydolphinscheduler.""" |
@ -0,0 +1,122 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""Constants for pydolphinscheduler.""" |
||||||
|
|
||||||
|
|
||||||
|
class ProcessDefinitionReleaseState: |
||||||
|
"""Constants for :class:`pydolphinscheduler.core.process_definition.ProcessDefinition` release state.""" |
||||||
|
|
||||||
|
ONLINE: str = "ONLINE" |
||||||
|
OFFLINE: str = "OFFLINE" |
||||||
|
|
||||||
|
|
||||||
|
class ProcessDefinitionDefault: |
||||||
|
"""Constants default value for :class:`pydolphinscheduler.core.process_definition.ProcessDefinition`.""" |
||||||
|
|
||||||
|
PROJECT: str = "project-pydolphin" |
||||||
|
TENANT: str = "tenant_pydolphin" |
||||||
|
USER: str = "userPythonGateway" |
||||||
|
# TODO simple set password same as username |
||||||
|
USER_PWD: str = "userPythonGateway" |
||||||
|
USER_EMAIL: str = "userPythonGateway@dolphinscheduler.com" |
||||||
|
USER_PHONE: str = "11111111111" |
||||||
|
USER_STATE: int = 1 |
||||||
|
QUEUE: str = "queuePythonGateway" |
||||||
|
WORKER_GROUP: str = "default" |
||||||
|
TIME_ZONE: str = "Asia/Shanghai" |
||||||
|
|
||||||
|
|
||||||
|
class TaskPriority(str): |
||||||
|
"""Constants for task priority.""" |
||||||
|
|
||||||
|
HIGHEST = "HIGHEST" |
||||||
|
HIGH = "HIGH" |
||||||
|
MEDIUM = "MEDIUM" |
||||||
|
LOW = "LOW" |
||||||
|
LOWEST = "LOWEST" |
||||||
|
|
||||||
|
|
||||||
|
class TaskFlag(str): |
||||||
|
"""Constants for task flag.""" |
||||||
|
|
||||||
|
YES = "YES" |
||||||
|
NO = "NO" |
||||||
|
|
||||||
|
|
||||||
|
class TaskTimeoutFlag(str): |
||||||
|
"""Constants for task timeout flag.""" |
||||||
|
|
||||||
|
CLOSE = "CLOSE" |
||||||
|
|
||||||
|
|
||||||
|
class TaskType(str): |
||||||
|
"""Constants for task type, it will also show you which kind we support up to now.""" |
||||||
|
|
||||||
|
SHELL = "SHELL" |
||||||
|
HTTP = "HTTP" |
||||||
|
PYTHON = "PYTHON" |
||||||
|
SQL = "SQL" |
||||||
|
SUB_PROCESS = "SUB_PROCESS" |
||||||
|
PROCEDURE = "PROCEDURE" |
||||||
|
DATAX = "DATAX" |
||||||
|
DEPENDENT = "DEPENDENT" |
||||||
|
CONDITIONS = "CONDITIONS" |
||||||
|
SWITCH = "SWITCH" |
||||||
|
|
||||||
|
|
||||||
|
class DefaultTaskCodeNum(str): |
||||||
|
"""Constants and default value for default task code number.""" |
||||||
|
|
||||||
|
DEFAULT = 1 |
||||||
|
|
||||||
|
|
||||||
|
class JavaGatewayDefault(str): |
||||||
|
"""Constants and default value for java gateway.""" |
||||||
|
|
||||||
|
RESULT_MESSAGE_KEYWORD = "msg" |
||||||
|
RESULT_MESSAGE_SUCCESS = "success" |
||||||
|
|
||||||
|
RESULT_STATUS_KEYWORD = "status" |
||||||
|
RESULT_STATUS_SUCCESS = "SUCCESS" |
||||||
|
|
||||||
|
RESULT_DATA = "data" |
||||||
|
|
||||||
|
|
||||||
|
class Delimiter(str): |
||||||
|
"""Constants for delimiter.""" |
||||||
|
|
||||||
|
BAR = "-" |
||||||
|
DASH = "/" |
||||||
|
COLON = ":" |
||||||
|
UNDERSCORE = "_" |
||||||
|
DIRECTION = "->" |
||||||
|
|
||||||
|
|
||||||
|
class Time(str): |
||||||
|
"""Constants for date.""" |
||||||
|
|
||||||
|
FMT_STD_DATE = "%Y-%m-%d" |
||||||
|
LEN_STD_DATE = 10 |
||||||
|
|
||||||
|
FMT_DASH_DATE = "%Y/%m/%d" |
||||||
|
|
||||||
|
FMT_SHORT_DATE = "%Y%m%d" |
||||||
|
LEN_SHORT_DATE = 8 |
||||||
|
|
||||||
|
FMT_STD_TIME = "%H:%M:%S" |
||||||
|
FMT_NO_COLON_TIME = "%H%M%S" |
@ -0,0 +1,18 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""Init pydolphinscheduler.core package.""" |
@ -0,0 +1,74 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""DolphinScheduler Base object.""" |
||||||
|
|
||||||
|
from typing import Dict, Optional |
||||||
|
|
||||||
|
# from pydolphinscheduler.side.user import User |
||||||
|
from pydolphinscheduler.utils.string import attr2camel |
||||||
|
|
||||||
|
|
||||||
|
class Base: |
||||||
|
"""DolphinScheduler Base object.""" |
||||||
|
|
||||||
|
# Object key attribute, to test whether object equals and so on. |
||||||
|
_KEY_ATTR: set = {"name", "description"} |
||||||
|
|
||||||
|
# Object defines attribute, use when needs to communicate with Java gateway server. |
||||||
|
_DEFINE_ATTR: set = set() |
||||||
|
|
||||||
|
# Object default attribute, will add those attribute to `_DEFINE_ATTR` when init assign missing. |
||||||
|
_DEFAULT_ATTR: Dict = {} |
||||||
|
|
||||||
|
def __init__(self, name: str, description: Optional[str] = None): |
||||||
|
self.name = name |
||||||
|
self.description = description |
||||||
|
|
||||||
|
def __repr__(self) -> str: |
||||||
|
return f'<{type(self).__name__}: name="{self.name}">' |
||||||
|
|
||||||
|
def __eq__(self, other): |
||||||
|
return type(self) == type(other) and all( |
||||||
|
getattr(self, a, None) == getattr(other, a, None) for a in self._KEY_ATTR |
||||||
|
) |
||||||
|
|
||||||
|
def get_define_custom( |
||||||
|
self, camel_attr: bool = True, custom_attr: set = None |
||||||
|
) -> Dict: |
||||||
|
"""Get object definition attribute by given attr set.""" |
||||||
|
content = {} |
||||||
|
for attr in custom_attr: |
||||||
|
val = getattr(self, attr, None) |
||||||
|
if camel_attr: |
||||||
|
content[attr2camel(attr)] = val |
||||||
|
else: |
||||||
|
content[attr] = val |
||||||
|
return content |
||||||
|
|
||||||
|
def get_define(self, camel_attr: bool = True) -> Dict: |
||||||
|
"""Get object definition attribute communicate to Java gateway server. |
||||||
|
|
||||||
|
use attribute `self._DEFINE_ATTR` to determine which attributes should including when |
||||||
|
object tries to communicate with Java gateway server. |
||||||
|
""" |
||||||
|
content = self.get_define_custom(camel_attr, self._DEFINE_ATTR) |
||||||
|
update_default = { |
||||||
|
k: self._DEFAULT_ATTR.get(k) for k in self._DEFAULT_ATTR if k not in content |
||||||
|
} |
||||||
|
content.update(update_default) |
||||||
|
return content |
@ -0,0 +1,40 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""Module for side object.""" |
||||||
|
|
||||||
|
from typing import Optional |
||||||
|
|
||||||
|
from pydolphinscheduler.constants import ProcessDefinitionDefault |
||||||
|
from pydolphinscheduler.core.base import Base |
||||||
|
|
||||||
|
|
||||||
|
class BaseSide(Base): |
||||||
|
"""Base class for side object, it declare base behavior for them.""" |
||||||
|
|
||||||
|
def __init__(self, name: str, description: Optional[str] = None): |
||||||
|
super().__init__(name, description) |
||||||
|
|
||||||
|
@classmethod |
||||||
|
def create_if_not_exists( |
||||||
|
cls, |
||||||
|
# TODO comment for avoiding cycle import |
||||||
|
# user: Optional[User] = ProcessDefinitionDefault.USER |
||||||
|
user=ProcessDefinitionDefault.USER, |
||||||
|
): |
||||||
|
"""Create Base if not exists.""" |
||||||
|
raise NotImplementedError |
@ -0,0 +1,64 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""Module database.""" |
||||||
|
|
||||||
|
import logging |
||||||
|
from typing import Dict |
||||||
|
|
||||||
|
from py4j.protocol import Py4JJavaError |
||||||
|
|
||||||
|
from pydolphinscheduler.java_gateway import launch_gateway |
||||||
|
|
||||||
|
|
||||||
|
class Database(dict): |
||||||
|
"""database object, get information about database. |
||||||
|
|
||||||
|
You provider database_name contain connection information, it decisions which |
||||||
|
database type and database instance would run task. |
||||||
|
""" |
||||||
|
|
||||||
|
def __init__(self, database_name: str, type_key, database_key, *args, **kwargs): |
||||||
|
super().__init__(*args, **kwargs) |
||||||
|
self._database = {} |
||||||
|
self.database_name = database_name |
||||||
|
self[type_key] = self.database_type |
||||||
|
self[database_key] = self.database_id |
||||||
|
|
||||||
|
@property |
||||||
|
def database_type(self) -> str: |
||||||
|
"""Get database type from java gateway, a wrapper for :func:`get_database_info`.""" |
||||||
|
return self.get_database_info(self.database_name).get("type") |
||||||
|
|
||||||
|
@property |
||||||
|
def database_id(self) -> str: |
||||||
|
"""Get database id from java gateway, a wrapper for :func:`get_database_info`.""" |
||||||
|
return self.get_database_info(self.database_name).get("id") |
||||||
|
|
||||||
|
def get_database_info(self, name) -> Dict: |
||||||
|
"""Get database info from java gateway, contains database id, type, name.""" |
||||||
|
if self._database: |
||||||
|
return self._database |
||||||
|
else: |
||||||
|
gateway = launch_gateway() |
||||||
|
try: |
||||||
|
self._database = gateway.entry_point.getDatasourceInfo(name) |
||||||
|
# Handler database source do not exists error, for now we just terminate the process. |
||||||
|
except Py4JJavaError: |
||||||
|
logging.error("Datasource name `%s` do not exists.", name) |
||||||
|
exit(1) |
||||||
|
return self._database |
@ -0,0 +1,360 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""Module process definition, core class for workflow define.""" |
||||||
|
|
||||||
|
import json |
||||||
|
from datetime import datetime |
||||||
|
from typing import Any, Dict, List, Optional, Set |
||||||
|
|
||||||
|
from pydolphinscheduler.constants import ( |
||||||
|
ProcessDefinitionDefault, |
||||||
|
ProcessDefinitionReleaseState, |
||||||
|
) |
||||||
|
from pydolphinscheduler.core.base import Base |
||||||
|
from pydolphinscheduler.exceptions import PyDSParamException, PyDSTaskNoFoundException |
||||||
|
from pydolphinscheduler.java_gateway import launch_gateway |
||||||
|
from pydolphinscheduler.side import Project, Tenant, User |
||||||
|
from pydolphinscheduler.utils.date import MAX_DATETIME, conv_from_str, conv_to_schedule |
||||||
|
|
||||||
|
|
||||||
|
class ProcessDefinitionContext: |
||||||
|
"""Class process definition context, use when task get process definition from context expression.""" |
||||||
|
|
||||||
|
_context_managed_process_definition: Optional["ProcessDefinition"] = None |
||||||
|
|
||||||
|
@classmethod |
||||||
|
def set(cls, pd: "ProcessDefinition") -> None: |
||||||
|
"""Set attribute self._context_managed_process_definition.""" |
||||||
|
cls._context_managed_process_definition = pd |
||||||
|
|
||||||
|
@classmethod |
||||||
|
def get(cls) -> Optional["ProcessDefinition"]: |
||||||
|
"""Get attribute self._context_managed_process_definition.""" |
||||||
|
return cls._context_managed_process_definition |
||||||
|
|
||||||
|
@classmethod |
||||||
|
def delete(cls) -> None: |
||||||
|
"""Delete attribute self._context_managed_process_definition.""" |
||||||
|
cls._context_managed_process_definition = None |
||||||
|
|
||||||
|
|
||||||
|
class ProcessDefinition(Base): |
||||||
|
"""process definition object, will define process definition attribute, task, relation. |
||||||
|
|
||||||
|
TODO: maybe we should rename this class, currently use DS object name. |
||||||
|
""" |
||||||
|
|
||||||
|
# key attribute for identify ProcessDefinition object |
||||||
|
_KEY_ATTR = { |
||||||
|
"name", |
||||||
|
"project", |
||||||
|
"tenant", |
||||||
|
"release_state", |
||||||
|
"param", |
||||||
|
} |
||||||
|
|
||||||
|
_DEFINE_ATTR = { |
||||||
|
"name", |
||||||
|
"description", |
||||||
|
"_project", |
||||||
|
"_tenant", |
||||||
|
"worker_group", |
||||||
|
"timeout", |
||||||
|
"release_state", |
||||||
|
"param", |
||||||
|
"tasks", |
||||||
|
"task_definition_json", |
||||||
|
"task_relation_json", |
||||||
|
} |
||||||
|
|
||||||
|
def __init__( |
||||||
|
self, |
||||||
|
name: str, |
||||||
|
description: Optional[str] = None, |
||||||
|
schedule: Optional[str] = None, |
||||||
|
start_time: Optional[str] = None, |
||||||
|
end_time: Optional[str] = None, |
||||||
|
timezone: Optional[str] = ProcessDefinitionDefault.TIME_ZONE, |
||||||
|
user: Optional[str] = ProcessDefinitionDefault.USER, |
||||||
|
project: Optional[str] = ProcessDefinitionDefault.PROJECT, |
||||||
|
tenant: Optional[str] = ProcessDefinitionDefault.TENANT, |
||||||
|
queue: Optional[str] = ProcessDefinitionDefault.QUEUE, |
||||||
|
worker_group: Optional[str] = ProcessDefinitionDefault.WORKER_GROUP, |
||||||
|
timeout: Optional[int] = 0, |
||||||
|
release_state: Optional[str] = ProcessDefinitionReleaseState.ONLINE, |
||||||
|
param: Optional[List] = None, |
||||||
|
): |
||||||
|
super().__init__(name, description) |
||||||
|
self.schedule = schedule |
||||||
|
self._start_time = start_time |
||||||
|
self._end_time = end_time |
||||||
|
self.timezone = timezone |
||||||
|
self._user = user |
||||||
|
self._project = project |
||||||
|
self._tenant = tenant |
||||||
|
self._queue = queue |
||||||
|
self.worker_group = worker_group |
||||||
|
self.timeout = timeout |
||||||
|
self.release_state = release_state |
||||||
|
self.param = param |
||||||
|
self.tasks: dict = {} |
||||||
|
# TODO how to fix circle import |
||||||
|
self._task_relations: set["TaskRelation"] = set() # noqa: F821 |
||||||
|
self._process_definition_code = None |
||||||
|
|
||||||
|
def __enter__(self) -> "ProcessDefinition": |
||||||
|
ProcessDefinitionContext.set(self) |
||||||
|
return self |
||||||
|
|
||||||
|
def __exit__(self, exc_type, exc_val, exc_tb) -> None: |
||||||
|
ProcessDefinitionContext.delete() |
||||||
|
|
||||||
|
@property |
||||||
|
def tenant(self) -> Tenant: |
||||||
|
"""Get attribute tenant.""" |
||||||
|
return Tenant(self._tenant) |
||||||
|
|
||||||
|
@tenant.setter |
||||||
|
def tenant(self, tenant: Tenant) -> None: |
||||||
|
"""Set attribute tenant.""" |
||||||
|
self._tenant = tenant.name |
||||||
|
|
||||||
|
@property |
||||||
|
def project(self) -> Project: |
||||||
|
"""Get attribute project.""" |
||||||
|
return Project(self._project) |
||||||
|
|
||||||
|
@project.setter |
||||||
|
def project(self, project: Project) -> None: |
||||||
|
"""Set attribute project.""" |
||||||
|
self._project = project.name |
||||||
|
|
||||||
|
@property |
||||||
|
def user(self) -> User: |
||||||
|
"""Get user object. |
||||||
|
|
||||||
|
For now we just get from python side but not from java gateway side, so it may not correct. |
||||||
|
""" |
||||||
|
return User( |
||||||
|
self._user, |
||||||
|
ProcessDefinitionDefault.USER_PWD, |
||||||
|
ProcessDefinitionDefault.USER_EMAIL, |
||||||
|
ProcessDefinitionDefault.USER_PHONE, |
||||||
|
self._tenant, |
||||||
|
self._queue, |
||||||
|
ProcessDefinitionDefault.USER_STATE, |
||||||
|
) |
||||||
|
|
||||||
|
@staticmethod |
||||||
|
def _parse_datetime(val: Any) -> Any: |
||||||
|
if val is None or isinstance(val, datetime): |
||||||
|
return val |
||||||
|
elif isinstance(val, str): |
||||||
|
return conv_from_str(val) |
||||||
|
else: |
||||||
|
raise PyDSParamException("Do not support value type %s for now", type(val)) |
||||||
|
|
||||||
|
@property |
||||||
|
def start_time(self) -> Any: |
||||||
|
"""Get attribute start_time.""" |
||||||
|
return self._parse_datetime(self._start_time) |
||||||
|
|
||||||
|
@start_time.setter |
||||||
|
def start_time(self, val) -> None: |
||||||
|
"""Set attribute start_time.""" |
||||||
|
self._start_time = val |
||||||
|
|
||||||
|
@property |
||||||
|
def end_time(self) -> Any: |
||||||
|
"""Get attribute end_time.""" |
||||||
|
return self._parse_datetime(self._end_time) |
||||||
|
|
||||||
|
@end_time.setter |
||||||
|
def end_time(self, val) -> None: |
||||||
|
"""Set attribute end_time.""" |
||||||
|
self._end_time = val |
||||||
|
|
||||||
|
@property |
||||||
|
def task_definition_json(self) -> List[Dict]: |
||||||
|
"""Return all tasks definition in list of dict.""" |
||||||
|
if not self.tasks: |
||||||
|
return [self.tasks] |
||||||
|
else: |
||||||
|
return [task.get_define() for task in self.tasks.values()] |
||||||
|
|
||||||
|
@property |
||||||
|
def task_relation_json(self) -> List[Dict]: |
||||||
|
"""Return all relation between tasks pair in list of dict.""" |
||||||
|
if not self.tasks: |
||||||
|
return [self.tasks] |
||||||
|
else: |
||||||
|
self._handle_root_relation() |
||||||
|
return [tr.get_define() for tr in self._task_relations] |
||||||
|
|
||||||
|
@property |
||||||
|
def schedule_json(self) -> Optional[Dict]: |
||||||
|
"""Get schedule parameter json object. This is requests from java gateway interface.""" |
||||||
|
if not self.schedule: |
||||||
|
return None |
||||||
|
else: |
||||||
|
start_time = conv_to_schedule( |
||||||
|
self.start_time if self.start_time else datetime.now() |
||||||
|
) |
||||||
|
end_time = conv_to_schedule( |
||||||
|
self.end_time if self.end_time else MAX_DATETIME |
||||||
|
) |
||||||
|
return { |
||||||
|
"startTime": start_time, |
||||||
|
"endTime": end_time, |
||||||
|
"crontab": self.schedule, |
||||||
|
"timezoneId": self.timezone, |
||||||
|
} |
||||||
|
|
||||||
|
# TODO inti DAG's tasks are in the same location with default {x: 0, y: 0} |
||||||
|
@property |
||||||
|
def task_location(self) -> List[Dict]: |
||||||
|
"""Return all tasks location for all process definition. |
||||||
|
|
||||||
|
For now, we only set all location with same x and y valued equal to 0. Because we do not |
||||||
|
find a good way to set task locations. This is requests from java gateway interface. |
||||||
|
""" |
||||||
|
if not self.tasks: |
||||||
|
return [self.tasks] |
||||||
|
else: |
||||||
|
return [{"taskCode": task_code, "x": 0, "y": 0} for task_code in self.tasks] |
||||||
|
|
||||||
|
@property |
||||||
|
def task_list(self) -> List["Task"]: # noqa: F821 |
||||||
|
"""Return list of tasks objects.""" |
||||||
|
return list(self.tasks.values()) |
||||||
|
|
||||||
|
def _handle_root_relation(self): |
||||||
|
"""Handle root task property :class:`pydolphinscheduler.core.task.TaskRelation`. |
||||||
|
|
||||||
|
Root task in DAG do not have dominant upstream node, but we have to add an exactly default |
||||||
|
upstream task with task_code equal to `0`. This is requests from java gateway interface. |
||||||
|
""" |
||||||
|
from pydolphinscheduler.core.task import TaskRelation |
||||||
|
|
||||||
|
post_relation_code = set() |
||||||
|
for relation in self._task_relations: |
||||||
|
post_relation_code.add(relation.post_task_code) |
||||||
|
for task in self.task_list: |
||||||
|
if task.code not in post_relation_code: |
||||||
|
root_relation = TaskRelation(pre_task_code=0, post_task_code=task.code) |
||||||
|
self._task_relations.add(root_relation) |
||||||
|
|
||||||
|
def add_task(self, task: "Task") -> None: # noqa: F821 |
||||||
|
"""Add a single task to process definition.""" |
||||||
|
self.tasks[task.code] = task |
||||||
|
task._process_definition = self |
||||||
|
|
||||||
|
def add_tasks(self, tasks: List["Task"]) -> None: # noqa: F821 |
||||||
|
"""Add task sequence to process definition, it a wrapper of :func:`add_task`.""" |
||||||
|
for task in tasks: |
||||||
|
self.add_task(task) |
||||||
|
|
||||||
|
def get_task(self, code: str) -> "Task": # noqa: F821 |
||||||
|
"""Get task object from process definition by given code.""" |
||||||
|
if code not in self.tasks: |
||||||
|
raise PyDSTaskNoFoundException( |
||||||
|
"Task with code %s can not found in process definition %", |
||||||
|
(code, self.name), |
||||||
|
) |
||||||
|
return self.tasks[code] |
||||||
|
|
||||||
|
# TODO which tying should return in this case |
||||||
|
def get_tasks_by_name(self, name: str) -> Set["Task"]: # noqa: F821 |
||||||
|
"""Get tasks object by given name, if will return all tasks with this name.""" |
||||||
|
find = set() |
||||||
|
for task in self.tasks.values(): |
||||||
|
if task.name == name: |
||||||
|
find.add(task) |
||||||
|
return find |
||||||
|
|
||||||
|
def get_one_task_by_name(self, name: str) -> "Task": # noqa: F821 |
||||||
|
"""Get exact one task from process definition by given name. |
||||||
|
|
||||||
|
Function always return one task even though this process definition have more than one task with |
||||||
|
this name. |
||||||
|
""" |
||||||
|
tasks = self.get_tasks_by_name(name) |
||||||
|
if not tasks: |
||||||
|
raise PyDSTaskNoFoundException(f"Can not find task with name {name}.") |
||||||
|
return tasks.pop() |
||||||
|
|
||||||
|
def run(self): |
||||||
|
"""Submit and Start ProcessDefinition instance. |
||||||
|
|
||||||
|
Shortcut for function :func:`submit` and function :func:`start`. Only support manual start workflow |
||||||
|
for now, and schedule run will coming soon. |
||||||
|
:return: |
||||||
|
""" |
||||||
|
self.submit() |
||||||
|
self.start() |
||||||
|
|
||||||
|
def _ensure_side_model_exists(self): |
||||||
|
"""Ensure process definition side model exists. |
||||||
|
|
||||||
|
For now, side object including :class:`pydolphinscheduler.side.project.Project`, |
||||||
|
:class:`pydolphinscheduler.side.tenant.Tenant`, :class:`pydolphinscheduler.side.user.User`. |
||||||
|
If these model not exists, would create default value in |
||||||
|
:class:`pydolphinscheduler.constants.ProcessDefinitionDefault`. |
||||||
|
""" |
||||||
|
# TODO used metaclass for more pythonic |
||||||
|
self.tenant.create_if_not_exists(self._queue) |
||||||
|
# model User have to create after Tenant created |
||||||
|
self.user.create_if_not_exists() |
||||||
|
# Project model need User object exists |
||||||
|
self.project.create_if_not_exists(self._user) |
||||||
|
|
||||||
|
def submit(self) -> int: |
||||||
|
"""Submit ProcessDefinition instance to java gateway.""" |
||||||
|
self._ensure_side_model_exists() |
||||||
|
gateway = launch_gateway() |
||||||
|
self._process_definition_code = gateway.entry_point.createOrUpdateProcessDefinition( |
||||||
|
self._user, |
||||||
|
self._project, |
||||||
|
self.name, |
||||||
|
str(self.description) if self.description else "", |
||||||
|
str(self.param) if self.param else None, |
||||||
|
json.dumps(self.schedule_json) if self.schedule_json else None, |
||||||
|
json.dumps(self.task_location), |
||||||
|
self.timeout, |
||||||
|
self.worker_group, |
||||||
|
self._tenant, |
||||||
|
# TODO add serialization function |
||||||
|
json.dumps(self.task_relation_json), |
||||||
|
json.dumps(self.task_definition_json), |
||||||
|
) |
||||||
|
return self._process_definition_code |
||||||
|
|
||||||
|
def start(self) -> None: |
||||||
|
"""Create and start ProcessDefinition instance. |
||||||
|
|
||||||
|
which post to `start-process-instance` to java gateway |
||||||
|
""" |
||||||
|
gateway = launch_gateway() |
||||||
|
gateway.entry_point.execProcessInstance( |
||||||
|
self._user, |
||||||
|
self._project, |
||||||
|
self.name, |
||||||
|
"", |
||||||
|
self.worker_group, |
||||||
|
24 * 3600, |
||||||
|
) |
@ -0,0 +1,268 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""DolphinScheduler Task and TaskRelation object.""" |
||||||
|
|
||||||
|
import logging |
||||||
|
from typing import Dict, List, Optional, Sequence, Set, Tuple, Union |
||||||
|
|
||||||
|
from pydolphinscheduler.constants import ( |
||||||
|
Delimiter, |
||||||
|
ProcessDefinitionDefault, |
||||||
|
TaskFlag, |
||||||
|
TaskPriority, |
||||||
|
TaskTimeoutFlag, |
||||||
|
) |
||||||
|
from pydolphinscheduler.core.base import Base |
||||||
|
from pydolphinscheduler.core.process_definition import ( |
||||||
|
ProcessDefinition, |
||||||
|
ProcessDefinitionContext, |
||||||
|
) |
||||||
|
from pydolphinscheduler.java_gateway import launch_gateway |
||||||
|
|
||||||
|
|
||||||
|
class TaskRelation(Base): |
||||||
|
"""TaskRelation object, describe the relation of exactly two tasks.""" |
||||||
|
|
||||||
|
# Add attr `_KEY_ATTR` to overwrite :func:`__eq__`, it is make set |
||||||
|
# `Task.process_definition._task_relations` work correctly. |
||||||
|
_KEY_ATTR = { |
||||||
|
"pre_task_code", |
||||||
|
"post_task_code", |
||||||
|
} |
||||||
|
|
||||||
|
_DEFINE_ATTR = { |
||||||
|
"pre_task_code", |
||||||
|
"post_task_code", |
||||||
|
} |
||||||
|
|
||||||
|
_DEFAULT_ATTR = { |
||||||
|
"name": "", |
||||||
|
"preTaskVersion": 1, |
||||||
|
"postTaskVersion": 1, |
||||||
|
"conditionType": 0, |
||||||
|
"conditionParams": {}, |
||||||
|
} |
||||||
|
|
||||||
|
def __init__( |
||||||
|
self, |
||||||
|
pre_task_code: int, |
||||||
|
post_task_code: int, |
||||||
|
name: Optional[str] = None, |
||||||
|
): |
||||||
|
super().__init__(name) |
||||||
|
self.pre_task_code = pre_task_code |
||||||
|
self.post_task_code = post_task_code |
||||||
|
|
||||||
|
def __hash__(self): |
||||||
|
return hash(f"{self.pre_task_code} {Delimiter.DIRECTION} {self.post_task_code}") |
||||||
|
|
||||||
|
|
||||||
|
class Task(Base): |
||||||
|
"""Task object, parent class for all exactly task type.""" |
||||||
|
|
||||||
|
_DEFINE_ATTR = { |
||||||
|
"name", |
||||||
|
"code", |
||||||
|
"version", |
||||||
|
"task_type", |
||||||
|
"task_params", |
||||||
|
"description", |
||||||
|
"flag", |
||||||
|
"task_priority", |
||||||
|
"worker_group", |
||||||
|
"delay_time", |
||||||
|
"fail_retry_times", |
||||||
|
"fail_retry_interval", |
||||||
|
"timeout_flag", |
||||||
|
"timeout_notify_strategy", |
||||||
|
"timeout", |
||||||
|
} |
||||||
|
|
||||||
|
_task_custom_attr: set = set() |
||||||
|
|
||||||
|
DEFAULT_CONDITION_RESULT = {"successNode": [""], "failedNode": [""]} |
||||||
|
|
||||||
|
def __init__( |
||||||
|
self, |
||||||
|
name: str, |
||||||
|
task_type: str, |
||||||
|
description: Optional[str] = None, |
||||||
|
flag: Optional[str] = TaskFlag.YES, |
||||||
|
task_priority: Optional[str] = TaskPriority.MEDIUM, |
||||||
|
worker_group: Optional[str] = ProcessDefinitionDefault.WORKER_GROUP, |
||||||
|
delay_time: Optional[int] = 0, |
||||||
|
fail_retry_times: Optional[int] = 0, |
||||||
|
fail_retry_interval: Optional[int] = 1, |
||||||
|
timeout_flag: Optional[int] = TaskTimeoutFlag.CLOSE, |
||||||
|
timeout_notify_strategy: Optional = None, |
||||||
|
timeout: Optional[int] = 0, |
||||||
|
process_definition: Optional[ProcessDefinition] = None, |
||||||
|
local_params: Optional[List] = None, |
||||||
|
resource_list: Optional[List] = None, |
||||||
|
dependence: Optional[Dict] = None, |
||||||
|
wait_start_timeout: Optional[Dict] = None, |
||||||
|
condition_result: Optional[Dict] = None, |
||||||
|
): |
||||||
|
|
||||||
|
super().__init__(name, description) |
||||||
|
self.task_type = task_type |
||||||
|
self.flag = flag |
||||||
|
self.task_priority = task_priority |
||||||
|
self.worker_group = worker_group |
||||||
|
self.fail_retry_times = fail_retry_times |
||||||
|
self.fail_retry_interval = fail_retry_interval |
||||||
|
self.delay_time = delay_time |
||||||
|
self.timeout_flag = timeout_flag |
||||||
|
self.timeout_notify_strategy = timeout_notify_strategy |
||||||
|
self.timeout = timeout |
||||||
|
self._process_definition = None |
||||||
|
self.process_definition: ProcessDefinition = ( |
||||||
|
process_definition or ProcessDefinitionContext.get() |
||||||
|
) |
||||||
|
self._upstream_task_codes: Set[int] = set() |
||||||
|
self._downstream_task_codes: Set[int] = set() |
||||||
|
self._task_relation: Set[TaskRelation] = set() |
||||||
|
# move attribute code and version after _process_definition and process_definition declare |
||||||
|
self.code, self.version = self.gen_code_and_version() |
||||||
|
# Add task to process definition, maybe we could put into property process_definition latter |
||||||
|
if ( |
||||||
|
self.process_definition is not None |
||||||
|
and self.code not in self.process_definition.tasks |
||||||
|
): |
||||||
|
self.process_definition.add_task(self) |
||||||
|
else: |
||||||
|
logging.warning( |
||||||
|
"Task code %d already in process definition, prohibit re-add task.", |
||||||
|
self.code, |
||||||
|
) |
||||||
|
|
||||||
|
# Attribute for task param |
||||||
|
self.local_params = local_params or [] |
||||||
|
self.resource_list = resource_list or [] |
||||||
|
self.dependence = dependence or {} |
||||||
|
self.wait_start_timeout = wait_start_timeout or {} |
||||||
|
self.condition_result = condition_result or self.DEFAULT_CONDITION_RESULT |
||||||
|
|
||||||
|
@property |
||||||
|
def process_definition(self) -> Optional[ProcessDefinition]: |
||||||
|
"""Get attribute process_definition.""" |
||||||
|
return self._process_definition |
||||||
|
|
||||||
|
@process_definition.setter |
||||||
|
def process_definition(self, process_definition: Optional[ProcessDefinition]): |
||||||
|
"""Set attribute process_definition.""" |
||||||
|
self._process_definition = process_definition |
||||||
|
|
||||||
|
@property |
||||||
|
def task_params(self) -> Optional[Dict]: |
||||||
|
"""Get task parameter object. |
||||||
|
|
||||||
|
Will get result to combine _task_custom_attr and custom_attr. |
||||||
|
""" |
||||||
|
custom_attr = { |
||||||
|
"local_params", |
||||||
|
"resource_list", |
||||||
|
"dependence", |
||||||
|
"wait_start_timeout", |
||||||
|
"condition_result", |
||||||
|
} |
||||||
|
custom_attr |= self._task_custom_attr |
||||||
|
return self.get_define_custom(custom_attr=custom_attr) |
||||||
|
|
||||||
|
def __hash__(self): |
||||||
|
return hash(self.code) |
||||||
|
|
||||||
|
def __lshift__(self, other: Union["Task", Sequence["Task"]]): |
||||||
|
"""Implement Task << Task.""" |
||||||
|
self.set_upstream(other) |
||||||
|
return other |
||||||
|
|
||||||
|
def __rshift__(self, other: Union["Task", Sequence["Task"]]): |
||||||
|
"""Implement Task >> Task.""" |
||||||
|
self.set_downstream(other) |
||||||
|
return other |
||||||
|
|
||||||
|
def __rrshift__(self, other: Union["Task", Sequence["Task"]]): |
||||||
|
"""Call for Task >> [Task] because list don't have __rshift__ operators.""" |
||||||
|
self.__lshift__(other) |
||||||
|
return self |
||||||
|
|
||||||
|
def __rlshift__(self, other: Union["Task", Sequence["Task"]]): |
||||||
|
"""Call for Task << [Task] because list don't have __lshift__ operators.""" |
||||||
|
self.__rshift__(other) |
||||||
|
return self |
||||||
|
|
||||||
|
def _set_deps( |
||||||
|
self, tasks: Union["Task", Sequence["Task"]], upstream: bool = True |
||||||
|
) -> None: |
||||||
|
""" |
||||||
|
Set parameter tasks dependent to current task. |
||||||
|
|
||||||
|
it is a wrapper for :func:`set_upstream` and :func:`set_downstream`. |
||||||
|
""" |
||||||
|
if not isinstance(tasks, Sequence): |
||||||
|
tasks = [tasks] |
||||||
|
|
||||||
|
for task in tasks: |
||||||
|
if upstream: |
||||||
|
self._upstream_task_codes.add(task.code) |
||||||
|
task._downstream_task_codes.add(self.code) |
||||||
|
|
||||||
|
if self._process_definition: |
||||||
|
task_relation = TaskRelation( |
||||||
|
pre_task_code=task.code, |
||||||
|
post_task_code=self.code, |
||||||
|
name=f"{task.name} {Delimiter.DIRECTION} {self.name}", |
||||||
|
) |
||||||
|
self.process_definition._task_relations.add(task_relation) |
||||||
|
else: |
||||||
|
self._downstream_task_codes.add(task.code) |
||||||
|
task._upstream_task_codes.add(self.code) |
||||||
|
|
||||||
|
if self._process_definition: |
||||||
|
task_relation = TaskRelation( |
||||||
|
pre_task_code=self.code, |
||||||
|
post_task_code=task.code, |
||||||
|
name=f"{self.name} {Delimiter.DIRECTION} {task.name}", |
||||||
|
) |
||||||
|
self.process_definition._task_relations.add(task_relation) |
||||||
|
|
||||||
|
def set_upstream(self, tasks: Union["Task", Sequence["Task"]]) -> None: |
||||||
|
"""Set parameter tasks as upstream to current task.""" |
||||||
|
self._set_deps(tasks, upstream=True) |
||||||
|
|
||||||
|
def set_downstream(self, tasks: Union["Task", Sequence["Task"]]) -> None: |
||||||
|
"""Set parameter tasks as downstream to current task.""" |
||||||
|
self._set_deps(tasks, upstream=False) |
||||||
|
|
||||||
|
# TODO code should better generate in bulk mode when :ref: processDefinition run submit or start |
||||||
|
def gen_code_and_version(self) -> Tuple: |
||||||
|
""" |
||||||
|
Generate task code and version from java gateway. |
||||||
|
|
||||||
|
If task name do not exists in process definition before, if will generate new code and version id |
||||||
|
equal to 0 by java gateway, otherwise if will return the exists code and version. |
||||||
|
""" |
||||||
|
# TODO get code from specific project process definition and task name |
||||||
|
gateway = launch_gateway() |
||||||
|
result = gateway.entry_point.getCodeAndVersion( |
||||||
|
self.process_definition._project, self.name |
||||||
|
) |
||||||
|
# result = gateway.entry_point.genTaskCodeList(DefaultTaskCodeNum.DEFAULT) |
||||||
|
# gateway_result_checker(result) |
||||||
|
return result.get("code"), result.get("version") |
@ -0,0 +1,46 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""Exceptions for pydolphinscheduler.""" |
||||||
|
|
||||||
|
|
||||||
|
class PyDSBaseException(Exception): |
||||||
|
"""Base exception for pydolphinscheduler.""" |
||||||
|
|
||||||
|
pass |
||||||
|
|
||||||
|
|
||||||
|
class PyDSParamException(PyDSBaseException): |
||||||
|
"""Exception for pydolphinscheduler parameter verify error.""" |
||||||
|
|
||||||
|
pass |
||||||
|
|
||||||
|
|
||||||
|
class PyDSTaskNoFoundException(PyDSBaseException): |
||||||
|
"""Exception for pydolphinscheduler workflow task no found error.""" |
||||||
|
|
||||||
|
pass |
||||||
|
|
||||||
|
|
||||||
|
class PyDSJavaGatewayException(PyDSBaseException): |
||||||
|
"""Exception for pydolphinscheduler Java gateway error.""" |
||||||
|
|
||||||
|
pass |
||||||
|
|
||||||
|
|
||||||
|
class PyDSProcessDefinitionNotAssignException(PyDSBaseException): |
||||||
|
"""Exception for pydolphinscheduler process definition not assign error.""" |
@ -0,0 +1,55 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""Module java gateway, contain gateway behavior.""" |
||||||
|
|
||||||
|
from typing import Any, Optional |
||||||
|
|
||||||
|
from py4j.java_collections import JavaMap |
||||||
|
from py4j.java_gateway import GatewayParameters, JavaGateway |
||||||
|
|
||||||
|
from pydolphinscheduler.constants import JavaGatewayDefault |
||||||
|
from pydolphinscheduler.exceptions import PyDSJavaGatewayException |
||||||
|
|
||||||
|
|
||||||
|
def launch_gateway() -> JavaGateway: |
||||||
|
"""Launch java gateway to pydolphinscheduler. |
||||||
|
|
||||||
|
TODO Note that automatic conversion makes calling Java methods slightly less efficient because |
||||||
|
in the worst case, Py4J needs to go through all registered converters for all parameters. |
||||||
|
This is why automatic conversion is disabled by default. |
||||||
|
""" |
||||||
|
gateway = JavaGateway(gateway_parameters=GatewayParameters(auto_convert=True)) |
||||||
|
return gateway |
||||||
|
|
||||||
|
|
||||||
|
def gateway_result_checker( |
||||||
|
result: JavaMap, |
||||||
|
msg_check: Optional[str] = JavaGatewayDefault.RESULT_MESSAGE_SUCCESS, |
||||||
|
) -> Any: |
||||||
|
"""Check weather java gateway result success or not.""" |
||||||
|
if ( |
||||||
|
result[JavaGatewayDefault.RESULT_STATUS_KEYWORD].toString() |
||||||
|
!= JavaGatewayDefault.RESULT_STATUS_SUCCESS |
||||||
|
): |
||||||
|
raise PyDSJavaGatewayException("Failed when try to got result for java gateway") |
||||||
|
if ( |
||||||
|
msg_check is not None |
||||||
|
and result[JavaGatewayDefault.RESULT_MESSAGE_KEYWORD] != msg_check |
||||||
|
): |
||||||
|
raise PyDSJavaGatewayException("Get result state not success.") |
||||||
|
return result |
@ -0,0 +1,22 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""Init Side package, Side package keep object related to DolphinScheduler but not in the Core part.""" |
||||||
|
|
||||||
|
from pydolphinscheduler.side.project import Project |
||||||
|
from pydolphinscheduler.side.tenant import Tenant |
||||||
|
from pydolphinscheduler.side.user import User |
@ -0,0 +1,42 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""DolphinScheduler Project object.""" |
||||||
|
|
||||||
|
from typing import Optional |
||||||
|
|
||||||
|
from pydolphinscheduler.constants import ProcessDefinitionDefault |
||||||
|
from pydolphinscheduler.core.base_side import BaseSide |
||||||
|
from pydolphinscheduler.java_gateway import launch_gateway |
||||||
|
|
||||||
|
|
||||||
|
class Project(BaseSide): |
||||||
|
"""DolphinScheduler Project object.""" |
||||||
|
|
||||||
|
def __init__( |
||||||
|
self, |
||||||
|
name: str = ProcessDefinitionDefault.PROJECT, |
||||||
|
description: Optional[str] = None, |
||||||
|
): |
||||||
|
super().__init__(name, description) |
||||||
|
|
||||||
|
def create_if_not_exists(self, user=ProcessDefinitionDefault.USER) -> None: |
||||||
|
"""Create Project if not exists.""" |
||||||
|
gateway = launch_gateway() |
||||||
|
gateway.entry_point.createProject(user, self.name, self.description) |
||||||
|
# TODO recover result checker |
||||||
|
# gateway_result_checker(result, None) |
@ -0,0 +1,42 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""DolphinScheduler User object.""" |
||||||
|
|
||||||
|
from typing import Optional |
||||||
|
|
||||||
|
from pydolphinscheduler.constants import ProcessDefinitionDefault |
||||||
|
from pydolphinscheduler.core.base_side import BaseSide |
||||||
|
from pydolphinscheduler.java_gateway import gateway_result_checker, launch_gateway |
||||||
|
|
||||||
|
|
||||||
|
class Queue(BaseSide): |
||||||
|
"""DolphinScheduler Queue object.""" |
||||||
|
|
||||||
|
def __init__( |
||||||
|
self, |
||||||
|
name: str = ProcessDefinitionDefault.QUEUE, |
||||||
|
description: Optional[str] = "", |
||||||
|
): |
||||||
|
super().__init__(name, description) |
||||||
|
|
||||||
|
def create_if_not_exists(self, user=ProcessDefinitionDefault.USER) -> None: |
||||||
|
"""Create Queue if not exists.""" |
||||||
|
gateway = launch_gateway() |
||||||
|
# Here we set Queue.name and Queue.queueName same as self.name |
||||||
|
result = gateway.entry_point.createProject(user, self.name, self.name) |
||||||
|
gateway_result_checker(result, None) |
@ -0,0 +1,45 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""DolphinScheduler Tenant object.""" |
||||||
|
|
||||||
|
from typing import Optional |
||||||
|
|
||||||
|
from pydolphinscheduler.constants import ProcessDefinitionDefault |
||||||
|
from pydolphinscheduler.core.base_side import BaseSide |
||||||
|
from pydolphinscheduler.java_gateway import launch_gateway |
||||||
|
|
||||||
|
|
||||||
|
class Tenant(BaseSide): |
||||||
|
"""DolphinScheduler Tenant object.""" |
||||||
|
|
||||||
|
def __init__( |
||||||
|
self, |
||||||
|
name: str = ProcessDefinitionDefault.TENANT, |
||||||
|
queue: str = ProcessDefinitionDefault.QUEUE, |
||||||
|
description: Optional[str] = None, |
||||||
|
): |
||||||
|
super().__init__(name, description) |
||||||
|
self.queue = queue |
||||||
|
|
||||||
|
def create_if_not_exists( |
||||||
|
self, queue_name: str, user=ProcessDefinitionDefault.USER |
||||||
|
) -> None: |
||||||
|
"""Create Tenant if not exists.""" |
||||||
|
gateway = launch_gateway() |
||||||
|
gateway.entry_point.createTenant(self.name, self.description, queue_name) |
||||||
|
# gateway_result_checker(result, None) |
@ -0,0 +1,70 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""DolphinScheduler User object.""" |
||||||
|
|
||||||
|
from typing import Optional |
||||||
|
|
||||||
|
from pydolphinscheduler.core.base_side import BaseSide |
||||||
|
from pydolphinscheduler.java_gateway import launch_gateway |
||||||
|
|
||||||
|
|
||||||
|
class User(BaseSide): |
||||||
|
"""DolphinScheduler User object.""" |
||||||
|
|
||||||
|
_KEY_ATTR = { |
||||||
|
"name", |
||||||
|
"password", |
||||||
|
"email", |
||||||
|
"phone", |
||||||
|
"tenant", |
||||||
|
"queue", |
||||||
|
"status", |
||||||
|
} |
||||||
|
|
||||||
|
def __init__( |
||||||
|
self, |
||||||
|
name: str, |
||||||
|
password: str, |
||||||
|
email: str, |
||||||
|
phone: str, |
||||||
|
tenant: str, |
||||||
|
queue: Optional[str] = None, |
||||||
|
status: Optional[int] = 1, |
||||||
|
): |
||||||
|
super().__init__(name) |
||||||
|
self.password = password |
||||||
|
self.email = email |
||||||
|
self.phone = phone |
||||||
|
self.tenant = tenant |
||||||
|
self.queue = queue |
||||||
|
self.status = status |
||||||
|
|
||||||
|
def create_if_not_exists(self, **kwargs): |
||||||
|
"""Create User if not exists.""" |
||||||
|
gateway = launch_gateway() |
||||||
|
gateway.entry_point.createUser( |
||||||
|
self.name, |
||||||
|
self.password, |
||||||
|
self.email, |
||||||
|
self.phone, |
||||||
|
self.tenant, |
||||||
|
self.queue, |
||||||
|
self.status, |
||||||
|
) |
||||||
|
# TODO recover result checker |
||||||
|
# gateway_result_checker(result, None) |
@ -0,0 +1,30 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""DolphinScheduler Worker Group object.""" |
||||||
|
|
||||||
|
from typing import Optional |
||||||
|
|
||||||
|
from pydolphinscheduler.core.base_side import BaseSide |
||||||
|
|
||||||
|
|
||||||
|
class WorkerGroup(BaseSide): |
||||||
|
"""DolphinScheduler Worker Group object.""" |
||||||
|
|
||||||
|
def __init__(self, name: str, address: str, description: Optional[str] = None): |
||||||
|
super().__init__(name, description) |
||||||
|
self.address = address |
@ -0,0 +1,18 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""Init pydolphinscheduler.tasks package.""" |
@ -0,0 +1,185 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""Task Conditions.""" |
||||||
|
|
||||||
|
from typing import Dict, List |
||||||
|
|
||||||
|
from pydolphinscheduler.constants import TaskType |
||||||
|
from pydolphinscheduler.core.base import Base |
||||||
|
from pydolphinscheduler.core.task import Task |
||||||
|
from pydolphinscheduler.exceptions import PyDSParamException |
||||||
|
|
||||||
|
|
||||||
|
class Status(Base): |
||||||
|
"""Base class of Condition task status. |
||||||
|
|
||||||
|
It a parent class for :class:`SUCCESS` and :class:`FAILURE`. Provider status name |
||||||
|
and :func:`get_define` to sub class. |
||||||
|
""" |
||||||
|
|
||||||
|
def __init__(self, *tasks): |
||||||
|
super().__init__(f"Condition.{self.status_name()}") |
||||||
|
self.tasks = tasks |
||||||
|
|
||||||
|
def __repr__(self) -> str: |
||||||
|
return "depend_item_list" |
||||||
|
|
||||||
|
@classmethod |
||||||
|
def status_name(cls) -> str: |
||||||
|
"""Get name for Status or its sub class.""" |
||||||
|
return cls.__name__.upper() |
||||||
|
|
||||||
|
def get_define(self, camel_attr: bool = True) -> List: |
||||||
|
"""Get status definition attribute communicate to Java gateway server.""" |
||||||
|
content = [] |
||||||
|
for task in self.tasks: |
||||||
|
if not isinstance(task, Task): |
||||||
|
raise PyDSParamException( |
||||||
|
"%s only accept class Task or sub class Task, but get %s", |
||||||
|
(self.status_name(), type(task)), |
||||||
|
) |
||||||
|
content.append({"depTaskCode": task.code, "status": self.status_name()}) |
||||||
|
return content |
||||||
|
|
||||||
|
|
||||||
|
class SUCCESS(Status): |
||||||
|
"""Class SUCCESS to task condition, sub class of :class:`Status`.""" |
||||||
|
|
||||||
|
def __init__(self, *tasks): |
||||||
|
super().__init__(*tasks) |
||||||
|
|
||||||
|
|
||||||
|
class FAILURE(Status): |
||||||
|
"""Class FAILURE to task condition, sub class of :class:`Status`.""" |
||||||
|
|
||||||
|
def __init__(self, *tasks): |
||||||
|
super().__init__(*tasks) |
||||||
|
|
||||||
|
|
||||||
|
class ConditionOperator(Base): |
||||||
|
"""Set ConditionTask or ConditionOperator with specific operator.""" |
||||||
|
|
||||||
|
_DEFINE_ATTR = { |
||||||
|
"relation", |
||||||
|
} |
||||||
|
|
||||||
|
def __init__(self, *args): |
||||||
|
super().__init__(self.__class__.__name__) |
||||||
|
self.args = args |
||||||
|
|
||||||
|
def __repr__(self) -> str: |
||||||
|
return "depend_task_list" |
||||||
|
|
||||||
|
@classmethod |
||||||
|
def operator_name(cls) -> str: |
||||||
|
"""Get operator name in different class.""" |
||||||
|
return cls.__name__.upper() |
||||||
|
|
||||||
|
@property |
||||||
|
def relation(self) -> str: |
||||||
|
"""Get operator name in different class, for function :func:`get_define`.""" |
||||||
|
return self.operator_name() |
||||||
|
|
||||||
|
def set_define_attr(self) -> str: |
||||||
|
"""Set attribute to function :func:`get_define`. |
||||||
|
|
||||||
|
It is a wrapper for both `And` and `Or` operator. |
||||||
|
""" |
||||||
|
result = [] |
||||||
|
attr = None |
||||||
|
for condition in self.args: |
||||||
|
if isinstance(condition, (Status, ConditionOperator)): |
||||||
|
if attr is None: |
||||||
|
attr = repr(condition) |
||||||
|
elif repr(condition) != attr: |
||||||
|
raise PyDSParamException( |
||||||
|
"Condition %s operator parameter only support same type.", |
||||||
|
self.relation, |
||||||
|
) |
||||||
|
else: |
||||||
|
raise PyDSParamException( |
||||||
|
"Condition %s operator parameter support ConditionTask and ConditionOperator but got %s.", |
||||||
|
(self.relation, type(condition)), |
||||||
|
) |
||||||
|
if attr == "depend_item_list": |
||||||
|
result.extend(condition.get_define()) |
||||||
|
else: |
||||||
|
result.append(condition.get_define()) |
||||||
|
setattr(self, attr, result) |
||||||
|
return attr |
||||||
|
|
||||||
|
def get_define(self, camel_attr=True) -> Dict: |
||||||
|
"""Overwrite Base.get_define to get task Condition specific get define.""" |
||||||
|
attr = self.set_define_attr() |
||||||
|
dependent_define_attr = self._DEFINE_ATTR.union({attr}) |
||||||
|
return super().get_define_custom( |
||||||
|
camel_attr=True, custom_attr=dependent_define_attr |
||||||
|
) |
||||||
|
|
||||||
|
|
||||||
|
class And(ConditionOperator): |
||||||
|
"""Operator And for task condition. |
||||||
|
|
||||||
|
It could accept both :class:`Task` and children of :class:`ConditionOperator`, |
||||||
|
and set AND condition to those args. |
||||||
|
""" |
||||||
|
|
||||||
|
def __init__(self, *args): |
||||||
|
super().__init__(*args) |
||||||
|
|
||||||
|
|
||||||
|
class Or(ConditionOperator): |
||||||
|
"""Operator Or for task condition. |
||||||
|
|
||||||
|
It could accept both :class:`Task` and children of :class:`ConditionOperator`, |
||||||
|
and set OR condition to those args. |
||||||
|
""" |
||||||
|
|
||||||
|
def __init__(self, *args): |
||||||
|
super().__init__(*args) |
||||||
|
|
||||||
|
|
||||||
|
class Conditions(Task): |
||||||
|
"""Task condition object, declare behavior for condition task to dolphinscheduler.""" |
||||||
|
|
||||||
|
def __init__(self, name: str, condition: ConditionOperator, *args, **kwargs): |
||||||
|
super().__init__(name, TaskType.CONDITIONS, *args, **kwargs) |
||||||
|
self.condition = condition |
||||||
|
# Set condition tasks as current task downstream |
||||||
|
self._set_dep() |
||||||
|
|
||||||
|
def _set_dep(self) -> None: |
||||||
|
"""Set downstream according to parameter `condition`.""" |
||||||
|
downstream = [] |
||||||
|
for cond in self.condition.args: |
||||||
|
if isinstance(cond, ConditionOperator): |
||||||
|
for status in cond.args: |
||||||
|
downstream.extend(list(status.tasks)) |
||||||
|
self.set_downstream(downstream) |
||||||
|
|
||||||
|
@property |
||||||
|
def task_params(self, camel_attr: bool = True, custom_attr: set = None) -> Dict: |
||||||
|
"""Override Task.task_params for Condition task. |
||||||
|
|
||||||
|
Condition task have some specials attribute `dependence`, and in most of the task |
||||||
|
this attribute is None and use empty dict `{}` as default value. We do not use class |
||||||
|
attribute `_task_custom_attr` due to avoid attribute cover. |
||||||
|
""" |
||||||
|
params = super().task_params |
||||||
|
params["dependence"] = self.condition.get_define() |
||||||
|
return params |
@ -0,0 +1,121 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""Task datax.""" |
||||||
|
|
||||||
|
from typing import Dict, List, Optional |
||||||
|
|
||||||
|
from pydolphinscheduler.constants import TaskType |
||||||
|
from pydolphinscheduler.core.database import Database |
||||||
|
from pydolphinscheduler.core.task import Task |
||||||
|
|
||||||
|
|
||||||
|
class CustomDataX(Task): |
||||||
|
"""Task CustomDatax object, declare behavior for custom DataX task to dolphinscheduler. |
||||||
|
|
||||||
|
You provider json template for DataX, it can synchronize data according to the template you provided. |
||||||
|
""" |
||||||
|
|
||||||
|
CUSTOM_CONFIG = 1 |
||||||
|
|
||||||
|
_task_custom_attr = {"custom_config", "json", "xms", "xmx"} |
||||||
|
|
||||||
|
def __init__( |
||||||
|
self, |
||||||
|
name: str, |
||||||
|
json: str, |
||||||
|
xms: Optional[int] = 1, |
||||||
|
xmx: Optional[int] = 1, |
||||||
|
*args, |
||||||
|
**kwargs |
||||||
|
): |
||||||
|
super().__init__(name, TaskType.DATAX, *args, **kwargs) |
||||||
|
self.custom_config = self.CUSTOM_CONFIG |
||||||
|
self.json = json |
||||||
|
self.xms = xms |
||||||
|
self.xmx = xmx |
||||||
|
|
||||||
|
|
||||||
|
class DataX(Task): |
||||||
|
"""Task DataX object, declare behavior for DataX task to dolphinscheduler. |
||||||
|
|
||||||
|
It should run database datax job in multiply sql link engine, such as: |
||||||
|
- MySQL |
||||||
|
- Oracle |
||||||
|
- Postgresql |
||||||
|
- SQLServer |
||||||
|
You provider datasource_name and datatarget_name contain connection information, it decisions which |
||||||
|
database type and database instance would synchronous data. |
||||||
|
""" |
||||||
|
|
||||||
|
CUSTOM_CONFIG = 0 |
||||||
|
|
||||||
|
_task_custom_attr = { |
||||||
|
"custom_config", |
||||||
|
"sql", |
||||||
|
"target_table", |
||||||
|
"job_speed_byte", |
||||||
|
"job_speed_record", |
||||||
|
"pre_statements", |
||||||
|
"post_statements", |
||||||
|
"xms", |
||||||
|
"xmx", |
||||||
|
} |
||||||
|
|
||||||
|
def __init__( |
||||||
|
self, |
||||||
|
name: str, |
||||||
|
datasource_name: str, |
||||||
|
datatarget_name: str, |
||||||
|
sql: str, |
||||||
|
target_table: str, |
||||||
|
job_speed_byte: Optional[int] = 0, |
||||||
|
job_speed_record: Optional[int] = 1000, |
||||||
|
pre_statements: Optional[List[str]] = None, |
||||||
|
post_statements: Optional[List[str]] = None, |
||||||
|
xms: Optional[int] = 1, |
||||||
|
xmx: Optional[int] = 1, |
||||||
|
*args, |
||||||
|
**kwargs |
||||||
|
): |
||||||
|
super().__init__(name, TaskType.DATAX, *args, **kwargs) |
||||||
|
self.sql = sql |
||||||
|
self.custom_config = self.CUSTOM_CONFIG |
||||||
|
self.datasource_name = datasource_name |
||||||
|
self.datatarget_name = datatarget_name |
||||||
|
self.target_table = target_table |
||||||
|
self.job_speed_byte = job_speed_byte |
||||||
|
self.job_speed_record = job_speed_record |
||||||
|
self.pre_statements = pre_statements or [] |
||||||
|
self.post_statements = post_statements or [] |
||||||
|
self.xms = xms |
||||||
|
self.xmx = xmx |
||||||
|
|
||||||
|
@property |
||||||
|
def task_params(self, camel_attr: bool = True, custom_attr: set = None) -> Dict: |
||||||
|
"""Override Task.task_params for datax task. |
||||||
|
|
||||||
|
datax task have some specials attribute for task_params, and is odd if we |
||||||
|
directly set as python property, so we Override Task.task_params here. |
||||||
|
""" |
||||||
|
params = super().task_params |
||||||
|
datasource = Database(self.datasource_name, "dsType", "dataSource") |
||||||
|
params.update(datasource) |
||||||
|
|
||||||
|
datatarget = Database(self.datatarget_name, "dtType", "dataTarget") |
||||||
|
params.update(datatarget) |
||||||
|
return params |
@ -0,0 +1,274 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""Task dependent.""" |
||||||
|
|
||||||
|
from typing import Dict, Optional, Tuple |
||||||
|
|
||||||
|
from pydolphinscheduler.constants import TaskType |
||||||
|
from pydolphinscheduler.core.base import Base |
||||||
|
from pydolphinscheduler.core.task import Task |
||||||
|
from pydolphinscheduler.exceptions import PyDSJavaGatewayException, PyDSParamException |
||||||
|
from pydolphinscheduler.java_gateway import launch_gateway |
||||||
|
|
||||||
|
DEPENDENT_ALL_TASK_IN_WORKFLOW = "0" |
||||||
|
|
||||||
|
|
||||||
|
class DependentDate(str): |
||||||
|
"""Constant of Dependent date value. |
||||||
|
|
||||||
|
These values set according to Java server side, if you want to add and change it, |
||||||
|
please change Java server side first. |
||||||
|
""" |
||||||
|
|
||||||
|
# TODO Maybe we should add parent level to DependentDate for easy to use, such as |
||||||
|
# DependentDate.MONTH.THIS_MONTH |
||||||
|
|
||||||
|
# Hour |
||||||
|
CURRENT_HOUR = "currentHour" |
||||||
|
LAST_ONE_HOUR = "last1Hour" |
||||||
|
LAST_TWO_HOURS = "last2Hours" |
||||||
|
LAST_THREE_HOURS = "last3Hours" |
||||||
|
LAST_TWENTY_FOUR_HOURS = "last24Hours" |
||||||
|
|
||||||
|
# Day |
||||||
|
TODAY = "today" |
||||||
|
LAST_ONE_DAYS = "last1Days" |
||||||
|
LAST_TWO_DAYS = "last2Days" |
||||||
|
LAST_THREE_DAYS = "last3Days" |
||||||
|
LAST_SEVEN_DAYS = "last7Days" |
||||||
|
|
||||||
|
# Week |
||||||
|
THIS_WEEK = "thisWeek" |
||||||
|
LAST_WEEK = "lastWeek" |
||||||
|
LAST_MONDAY = "lastMonday" |
||||||
|
LAST_TUESDAY = "lastTuesday" |
||||||
|
LAST_WEDNESDAY = "lastWednesday" |
||||||
|
LAST_THURSDAY = "lastThursday" |
||||||
|
LAST_FRIDAY = "lastFriday" |
||||||
|
LAST_SATURDAY = "lastSaturday" |
||||||
|
LAST_SUNDAY = "lastSunday" |
||||||
|
|
||||||
|
# Month |
||||||
|
THIS_MONTH = "thisMonth" |
||||||
|
LAST_MONTH = "lastMonth" |
||||||
|
LAST_MONTH_BEGIN = "lastMonthBegin" |
||||||
|
LAST_MONTH_END = "lastMonthEnd" |
||||||
|
|
||||||
|
|
||||||
|
class DependentItem(Base): |
||||||
|
"""Dependent item object, minimal unit for task dependent. |
||||||
|
|
||||||
|
It declare which project, process_definition, task are dependent to this task. |
||||||
|
""" |
||||||
|
|
||||||
|
_DEFINE_ATTR = { |
||||||
|
"project_code", |
||||||
|
"definition_code", |
||||||
|
"dep_task_code", |
||||||
|
"cycle", |
||||||
|
"date_value", |
||||||
|
} |
||||||
|
|
||||||
|
# TODO maybe we should conside overwrite operator `and` and `or` for DependentItem to |
||||||
|
# support more easy way to set relation |
||||||
|
def __init__( |
||||||
|
self, |
||||||
|
project_name: str, |
||||||
|
process_definition_name: str, |
||||||
|
dependent_task_name: Optional[str] = DEPENDENT_ALL_TASK_IN_WORKFLOW, |
||||||
|
dependent_date: Optional[DependentDate] = DependentDate.TODAY, |
||||||
|
): |
||||||
|
obj_name = f"{project_name}.{process_definition_name}.{dependent_task_name}.{dependent_date}" |
||||||
|
super().__init__(obj_name) |
||||||
|
self.project_name = project_name |
||||||
|
self.process_definition_name = process_definition_name |
||||||
|
self.dependent_task_name = dependent_task_name |
||||||
|
if dependent_date is None: |
||||||
|
raise PyDSParamException( |
||||||
|
"Parameter dependent_date must provider by got None." |
||||||
|
) |
||||||
|
else: |
||||||
|
self.dependent_date = dependent_date |
||||||
|
self._code = {} |
||||||
|
|
||||||
|
def __repr__(self) -> str: |
||||||
|
return "depend_item_list" |
||||||
|
|
||||||
|
@property |
||||||
|
def project_code(self) -> str: |
||||||
|
"""Get dependent project code.""" |
||||||
|
return self.get_code_from_gateway().get("projectCode") |
||||||
|
|
||||||
|
@property |
||||||
|
def definition_code(self) -> str: |
||||||
|
"""Get dependent definition code.""" |
||||||
|
return self.get_code_from_gateway().get("processDefinitionCode") |
||||||
|
|
||||||
|
@property |
||||||
|
def dep_task_code(self) -> str: |
||||||
|
"""Get dependent tasks code list.""" |
||||||
|
if self.is_all_task: |
||||||
|
return DEPENDENT_ALL_TASK_IN_WORKFLOW |
||||||
|
else: |
||||||
|
return self.get_code_from_gateway().get("taskDefinitionCode") |
||||||
|
|
||||||
|
# TODO Maybe we should get cycle from dependent date class. |
||||||
|
@property |
||||||
|
def cycle(self) -> str: |
||||||
|
"""Get dependent cycle.""" |
||||||
|
if "Hour" in self.dependent_date: |
||||||
|
return "hour" |
||||||
|
elif self.dependent_date == "today" or "Days" in self.dependent_date: |
||||||
|
return "day" |
||||||
|
elif "Month" in self.dependent_date: |
||||||
|
return "month" |
||||||
|
else: |
||||||
|
return "week" |
||||||
|
|
||||||
|
@property |
||||||
|
def date_value(self) -> str: |
||||||
|
"""Get dependent date.""" |
||||||
|
return self.dependent_date |
||||||
|
|
||||||
|
@property |
||||||
|
def is_all_task(self) -> bool: |
||||||
|
"""Check whether dependent all tasks or not.""" |
||||||
|
return self.dependent_task_name == DEPENDENT_ALL_TASK_IN_WORKFLOW |
||||||
|
|
||||||
|
@property |
||||||
|
def code_parameter(self) -> Tuple: |
||||||
|
"""Get name info parameter to query code.""" |
||||||
|
param = ( |
||||||
|
self.project_name, |
||||||
|
self.process_definition_name, |
||||||
|
self.dependent_task_name if not self.is_all_task else None, |
||||||
|
) |
||||||
|
return param |
||||||
|
|
||||||
|
def get_code_from_gateway(self) -> Dict: |
||||||
|
"""Get project, definition, task code from given parameter.""" |
||||||
|
if self._code: |
||||||
|
return self._code |
||||||
|
else: |
||||||
|
gateway = launch_gateway() |
||||||
|
try: |
||||||
|
self._code = gateway.entry_point.getDependentInfo(*self.code_parameter) |
||||||
|
return self._code |
||||||
|
except Exception: |
||||||
|
raise PyDSJavaGatewayException("Function get_code_from_gateway error.") |
||||||
|
|
||||||
|
|
||||||
|
class DependentOperator(Base): |
||||||
|
"""Set DependentItem or dependItemList with specific operator.""" |
||||||
|
|
||||||
|
_DEFINE_ATTR = { |
||||||
|
"relation", |
||||||
|
} |
||||||
|
|
||||||
|
def __init__(self, *args): |
||||||
|
super().__init__(self.__class__.__name__) |
||||||
|
self.args = args |
||||||
|
|
||||||
|
def __repr__(self) -> str: |
||||||
|
return "depend_task_list" |
||||||
|
|
||||||
|
@classmethod |
||||||
|
def operator_name(cls) -> str: |
||||||
|
"""Get operator name in different class.""" |
||||||
|
return cls.__name__.upper() |
||||||
|
|
||||||
|
@property |
||||||
|
def relation(self) -> str: |
||||||
|
"""Get operator name in different class, for function :func:`get_define`.""" |
||||||
|
return self.operator_name() |
||||||
|
|
||||||
|
def set_define_attr(self) -> str: |
||||||
|
"""Set attribute to function :func:`get_define`. |
||||||
|
|
||||||
|
It is a wrapper for both `And` and `Or` operator. |
||||||
|
""" |
||||||
|
result = [] |
||||||
|
attr = None |
||||||
|
for dependent in self.args: |
||||||
|
if isinstance(dependent, (DependentItem, DependentOperator)): |
||||||
|
if attr is None: |
||||||
|
attr = repr(dependent) |
||||||
|
elif repr(dependent) != attr: |
||||||
|
raise PyDSParamException( |
||||||
|
"Dependent %s operator parameter only support same type.", |
||||||
|
self.relation, |
||||||
|
) |
||||||
|
else: |
||||||
|
raise PyDSParamException( |
||||||
|
"Dependent %s operator parameter support DependentItem and " |
||||||
|
"DependentOperator but got %s.", |
||||||
|
(self.relation, type(dependent)), |
||||||
|
) |
||||||
|
result.append(dependent.get_define()) |
||||||
|
setattr(self, attr, result) |
||||||
|
return attr |
||||||
|
|
||||||
|
def get_define(self, camel_attr=True) -> Dict: |
||||||
|
"""Overwrite Base.get_define to get task dependent specific get define.""" |
||||||
|
attr = self.set_define_attr() |
||||||
|
dependent_define_attr = self._DEFINE_ATTR.union({attr}) |
||||||
|
return super().get_define_custom( |
||||||
|
camel_attr=True, custom_attr=dependent_define_attr |
||||||
|
) |
||||||
|
|
||||||
|
|
||||||
|
class And(DependentOperator): |
||||||
|
"""Operator And for task dependent. |
||||||
|
|
||||||
|
It could accept both :class:`DependentItem` and children of :class:`DependentOperator`, |
||||||
|
and set AND condition to those args. |
||||||
|
""" |
||||||
|
|
||||||
|
def __init__(self, *args): |
||||||
|
super().__init__(*args) |
||||||
|
|
||||||
|
|
||||||
|
class Or(DependentOperator): |
||||||
|
"""Operator Or for task dependent. |
||||||
|
|
||||||
|
It could accept both :class:`DependentItem` and children of :class:`DependentOperator`, |
||||||
|
and set OR condition to those args. |
||||||
|
""" |
||||||
|
|
||||||
|
def __init__(self, *args): |
||||||
|
super().__init__(*args) |
||||||
|
|
||||||
|
|
||||||
|
class Dependent(Task): |
||||||
|
"""Task dependent object, declare behavior for dependent task to dolphinscheduler.""" |
||||||
|
|
||||||
|
def __init__(self, name: str, dependence: DependentOperator, *args, **kwargs): |
||||||
|
super().__init__(name, TaskType.DEPENDENT, *args, **kwargs) |
||||||
|
self.dependence = dependence |
||||||
|
|
||||||
|
@property |
||||||
|
def task_params(self, camel_attr: bool = True, custom_attr: set = None) -> Dict: |
||||||
|
"""Override Task.task_params for dependent task. |
||||||
|
|
||||||
|
Dependent task have some specials attribute `dependence`, and in most of the task |
||||||
|
this attribute is None and use empty dict `{}` as default value. We do not use class |
||||||
|
attribute `_task_custom_attr` due to avoid attribute cover. |
||||||
|
""" |
||||||
|
params = super().task_params |
||||||
|
params["dependence"] = self.dependence.get_define() |
||||||
|
return params |
@ -0,0 +1,101 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""Task shell.""" |
||||||
|
|
||||||
|
from typing import Optional |
||||||
|
|
||||||
|
from pydolphinscheduler.constants import TaskType |
||||||
|
from pydolphinscheduler.core.task import Task |
||||||
|
from pydolphinscheduler.exceptions import PyDSParamException |
||||||
|
|
||||||
|
|
||||||
|
class HttpMethod: |
||||||
|
"""Constant of HTTP method.""" |
||||||
|
|
||||||
|
GET = "GET" |
||||||
|
POST = "POST" |
||||||
|
HEAD = "HEAD" |
||||||
|
PUT = "PUT" |
||||||
|
DELETE = "DELETE" |
||||||
|
|
||||||
|
|
||||||
|
class HttpCheckCondition: |
||||||
|
"""Constant of HTTP check condition. |
||||||
|
|
||||||
|
For now it contain four value: |
||||||
|
- STATUS_CODE_DEFAULT: when http response code equal to 200, mark as success. |
||||||
|
- STATUS_CODE_CUSTOM: when http response code equal to the code user define, mark as success. |
||||||
|
- BODY_CONTAINS: when http response body contain text user define, mark as success. |
||||||
|
- BODY_NOT_CONTAINS: when http response body do not contain text user define, mark as success. |
||||||
|
""" |
||||||
|
|
||||||
|
STATUS_CODE_DEFAULT = "STATUS_CODE_DEFAULT" |
||||||
|
STATUS_CODE_CUSTOM = "STATUS_CODE_CUSTOM" |
||||||
|
BODY_CONTAINS = "BODY_CONTAINS" |
||||||
|
BODY_NOT_CONTAINS = "BODY_NOT_CONTAINS" |
||||||
|
|
||||||
|
|
||||||
|
class Http(Task): |
||||||
|
"""Task HTTP object, declare behavior for HTTP task to dolphinscheduler.""" |
||||||
|
|
||||||
|
_task_custom_attr = { |
||||||
|
"url", |
||||||
|
"http_method", |
||||||
|
"http_params", |
||||||
|
"http_check_condition", |
||||||
|
"condition", |
||||||
|
"connect_timeout", |
||||||
|
"socket_timeout", |
||||||
|
} |
||||||
|
|
||||||
|
def __init__( |
||||||
|
self, |
||||||
|
name: str, |
||||||
|
url: str, |
||||||
|
http_method: Optional[str] = HttpMethod.GET, |
||||||
|
http_params: Optional[str] = None, |
||||||
|
http_check_condition: Optional[str] = HttpCheckCondition.STATUS_CODE_DEFAULT, |
||||||
|
condition: Optional[str] = None, |
||||||
|
connect_timeout: Optional[int] = 60000, |
||||||
|
socket_timeout: Optional[int] = 60000, |
||||||
|
*args, |
||||||
|
**kwargs |
||||||
|
): |
||||||
|
super().__init__(name, TaskType.HTTP, *args, **kwargs) |
||||||
|
self.url = url |
||||||
|
if not hasattr(HttpMethod, http_method): |
||||||
|
raise PyDSParamException( |
||||||
|
"Parameter http_method %s not support.", http_method |
||||||
|
) |
||||||
|
self.http_method = http_method |
||||||
|
self.http_params = http_params or [] |
||||||
|
if not hasattr(HttpCheckCondition, http_check_condition): |
||||||
|
raise PyDSParamException( |
||||||
|
"Parameter http_check_condition %s not support.", http_check_condition |
||||||
|
) |
||||||
|
self.http_check_condition = http_check_condition |
||||||
|
if ( |
||||||
|
http_check_condition != HttpCheckCondition.STATUS_CODE_DEFAULT |
||||||
|
and condition is None |
||||||
|
): |
||||||
|
raise PyDSParamException( |
||||||
|
"Parameter condition must provider if http_check_condition not equal to STATUS_CODE_DEFAULT" |
||||||
|
) |
||||||
|
self.condition = condition |
||||||
|
self.connect_timeout = connect_timeout |
||||||
|
self.socket_timeout = socket_timeout |
@ -0,0 +1,60 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""Task procedure.""" |
||||||
|
|
||||||
|
from typing import Dict |
||||||
|
|
||||||
|
from pydolphinscheduler.constants import TaskType |
||||||
|
from pydolphinscheduler.core.database import Database |
||||||
|
from pydolphinscheduler.core.task import Task |
||||||
|
|
||||||
|
|
||||||
|
class Procedure(Task): |
||||||
|
"""Task Procedure object, declare behavior for Procedure task to dolphinscheduler. |
||||||
|
|
||||||
|
It should run database procedure job in multiply sql lik engine, such as: |
||||||
|
- ClickHouse |
||||||
|
- DB2 |
||||||
|
- HIVE |
||||||
|
- MySQL |
||||||
|
- Oracle |
||||||
|
- Postgresql |
||||||
|
- Presto |
||||||
|
- SQLServer |
||||||
|
You provider datasource_name contain connection information, it decisions which |
||||||
|
database type and database instance would run this sql. |
||||||
|
""" |
||||||
|
|
||||||
|
_task_custom_attr = {"method"} |
||||||
|
|
||||||
|
def __init__(self, name: str, datasource_name: str, method: str, *args, **kwargs): |
||||||
|
super().__init__(name, TaskType.PROCEDURE, *args, **kwargs) |
||||||
|
self.datasource_name = datasource_name |
||||||
|
self.method = method |
||||||
|
|
||||||
|
@property |
||||||
|
def task_params(self, camel_attr: bool = True, custom_attr: set = None) -> Dict: |
||||||
|
"""Override Task.task_params for produce task. |
||||||
|
|
||||||
|
produce task have some specials attribute for task_params, and is odd if we |
||||||
|
directly set as python property, so we Override Task.task_params here. |
||||||
|
""" |
||||||
|
params = super().task_params |
||||||
|
datasource = Database(self.datasource_name, "type", "datasource") |
||||||
|
params.update(datasource) |
||||||
|
return params |
@ -0,0 +1,51 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""Task Python.""" |
||||||
|
|
||||||
|
import inspect |
||||||
|
import types |
||||||
|
from typing import Any |
||||||
|
|
||||||
|
from pydolphinscheduler.constants import TaskType |
||||||
|
from pydolphinscheduler.core.task import Task |
||||||
|
from pydolphinscheduler.exceptions import PyDSParamException |
||||||
|
|
||||||
|
|
||||||
|
class Python(Task): |
||||||
|
"""Task Python object, declare behavior for Python task to dolphinscheduler.""" |
||||||
|
|
||||||
|
_task_custom_attr = { |
||||||
|
"raw_script", |
||||||
|
} |
||||||
|
|
||||||
|
def __init__(self, name: str, code: Any, *args, **kwargs): |
||||||
|
super().__init__(name, TaskType.PYTHON, *args, **kwargs) |
||||||
|
self._code = code |
||||||
|
|
||||||
|
@property |
||||||
|
def raw_script(self) -> str: |
||||||
|
"""Get python task define attribute `raw_script`.""" |
||||||
|
if isinstance(self._code, str): |
||||||
|
return self._code |
||||||
|
elif isinstance(self._code, types.FunctionType): |
||||||
|
py_function = inspect.getsource(self._code) |
||||||
|
return py_function |
||||||
|
else: |
||||||
|
raise PyDSParamException( |
||||||
|
"Parameter code do not support % for now.", type(self._code) |
||||||
|
) |
@ -0,0 +1,38 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""Task shell.""" |
||||||
|
|
||||||
|
from pydolphinscheduler.constants import TaskType |
||||||
|
from pydolphinscheduler.core.task import Task |
||||||
|
|
||||||
|
|
||||||
|
class Shell(Task): |
||||||
|
"""Task shell object, declare behavior for shell task to dolphinscheduler. |
||||||
|
|
||||||
|
TODO maybe we could use instance name to replace attribute `name` |
||||||
|
which is simplify as `task_shell = Shell(command = "echo 1")` and |
||||||
|
task.name assign to `task_shell` |
||||||
|
""" |
||||||
|
|
||||||
|
_task_custom_attr = { |
||||||
|
"raw_script", |
||||||
|
} |
||||||
|
|
||||||
|
def __init__(self, name: str, command: str, *args, **kwargs): |
||||||
|
super().__init__(name, TaskType.SHELL, *args, **kwargs) |
||||||
|
self.raw_script = command |
@ -0,0 +1,99 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""Task sql.""" |
||||||
|
|
||||||
|
import re |
||||||
|
from typing import Dict, Optional |
||||||
|
|
||||||
|
from pydolphinscheduler.constants import TaskType |
||||||
|
from pydolphinscheduler.core.database import Database |
||||||
|
from pydolphinscheduler.core.task import Task |
||||||
|
|
||||||
|
|
||||||
|
class SqlType: |
||||||
|
"""SQL type, for now it just contain `SELECT` and `NO_SELECT`.""" |
||||||
|
|
||||||
|
SELECT = 0 |
||||||
|
NOT_SELECT = 1 |
||||||
|
|
||||||
|
|
||||||
|
class Sql(Task): |
||||||
|
"""Task SQL object, declare behavior for SQL task to dolphinscheduler. |
||||||
|
|
||||||
|
It should run sql job in multiply sql lik engine, such as: |
||||||
|
- ClickHouse |
||||||
|
- DB2 |
||||||
|
- HIVE |
||||||
|
- MySQL |
||||||
|
- Oracle |
||||||
|
- Postgresql |
||||||
|
- Presto |
||||||
|
- SQLServer |
||||||
|
You provider datasource_name contain connection information, it decisions which |
||||||
|
database type and database instance would run this sql. |
||||||
|
""" |
||||||
|
|
||||||
|
_task_custom_attr = { |
||||||
|
"sql", |
||||||
|
"sql_type", |
||||||
|
"pre_statements", |
||||||
|
"post_statements", |
||||||
|
"display_rows", |
||||||
|
} |
||||||
|
|
||||||
|
def __init__( |
||||||
|
self, |
||||||
|
name: str, |
||||||
|
datasource_name: str, |
||||||
|
sql: str, |
||||||
|
pre_statements: Optional[str] = None, |
||||||
|
post_statements: Optional[str] = None, |
||||||
|
display_rows: Optional[int] = 10, |
||||||
|
*args, |
||||||
|
**kwargs |
||||||
|
): |
||||||
|
super().__init__(name, TaskType.SQL, *args, **kwargs) |
||||||
|
self.sql = sql |
||||||
|
self.datasource_name = datasource_name |
||||||
|
self.pre_statements = pre_statements or [] |
||||||
|
self.post_statements = post_statements or [] |
||||||
|
self.display_rows = display_rows |
||||||
|
|
||||||
|
@property |
||||||
|
def sql_type(self) -> int: |
||||||
|
"""Judgement sql type, use regexp to check which type of the sql is.""" |
||||||
|
pattern_select_str = ( |
||||||
|
"^(?!(.* |)insert |(.* |)delete |(.* |)drop |(.* |)update |(.* |)alter ).*" |
||||||
|
) |
||||||
|
pattern_select = re.compile(pattern_select_str, re.IGNORECASE) |
||||||
|
if pattern_select.match(self.sql) is None: |
||||||
|
return SqlType.NOT_SELECT |
||||||
|
else: |
||||||
|
return SqlType.SELECT |
||||||
|
|
||||||
|
@property |
||||||
|
def task_params(self, camel_attr: bool = True, custom_attr: set = None) -> Dict: |
||||||
|
"""Override Task.task_params for sql task. |
||||||
|
|
||||||
|
sql task have some specials attribute for task_params, and is odd if we |
||||||
|
directly set as python property, so we Override Task.task_params here. |
||||||
|
""" |
||||||
|
params = super().task_params |
||||||
|
datasource = Database(self.datasource_name, "type", "datasource") |
||||||
|
params.update(datasource) |
||||||
|
return params |
@ -0,0 +1,55 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""Task sub_process.""" |
||||||
|
|
||||||
|
from typing import Dict |
||||||
|
|
||||||
|
from pydolphinscheduler.constants import TaskType |
||||||
|
from pydolphinscheduler.core.task import Task |
||||||
|
from pydolphinscheduler.exceptions import PyDSProcessDefinitionNotAssignException |
||||||
|
from pydolphinscheduler.java_gateway import launch_gateway |
||||||
|
|
||||||
|
|
||||||
|
class SubProcess(Task): |
||||||
|
"""Task SubProcess object, declare behavior for SubProcess task to dolphinscheduler.""" |
||||||
|
|
||||||
|
_task_custom_attr = {"process_definition_code"} |
||||||
|
|
||||||
|
def __init__(self, name: str, process_definition_name: str, *args, **kwargs): |
||||||
|
super().__init__(name, TaskType.SUB_PROCESS, *args, **kwargs) |
||||||
|
self.process_definition_name = process_definition_name |
||||||
|
|
||||||
|
@property |
||||||
|
def process_definition_code(self) -> str: |
||||||
|
"""Get process definition code, a wrapper for :func:`get_process_definition_info`.""" |
||||||
|
return self.get_process_definition_info(self.process_definition_name).get( |
||||||
|
"code" |
||||||
|
) |
||||||
|
|
||||||
|
def get_process_definition_info(self, process_definition_name: str) -> Dict: |
||||||
|
"""Get process definition info from java gateway, contains process definition id, name, code.""" |
||||||
|
if not self.process_definition: |
||||||
|
raise PyDSProcessDefinitionNotAssignException( |
||||||
|
"ProcessDefinition must be provider for task SubProcess." |
||||||
|
) |
||||||
|
gateway = launch_gateway() |
||||||
|
return gateway.entry_point.getProcessDefinitionInfo( |
||||||
|
self.process_definition.user.name, |
||||||
|
self.process_definition.project.name, |
||||||
|
process_definition_name, |
||||||
|
) |
@ -0,0 +1,158 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""Task Switch.""" |
||||||
|
|
||||||
|
from typing import Dict, Optional |
||||||
|
|
||||||
|
from pydolphinscheduler.constants import TaskType |
||||||
|
from pydolphinscheduler.core.base import Base |
||||||
|
from pydolphinscheduler.core.task import Task |
||||||
|
from pydolphinscheduler.exceptions import PyDSParamException |
||||||
|
|
||||||
|
|
||||||
|
class SwitchBranch(Base): |
||||||
|
"""Base class of ConditionBranch of task switch. |
||||||
|
|
||||||
|
It a parent class for :class:`Branch` and :class:`Default`. |
||||||
|
""" |
||||||
|
|
||||||
|
_DEFINE_ATTR = { |
||||||
|
"next_node", |
||||||
|
} |
||||||
|
|
||||||
|
def __init__(self, task: Task, exp: Optional[str] = None): |
||||||
|
super().__init__(f"Switch.{self.__class__.__name__.upper()}") |
||||||
|
self.task = task |
||||||
|
self.exp = exp |
||||||
|
|
||||||
|
@property |
||||||
|
def next_node(self) -> str: |
||||||
|
"""Get task switch property next_node, it return task code when init class switch.""" |
||||||
|
return self.task.code |
||||||
|
|
||||||
|
@property |
||||||
|
def condition(self) -> Optional[str]: |
||||||
|
"""Get task switch property condition.""" |
||||||
|
return self.exp |
||||||
|
|
||||||
|
def get_define(self, camel_attr: bool = True) -> Dict: |
||||||
|
"""Get :class:`ConditionBranch` definition attribute communicate to Java gateway server.""" |
||||||
|
if self.condition: |
||||||
|
self._DEFINE_ATTR.add("condition") |
||||||
|
return super().get_define() |
||||||
|
|
||||||
|
|
||||||
|
class Branch(SwitchBranch): |
||||||
|
"""Common condition branch for switch task. |
||||||
|
|
||||||
|
If any condition in :class:`Branch` match, would set this :class:`Branch`'s task as downstream of task |
||||||
|
switch. If all condition branch do not match would set :class:`Default`'s task as task switch downstream. |
||||||
|
""" |
||||||
|
|
||||||
|
def __init__(self, condition: str, task: Task): |
||||||
|
super().__init__(task, condition) |
||||||
|
|
||||||
|
|
||||||
|
class Default(SwitchBranch): |
||||||
|
"""Class default branch for switch task. |
||||||
|
|
||||||
|
If all condition of :class:`Branch` do not match, task switch would run the tasks in :class:`Default` |
||||||
|
and set :class:`Default`'s task as switch downstream. Please notice that each switch condition |
||||||
|
could only have one single :class:`Default`. |
||||||
|
""" |
||||||
|
|
||||||
|
def __init__(self, task: Task): |
||||||
|
super().__init__(task) |
||||||
|
|
||||||
|
|
||||||
|
class SwitchCondition(Base): |
||||||
|
"""Set switch condition of given parameter.""" |
||||||
|
|
||||||
|
_DEFINE_ATTR = { |
||||||
|
"depend_task_list", |
||||||
|
} |
||||||
|
|
||||||
|
def __init__(self, *args): |
||||||
|
super().__init__(self.__class__.__name__) |
||||||
|
self.args = args |
||||||
|
|
||||||
|
def set_define_attr(self) -> None: |
||||||
|
"""Set attribute to function :func:`get_define`. |
||||||
|
|
||||||
|
It is a wrapper for both `And` and `Or` operator. |
||||||
|
""" |
||||||
|
result = [] |
||||||
|
num_branch_default = 0 |
||||||
|
for condition in self.args: |
||||||
|
if isinstance(condition, SwitchBranch): |
||||||
|
if num_branch_default < 1: |
||||||
|
if isinstance(condition, Default): |
||||||
|
self._DEFINE_ATTR.add("next_node") |
||||||
|
setattr(self, "next_node", condition.next_node) |
||||||
|
num_branch_default += 1 |
||||||
|
elif isinstance(condition, Branch): |
||||||
|
result.append(condition.get_define()) |
||||||
|
else: |
||||||
|
raise PyDSParamException( |
||||||
|
"Task Switch's parameter only support exactly one default branch." |
||||||
|
) |
||||||
|
else: |
||||||
|
raise PyDSParamException( |
||||||
|
"Task Switch's parameter only support SwitchBranch but got %s.", |
||||||
|
type(condition), |
||||||
|
) |
||||||
|
# Handle switch default branch, default value is `""` if not provide. |
||||||
|
if num_branch_default == 0: |
||||||
|
self._DEFINE_ATTR.add("next_node") |
||||||
|
setattr(self, "next_node", "") |
||||||
|
setattr(self, "depend_task_list", result) |
||||||
|
|
||||||
|
def get_define(self, camel_attr=True) -> Dict: |
||||||
|
"""Overwrite Base.get_define to get task Condition specific get define.""" |
||||||
|
self.set_define_attr() |
||||||
|
return super().get_define() |
||||||
|
|
||||||
|
|
||||||
|
class Switch(Task): |
||||||
|
"""Task switch object, declare behavior for switch task to dolphinscheduler.""" |
||||||
|
|
||||||
|
def __init__(self, name: str, condition: SwitchCondition, *args, **kwargs): |
||||||
|
super().__init__(name, TaskType.SWITCH, *args, **kwargs) |
||||||
|
self.condition = condition |
||||||
|
# Set condition tasks as current task downstream |
||||||
|
self._set_dep() |
||||||
|
|
||||||
|
def _set_dep(self) -> None: |
||||||
|
"""Set downstream according to parameter `condition`.""" |
||||||
|
downstream = [] |
||||||
|
for condition in self.condition.args: |
||||||
|
if isinstance(condition, SwitchBranch): |
||||||
|
downstream.append(condition.task) |
||||||
|
self.set_downstream(downstream) |
||||||
|
|
||||||
|
@property |
||||||
|
def task_params(self, camel_attr: bool = True, custom_attr: set = None) -> Dict: |
||||||
|
"""Override Task.task_params for switch task. |
||||||
|
|
||||||
|
switch task have some specials attribute `switch`, and in most of the task |
||||||
|
this attribute is None and use empty dict `{}` as default value. We do not use class |
||||||
|
attribute `_task_custom_attr` due to avoid attribute cover. |
||||||
|
""" |
||||||
|
params = super().task_params |
||||||
|
params["switchResult"] = self.condition.get_define() |
||||||
|
return params |
@ -0,0 +1,18 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""Init utils package.""" |
@ -0,0 +1,82 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""Date util function collections.""" |
||||||
|
|
||||||
|
from datetime import datetime |
||||||
|
|
||||||
|
from pydolphinscheduler.constants import Delimiter, Time |
||||||
|
|
||||||
|
LEN_SUPPORT_DATETIME = ( |
||||||
|
15, |
||||||
|
19, |
||||||
|
) |
||||||
|
|
||||||
|
FMT_SHORT = f"{Time.FMT_SHORT_DATE} {Time.FMT_NO_COLON_TIME}" |
||||||
|
FMT_DASH = f"{Time.FMT_DASH_DATE} {Time.FMT_STD_TIME}" |
||||||
|
FMT_STD = f"{Time.FMT_STD_DATE} {Time.FMT_STD_TIME}" |
||||||
|
|
||||||
|
MAX_DATETIME = datetime(9999, 12, 31, 23, 59, 59) |
||||||
|
|
||||||
|
|
||||||
|
def conv_to_schedule(src: datetime) -> str: |
||||||
|
"""Convert given datetime to schedule date string.""" |
||||||
|
return datetime.strftime(src, FMT_STD) |
||||||
|
|
||||||
|
|
||||||
|
def conv_from_str(src: str) -> datetime: |
||||||
|
"""Convert given string to datetime. |
||||||
|
|
||||||
|
This function give an ability to convert string to datetime, and for now it could handle |
||||||
|
format like: |
||||||
|
- %Y-%m-%d |
||||||
|
- %Y/%m/%d |
||||||
|
- %Y%m%d |
||||||
|
- %Y-%m-%d %H:%M:%S |
||||||
|
- %Y/%m/%d %H:%M:%S |
||||||
|
- %Y%m%d %H%M%S |
||||||
|
If pattern not like above be given will raise NotImplementedError. |
||||||
|
""" |
||||||
|
len_ = len(src) |
||||||
|
if len_ == Time.LEN_SHORT_DATE: |
||||||
|
return datetime.strptime(src, Time.FMT_SHORT_DATE) |
||||||
|
elif len_ == Time.LEN_STD_DATE: |
||||||
|
if Delimiter.BAR in src: |
||||||
|
return datetime.strptime(src, Time.FMT_STD_DATE) |
||||||
|
elif Delimiter.DASH in src: |
||||||
|
return datetime.strptime(src, Time.FMT_DASH_DATE) |
||||||
|
else: |
||||||
|
raise NotImplementedError( |
||||||
|
"%s could not be convert to datetime for now.", src |
||||||
|
) |
||||||
|
elif len_ in LEN_SUPPORT_DATETIME: |
||||||
|
if Delimiter.BAR in src and Delimiter.COLON in src: |
||||||
|
return datetime.strptime(src, FMT_STD) |
||||||
|
elif Delimiter.DASH in src and Delimiter.COLON in src: |
||||||
|
return datetime.strptime(src, FMT_DASH) |
||||||
|
elif ( |
||||||
|
Delimiter.DASH not in src |
||||||
|
and Delimiter.BAR not in src |
||||||
|
and Delimiter.COLON not in src |
||||||
|
): |
||||||
|
return datetime.strptime(src, FMT_SHORT) |
||||||
|
else: |
||||||
|
raise NotImplementedError( |
||||||
|
"%s could not be convert to datetime for now.", src |
||||||
|
) |
||||||
|
else: |
||||||
|
raise NotImplementedError("%s could not be convert to datetime for now.", src) |
@ -0,0 +1,39 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""String util function collections.""" |
||||||
|
|
||||||
|
from pydolphinscheduler.constants import Delimiter |
||||||
|
|
||||||
|
|
||||||
|
def attr2camel(attr: str, include_private=True): |
||||||
|
"""Covert class attribute name to camel case.""" |
||||||
|
if include_private: |
||||||
|
attr = attr.lstrip(Delimiter.UNDERSCORE) |
||||||
|
return snake2camel(attr) |
||||||
|
|
||||||
|
|
||||||
|
def snake2camel(snake: str): |
||||||
|
"""Covert snake case to camel case.""" |
||||||
|
components = snake.split(Delimiter.UNDERSCORE) |
||||||
|
return components[0] + "".join(x.title() for x in components[1:]) |
||||||
|
|
||||||
|
|
||||||
|
def class_name2camel(class_name: str): |
||||||
|
"""Covert class name string to camel case.""" |
||||||
|
class_name = class_name.lstrip(Delimiter.UNDERSCORE) |
||||||
|
return class_name[0].lower() + snake2camel(class_name[1:]) |
@ -0,0 +1,18 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""Init tests package.""" |
@ -0,0 +1,18 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""Init core package tests.""" |
@ -0,0 +1,54 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""Test Database.""" |
||||||
|
|
||||||
|
|
||||||
|
from unittest.mock import patch |
||||||
|
|
||||||
|
import pytest |
||||||
|
|
||||||
|
from pydolphinscheduler.core.database import Database |
||||||
|
|
||||||
|
TEST_DATABASE_DATASOURCE_NAME = "test_datasource" |
||||||
|
TEST_DATABASE_TYPE_KEY = "type" |
||||||
|
TEST_DATABASE_KEY = "datasource" |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
"expect", |
||||||
|
[ |
||||||
|
{ |
||||||
|
TEST_DATABASE_TYPE_KEY: "mock_type", |
||||||
|
TEST_DATABASE_KEY: 1, |
||||||
|
} |
||||||
|
], |
||||||
|
) |
||||||
|
@patch( |
||||||
|
"pydolphinscheduler.core.task.Task.gen_code_and_version", |
||||||
|
return_value=(123, 1), |
||||||
|
) |
||||||
|
@patch( |
||||||
|
"pydolphinscheduler.core.database.Database.get_database_info", |
||||||
|
return_value=({"id": 1, "type": "mock_type"}), |
||||||
|
) |
||||||
|
def test_get_datasource_detail(mock_datasource, mock_code_version, expect): |
||||||
|
"""Test :func:`get_database_type` and :func:`get_database_id` can return expect value.""" |
||||||
|
database_info = Database( |
||||||
|
TEST_DATABASE_DATASOURCE_NAME, TEST_DATABASE_TYPE_KEY, TEST_DATABASE_KEY |
||||||
|
) |
||||||
|
assert expect == database_info |
@ -0,0 +1,342 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""Test process definition.""" |
||||||
|
|
||||||
|
from datetime import datetime |
||||||
|
from typing import Any |
||||||
|
|
||||||
|
import pytest |
||||||
|
from freezegun import freeze_time |
||||||
|
|
||||||
|
from pydolphinscheduler.constants import ( |
||||||
|
ProcessDefinitionDefault, |
||||||
|
ProcessDefinitionReleaseState, |
||||||
|
) |
||||||
|
from pydolphinscheduler.core.process_definition import ProcessDefinition |
||||||
|
from pydolphinscheduler.exceptions import PyDSParamException |
||||||
|
from pydolphinscheduler.side import Project, Tenant, User |
||||||
|
from pydolphinscheduler.utils.date import conv_to_schedule |
||||||
|
from tests.testing.task import Task |
||||||
|
|
||||||
|
TEST_PROCESS_DEFINITION_NAME = "simple-test-process-definition" |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("func", ["run", "submit", "start"]) |
||||||
|
def test_process_definition_key_attr(func): |
||||||
|
"""Test process definition have specific functions or attributes.""" |
||||||
|
with ProcessDefinition(TEST_PROCESS_DEFINITION_NAME) as pd: |
||||||
|
assert hasattr( |
||||||
|
pd, func |
||||||
|
), f"ProcessDefinition instance don't have attribute `{func}`" |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
"name,value", |
||||||
|
[ |
||||||
|
("timezone", ProcessDefinitionDefault.TIME_ZONE), |
||||||
|
("project", Project(ProcessDefinitionDefault.PROJECT)), |
||||||
|
("tenant", Tenant(ProcessDefinitionDefault.TENANT)), |
||||||
|
( |
||||||
|
"user", |
||||||
|
User( |
||||||
|
ProcessDefinitionDefault.USER, |
||||||
|
ProcessDefinitionDefault.USER_PWD, |
||||||
|
ProcessDefinitionDefault.USER_EMAIL, |
||||||
|
ProcessDefinitionDefault.USER_PHONE, |
||||||
|
ProcessDefinitionDefault.TENANT, |
||||||
|
ProcessDefinitionDefault.QUEUE, |
||||||
|
ProcessDefinitionDefault.USER_STATE, |
||||||
|
), |
||||||
|
), |
||||||
|
("worker_group", ProcessDefinitionDefault.WORKER_GROUP), |
||||||
|
("release_state", ProcessDefinitionReleaseState.ONLINE), |
||||||
|
], |
||||||
|
) |
||||||
|
def test_process_definition_default_value(name, value): |
||||||
|
"""Test process definition default attributes.""" |
||||||
|
with ProcessDefinition(TEST_PROCESS_DEFINITION_NAME) as pd: |
||||||
|
assert getattr(pd, name) == value, ( |
||||||
|
f"ProcessDefinition instance attribute `{name}` not with " |
||||||
|
f"except default value `{getattr(pd, name)}`" |
||||||
|
) |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
"name,cls,expect", |
||||||
|
[ |
||||||
|
("name", str, "name"), |
||||||
|
("description", str, "description"), |
||||||
|
("schedule", str, "schedule"), |
||||||
|
("timezone", str, "timezone"), |
||||||
|
("worker_group", str, "worker_group"), |
||||||
|
("timeout", int, 1), |
||||||
|
("release_state", str, "OFFLINE"), |
||||||
|
("param", dict, {"key": "value"}), |
||||||
|
], |
||||||
|
) |
||||||
|
def test_set_attr(name, cls, expect): |
||||||
|
"""Test process definition set attributes which get with same type.""" |
||||||
|
with ProcessDefinition(TEST_PROCESS_DEFINITION_NAME) as pd: |
||||||
|
setattr(pd, name, expect) |
||||||
|
assert ( |
||||||
|
getattr(pd, name) == expect |
||||||
|
), f"ProcessDefinition set attribute `{name}` do not work expect" |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
"set_attr,set_val,get_attr,get_val", |
||||||
|
[ |
||||||
|
("_project", "project", "project", Project("project")), |
||||||
|
("_tenant", "tenant", "tenant", Tenant("tenant")), |
||||||
|
("_start_time", "2021-01-01", "start_time", datetime(2021, 1, 1)), |
||||||
|
("_end_time", "2021-01-01", "end_time", datetime(2021, 1, 1)), |
||||||
|
], |
||||||
|
) |
||||||
|
def test_set_attr_return_special_object(set_attr, set_val, get_attr, get_val): |
||||||
|
"""Test process definition set attributes which get with different type.""" |
||||||
|
with ProcessDefinition(TEST_PROCESS_DEFINITION_NAME) as pd: |
||||||
|
setattr(pd, set_attr, set_val) |
||||||
|
assert get_val == getattr( |
||||||
|
pd, get_attr |
||||||
|
), f"Set attribute {set_attr} can not get back with {get_val}." |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
"val,expect", |
||||||
|
[ |
||||||
|
(datetime(2021, 1, 1), datetime(2021, 1, 1)), |
||||||
|
(None, None), |
||||||
|
("2021-01-01", datetime(2021, 1, 1)), |
||||||
|
("2021-01-01 01:01:01", datetime(2021, 1, 1, 1, 1, 1)), |
||||||
|
], |
||||||
|
) |
||||||
|
def test__parse_datetime(val, expect): |
||||||
|
"""Test process definition function _parse_datetime. |
||||||
|
|
||||||
|
Only two datetime test cases here because we have more test cases in tests/utils/test_date.py file. |
||||||
|
""" |
||||||
|
with ProcessDefinition(TEST_PROCESS_DEFINITION_NAME) as pd: |
||||||
|
assert expect == pd._parse_datetime( |
||||||
|
val |
||||||
|
), f"Function _parse_datetime with unexpect value by {val}." |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
"val", |
||||||
|
[ |
||||||
|
20210101, |
||||||
|
(2021, 1, 1), |
||||||
|
{"year": "2021", "month": "1", "day": 1}, |
||||||
|
], |
||||||
|
) |
||||||
|
def test__parse_datetime_not_support_type(val: Any): |
||||||
|
"""Test process definition function _parse_datetime not support type error.""" |
||||||
|
with ProcessDefinition(TEST_PROCESS_DEFINITION_NAME) as pd: |
||||||
|
with pytest.raises(PyDSParamException, match="Do not support value type.*?"): |
||||||
|
pd._parse_datetime(val) |
||||||
|
|
||||||
|
|
||||||
|
def test_process_definition_get_define_without_task(): |
||||||
|
"""Test process definition function get_define without task.""" |
||||||
|
expect = { |
||||||
|
"name": TEST_PROCESS_DEFINITION_NAME, |
||||||
|
"description": None, |
||||||
|
"project": ProcessDefinitionDefault.PROJECT, |
||||||
|
"tenant": ProcessDefinitionDefault.TENANT, |
||||||
|
"workerGroup": ProcessDefinitionDefault.WORKER_GROUP, |
||||||
|
"timeout": 0, |
||||||
|
"releaseState": ProcessDefinitionReleaseState.ONLINE, |
||||||
|
"param": None, |
||||||
|
"tasks": {}, |
||||||
|
"taskDefinitionJson": [{}], |
||||||
|
"taskRelationJson": [{}], |
||||||
|
} |
||||||
|
with ProcessDefinition(TEST_PROCESS_DEFINITION_NAME) as pd: |
||||||
|
assert pd.get_define() == expect |
||||||
|
|
||||||
|
|
||||||
|
def test_process_definition_simple_context_manager(): |
||||||
|
"""Test simple create workflow in process definition context manager mode.""" |
||||||
|
expect_tasks_num = 5 |
||||||
|
with ProcessDefinition(TEST_PROCESS_DEFINITION_NAME) as pd: |
||||||
|
for i in range(expect_tasks_num): |
||||||
|
curr_task = Task(name=f"task-{i}", task_type=f"type-{i}") |
||||||
|
# Set deps task i as i-1 parent |
||||||
|
if i > 0: |
||||||
|
pre_task = pd.get_one_task_by_name(f"task-{i - 1}") |
||||||
|
curr_task.set_upstream(pre_task) |
||||||
|
assert len(pd.tasks) == expect_tasks_num |
||||||
|
|
||||||
|
# Test if task process_definition same as origin one |
||||||
|
task: Task = pd.get_one_task_by_name("task-0") |
||||||
|
assert pd is task.process_definition |
||||||
|
|
||||||
|
# Test if all tasks with expect deps |
||||||
|
for i in range(expect_tasks_num): |
||||||
|
task: Task = pd.get_one_task_by_name(f"task-{i}") |
||||||
|
if i == 0: |
||||||
|
assert task._upstream_task_codes == set() |
||||||
|
assert task._downstream_task_codes == { |
||||||
|
pd.get_one_task_by_name("task-1").code |
||||||
|
} |
||||||
|
elif i == expect_tasks_num - 1: |
||||||
|
assert task._upstream_task_codes == { |
||||||
|
pd.get_one_task_by_name(f"task-{i - 1}").code |
||||||
|
} |
||||||
|
assert task._downstream_task_codes == set() |
||||||
|
else: |
||||||
|
assert task._upstream_task_codes == { |
||||||
|
pd.get_one_task_by_name(f"task-{i - 1}").code |
||||||
|
} |
||||||
|
assert task._downstream_task_codes == { |
||||||
|
pd.get_one_task_by_name(f"task-{i + 1}").code |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
def test_process_definition_simple_separate(): |
||||||
|
"""Test process definition simple create workflow in separate mode. |
||||||
|
|
||||||
|
This test just test basic information, cause most of test case is duplicate to |
||||||
|
test_process_definition_simple_context_manager. |
||||||
|
""" |
||||||
|
expect_tasks_num = 5 |
||||||
|
pd = ProcessDefinition(TEST_PROCESS_DEFINITION_NAME) |
||||||
|
for i in range(expect_tasks_num): |
||||||
|
curr_task = Task( |
||||||
|
name=f"task-{i}", |
||||||
|
task_type=f"type-{i}", |
||||||
|
process_definition=pd, |
||||||
|
) |
||||||
|
# Set deps task i as i-1 parent |
||||||
|
if i > 0: |
||||||
|
pre_task = pd.get_one_task_by_name(f"task-{i - 1}") |
||||||
|
curr_task.set_upstream(pre_task) |
||||||
|
assert len(pd.tasks) == expect_tasks_num |
||||||
|
assert all(["task-" in task.name for task in pd.task_list]) |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
"user_attrs", |
||||||
|
[ |
||||||
|
{"tenant": "tenant_specific"}, |
||||||
|
{"queue": "queue_specific"}, |
||||||
|
{"tenant": "tenant_specific", "queue": "queue_specific"}, |
||||||
|
], |
||||||
|
) |
||||||
|
def test_set_process_definition_user_attr(user_attrs): |
||||||
|
"""Test user with correct attributes if we specific assigned to process definition object.""" |
||||||
|
default_value = { |
||||||
|
"tenant": ProcessDefinitionDefault.TENANT, |
||||||
|
"queue": ProcessDefinitionDefault.QUEUE, |
||||||
|
} |
||||||
|
with ProcessDefinition(TEST_PROCESS_DEFINITION_NAME, **user_attrs) as pd: |
||||||
|
user = pd.user |
||||||
|
for attr in default_value: |
||||||
|
# Get assigned attribute if we specific, else get default value |
||||||
|
except_attr = ( |
||||||
|
user_attrs[attr] if attr in user_attrs else default_value[attr] |
||||||
|
) |
||||||
|
# Get actually attribute of user object |
||||||
|
actual_attr = getattr(user, attr) |
||||||
|
assert ( |
||||||
|
except_attr == actual_attr |
||||||
|
), f"Except attribute is {except_attr} but get {actual_attr}" |
||||||
|
|
||||||
|
|
||||||
|
def test_schedule_json_none_schedule(): |
||||||
|
"""Test function schedule_json with None as schedule.""" |
||||||
|
with ProcessDefinition( |
||||||
|
TEST_PROCESS_DEFINITION_NAME, |
||||||
|
schedule=None, |
||||||
|
) as pd: |
||||||
|
assert pd.schedule_json is None |
||||||
|
|
||||||
|
|
||||||
|
# We freeze time here, because we test start_time with None, and if will get datetime.datetime.now. If we do |
||||||
|
# not freeze time, it will cause flaky test here. |
||||||
|
@freeze_time("2021-01-01") |
||||||
|
@pytest.mark.parametrize( |
||||||
|
"start_time,end_time,expect_date", |
||||||
|
[ |
||||||
|
( |
||||||
|
"20210101", |
||||||
|
"20210201", |
||||||
|
{"start_time": "2021-01-01 00:00:00", "end_time": "2021-02-01 00:00:00"}, |
||||||
|
), |
||||||
|
( |
||||||
|
"2021-01-01", |
||||||
|
"2021-02-01", |
||||||
|
{"start_time": "2021-01-01 00:00:00", "end_time": "2021-02-01 00:00:00"}, |
||||||
|
), |
||||||
|
( |
||||||
|
"2021/01/01", |
||||||
|
"2021/02/01", |
||||||
|
{"start_time": "2021-01-01 00:00:00", "end_time": "2021-02-01 00:00:00"}, |
||||||
|
), |
||||||
|
# Test mix pattern |
||||||
|
( |
||||||
|
"2021/01/01 01:01:01", |
||||||
|
"2021-02-02 02:02:02", |
||||||
|
{"start_time": "2021-01-01 01:01:01", "end_time": "2021-02-02 02:02:02"}, |
||||||
|
), |
||||||
|
( |
||||||
|
"2021/01/01 01:01:01", |
||||||
|
"20210202 020202", |
||||||
|
{"start_time": "2021-01-01 01:01:01", "end_time": "2021-02-02 02:02:02"}, |
||||||
|
), |
||||||
|
( |
||||||
|
"20210101 010101", |
||||||
|
"2021-02-02 02:02:02", |
||||||
|
{"start_time": "2021-01-01 01:01:01", "end_time": "2021-02-02 02:02:02"}, |
||||||
|
), |
||||||
|
# Test None value |
||||||
|
( |
||||||
|
"2021/01/01 01:02:03", |
||||||
|
None, |
||||||
|
{"start_time": "2021-01-01 01:02:03", "end_time": "9999-12-31 23:59:59"}, |
||||||
|
), |
||||||
|
( |
||||||
|
None, |
||||||
|
None, |
||||||
|
{ |
||||||
|
"start_time": conv_to_schedule(datetime(2021, 1, 1)), |
||||||
|
"end_time": "9999-12-31 23:59:59", |
||||||
|
}, |
||||||
|
), |
||||||
|
], |
||||||
|
) |
||||||
|
def test_schedule_json_start_and_end_time(start_time, end_time, expect_date): |
||||||
|
"""Test function schedule_json about handle start_time and end_time. |
||||||
|
|
||||||
|
Only two datetime test cases here because we have more test cases in tests/utils/test_date.py file. |
||||||
|
""" |
||||||
|
schedule = "0 0 0 * * ? *" |
||||||
|
expect = { |
||||||
|
"crontab": schedule, |
||||||
|
"startTime": expect_date["start_time"], |
||||||
|
"endTime": expect_date["end_time"], |
||||||
|
"timezoneId": ProcessDefinitionDefault.TIME_ZONE, |
||||||
|
} |
||||||
|
with ProcessDefinition( |
||||||
|
TEST_PROCESS_DEFINITION_NAME, |
||||||
|
schedule=schedule, |
||||||
|
start_time=start_time, |
||||||
|
end_time=end_time, |
||||||
|
timezone=ProcessDefinitionDefault.TIME_ZONE, |
||||||
|
) as pd: |
||||||
|
assert pd.schedule_json == expect |
@ -0,0 +1,224 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""Test Task class function.""" |
||||||
|
|
||||||
|
from unittest.mock import patch |
||||||
|
|
||||||
|
import pytest |
||||||
|
|
||||||
|
from pydolphinscheduler.core.task import Task, TaskRelation |
||||||
|
from tests.testing.task import Task as testTask |
||||||
|
|
||||||
|
TEST_TASK_RELATION_SET = set() |
||||||
|
TEST_TASK_RELATION_SIZE = 0 |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
"attr, expect", |
||||||
|
[ |
||||||
|
( |
||||||
|
dict(), |
||||||
|
{ |
||||||
|
"localParams": [], |
||||||
|
"resourceList": [], |
||||||
|
"dependence": {}, |
||||||
|
"waitStartTimeout": {}, |
||||||
|
"conditionResult": {"successNode": [""], "failedNode": [""]}, |
||||||
|
}, |
||||||
|
), |
||||||
|
( |
||||||
|
{ |
||||||
|
"local_params": ["foo", "bar"], |
||||||
|
"resource_list": ["foo", "bar"], |
||||||
|
"dependence": {"foo", "bar"}, |
||||||
|
"wait_start_timeout": {"foo", "bar"}, |
||||||
|
"condition_result": {"foo": ["bar"]}, |
||||||
|
}, |
||||||
|
{ |
||||||
|
"localParams": ["foo", "bar"], |
||||||
|
"resourceList": ["foo", "bar"], |
||||||
|
"dependence": {"foo", "bar"}, |
||||||
|
"waitStartTimeout": {"foo", "bar"}, |
||||||
|
"conditionResult": {"foo": ["bar"]}, |
||||||
|
}, |
||||||
|
), |
||||||
|
], |
||||||
|
) |
||||||
|
def test_property_task_params(attr, expect): |
||||||
|
"""Test class task property.""" |
||||||
|
task = testTask( |
||||||
|
"test-property-task-params", |
||||||
|
"test-task", |
||||||
|
**attr, |
||||||
|
) |
||||||
|
assert expect == task.task_params |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
"pre_code, post_code, expect", |
||||||
|
[ |
||||||
|
(123, 456, hash("123 -> 456")), |
||||||
|
(12345678, 987654321, hash("12345678 -> 987654321")), |
||||||
|
], |
||||||
|
) |
||||||
|
def test_task_relation_hash_func(pre_code, post_code, expect): |
||||||
|
"""Test TaskRelation magic function :func:`__hash__`.""" |
||||||
|
task_param = TaskRelation(pre_task_code=pre_code, post_task_code=post_code) |
||||||
|
assert hash(task_param) == expect |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
"pre_code, post_code, size_add", |
||||||
|
[ |
||||||
|
(123, 456, 1), |
||||||
|
(123, 456, 0), |
||||||
|
(456, 456, 1), |
||||||
|
(123, 123, 1), |
||||||
|
(456, 123, 1), |
||||||
|
(0, 456, 1), |
||||||
|
(123, 0, 1), |
||||||
|
], |
||||||
|
) |
||||||
|
def test_task_relation_add_to_set(pre_code, post_code, size_add): |
||||||
|
"""Test TaskRelation with different pre_code and post_code add to set behavior. |
||||||
|
|
||||||
|
Here we use global variable to keep set of :class:`TaskRelation` instance and the number we expect |
||||||
|
of the size when we add a new task relation to exists set. |
||||||
|
""" |
||||||
|
task_relation = TaskRelation(pre_task_code=pre_code, post_task_code=post_code) |
||||||
|
TEST_TASK_RELATION_SET.add(task_relation) |
||||||
|
# hint python interpreter use global variable instead of local's |
||||||
|
global TEST_TASK_RELATION_SIZE |
||||||
|
TEST_TASK_RELATION_SIZE += size_add |
||||||
|
assert len(TEST_TASK_RELATION_SET) == TEST_TASK_RELATION_SIZE |
||||||
|
|
||||||
|
|
||||||
|
def test_task_relation_to_dict(): |
||||||
|
"""Test TaskRelation object function to_dict.""" |
||||||
|
pre_task_code = 123 |
||||||
|
post_task_code = 456 |
||||||
|
expect = { |
||||||
|
"name": "", |
||||||
|
"preTaskCode": pre_task_code, |
||||||
|
"postTaskCode": post_task_code, |
||||||
|
"preTaskVersion": 1, |
||||||
|
"postTaskVersion": 1, |
||||||
|
"conditionType": 0, |
||||||
|
"conditionParams": {}, |
||||||
|
} |
||||||
|
task_relation = TaskRelation( |
||||||
|
pre_task_code=pre_task_code, post_task_code=post_task_code |
||||||
|
) |
||||||
|
assert task_relation.get_define() == expect |
||||||
|
|
||||||
|
|
||||||
|
def test_task_get_define(): |
||||||
|
"""Test Task object function get_define.""" |
||||||
|
code = 123 |
||||||
|
version = 1 |
||||||
|
name = "test_task_get_define" |
||||||
|
task_type = "test_task_get_define_type" |
||||||
|
expect = { |
||||||
|
"code": code, |
||||||
|
"name": name, |
||||||
|
"version": version, |
||||||
|
"description": None, |
||||||
|
"delayTime": 0, |
||||||
|
"taskType": task_type, |
||||||
|
"taskParams": { |
||||||
|
"resourceList": [], |
||||||
|
"localParams": [], |
||||||
|
"dependence": {}, |
||||||
|
"conditionResult": {"successNode": [""], "failedNode": [""]}, |
||||||
|
"waitStartTimeout": {}, |
||||||
|
}, |
||||||
|
"flag": "YES", |
||||||
|
"taskPriority": "MEDIUM", |
||||||
|
"workerGroup": "default", |
||||||
|
"failRetryTimes": 0, |
||||||
|
"failRetryInterval": 1, |
||||||
|
"timeoutFlag": "CLOSE", |
||||||
|
"timeoutNotifyStrategy": None, |
||||||
|
"timeout": 0, |
||||||
|
} |
||||||
|
with patch( |
||||||
|
"pydolphinscheduler.core.task.Task.gen_code_and_version", |
||||||
|
return_value=(code, version), |
||||||
|
): |
||||||
|
task = Task(name=name, task_type=task_type) |
||||||
|
assert task.get_define() == expect |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("shift", ["<<", ">>"]) |
||||||
|
def test_two_tasks_shift(shift: str): |
||||||
|
"""Test bit operator between tasks. |
||||||
|
|
||||||
|
Here we test both `>>` and `<<` bit operator. |
||||||
|
""" |
||||||
|
upstream = testTask(name="upstream", task_type=shift) |
||||||
|
downstream = testTask(name="downstream", task_type=shift) |
||||||
|
if shift == "<<": |
||||||
|
downstream << upstream |
||||||
|
elif shift == ">>": |
||||||
|
upstream >> downstream |
||||||
|
else: |
||||||
|
assert False, f"Unexpect bit operator type {shift}." |
||||||
|
assert ( |
||||||
|
1 == len(upstream._downstream_task_codes) |
||||||
|
and downstream.code in upstream._downstream_task_codes |
||||||
|
), "Task downstream task attributes error, downstream codes size or specific code failed." |
||||||
|
assert ( |
||||||
|
1 == len(downstream._upstream_task_codes) |
||||||
|
and upstream.code in downstream._upstream_task_codes |
||||||
|
), "Task upstream task attributes error, upstream codes size or upstream code failed." |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
"dep_expr, flag", |
||||||
|
[ |
||||||
|
("task << tasks", "upstream"), |
||||||
|
("tasks << task", "downstream"), |
||||||
|
("task >> tasks", "downstream"), |
||||||
|
("tasks >> task", "upstream"), |
||||||
|
], |
||||||
|
) |
||||||
|
def test_tasks_list_shift(dep_expr: str, flag: str): |
||||||
|
"""Test bit operator between task and sequence of tasks. |
||||||
|
|
||||||
|
Here we test both `>>` and `<<` bit operator. |
||||||
|
""" |
||||||
|
reverse_dict = { |
||||||
|
"upstream": "downstream", |
||||||
|
"downstream": "upstream", |
||||||
|
} |
||||||
|
task_type = "dep_task_and_tasks" |
||||||
|
task = testTask(name="upstream", task_type=task_type) |
||||||
|
tasks = [ |
||||||
|
testTask(name="downstream1", task_type=task_type), |
||||||
|
testTask(name="downstream2", task_type=task_type), |
||||||
|
] |
||||||
|
|
||||||
|
# Use build-in function eval to simply test case and reduce duplicate code |
||||||
|
eval(dep_expr) |
||||||
|
direction_attr = f"_{flag}_task_codes" |
||||||
|
reverse_direction_attr = f"_{reverse_dict[flag]}_task_codes" |
||||||
|
assert 2 == len(getattr(task, direction_attr)) |
||||||
|
assert [t.code in getattr(task, direction_attr) for t in tasks] |
||||||
|
|
||||||
|
assert all([1 == len(getattr(t, reverse_direction_attr)) for t in tasks]) |
||||||
|
assert all([task.code in getattr(t, reverse_direction_attr) for t in tasks]) |
@ -0,0 +1,18 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""Init tasks package tests.""" |
@ -0,0 +1,439 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""Test Task dependent.""" |
||||||
|
from typing import List, Tuple |
||||||
|
from unittest.mock import patch |
||||||
|
|
||||||
|
import pytest |
||||||
|
|
||||||
|
from pydolphinscheduler.core.process_definition import ProcessDefinition |
||||||
|
from pydolphinscheduler.exceptions import PyDSParamException |
||||||
|
from pydolphinscheduler.tasks.condition import ( |
||||||
|
FAILURE, |
||||||
|
SUCCESS, |
||||||
|
And, |
||||||
|
ConditionOperator, |
||||||
|
Conditions, |
||||||
|
Or, |
||||||
|
Status, |
||||||
|
) |
||||||
|
from tests.testing.task import Task |
||||||
|
|
||||||
|
TEST_NAME = "test-name" |
||||||
|
TEST_PROJECT = "test-project" |
||||||
|
TEST_PROCESS_DEFINITION = "test-process-definition" |
||||||
|
TEST_TYPE = "test-type" |
||||||
|
TEST_PROJECT_CODE, TEST_DEFINITION_CODE, TEST_TASK_CODE = 12345, 123456, 1234567 |
||||||
|
|
||||||
|
TEST_OPERATOR_LIST = ("AND", "OR") |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
"obj, expect", |
||||||
|
[ |
||||||
|
(Status, "STATUS"), |
||||||
|
(SUCCESS, "SUCCESS"), |
||||||
|
(FAILURE, "FAILURE"), |
||||||
|
], |
||||||
|
) |
||||||
|
def test_class_status_status_name(obj: Status, expect: str): |
||||||
|
"""Test class status and sub class property status_name.""" |
||||||
|
assert obj.status_name() == expect |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
"obj, tasks", |
||||||
|
[ |
||||||
|
(Status, (1, 2, 3)), |
||||||
|
(SUCCESS, (1.1, 2.2, 3.3)), |
||||||
|
(FAILURE, (ConditionOperator(1), ConditionOperator(2), ConditionOperator(3))), |
||||||
|
], |
||||||
|
) |
||||||
|
def test_class_status_depend_item_list_no_expect_type(obj: Status, tasks: Tuple): |
||||||
|
"""Test class status and sub class raise error when assign not support type.""" |
||||||
|
with pytest.raises( |
||||||
|
PyDSParamException, match=".*?only accept class Task or sub class Task, but get" |
||||||
|
): |
||||||
|
obj(*tasks).get_define() |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
"obj, tasks", |
||||||
|
[ |
||||||
|
(Status, [Task(str(i), TEST_TYPE) for i in range(1)]), |
||||||
|
(Status, [Task(str(i), TEST_TYPE) for i in range(2)]), |
||||||
|
(Status, [Task(str(i), TEST_TYPE) for i in range(3)]), |
||||||
|
(SUCCESS, [Task(str(i), TEST_TYPE) for i in range(1)]), |
||||||
|
(SUCCESS, [Task(str(i), TEST_TYPE) for i in range(2)]), |
||||||
|
(SUCCESS, [Task(str(i), TEST_TYPE) for i in range(3)]), |
||||||
|
(FAILURE, [Task(str(i), TEST_TYPE) for i in range(1)]), |
||||||
|
(FAILURE, [Task(str(i), TEST_TYPE) for i in range(2)]), |
||||||
|
(FAILURE, [Task(str(i), TEST_TYPE) for i in range(3)]), |
||||||
|
], |
||||||
|
) |
||||||
|
def test_class_status_depend_item_list(obj: Status, tasks: Tuple): |
||||||
|
"""Test class status and sub class function :func:`depend_item_list`.""" |
||||||
|
status = obj.status_name() |
||||||
|
expect = [ |
||||||
|
{ |
||||||
|
"depTaskCode": i.code, |
||||||
|
"status": status, |
||||||
|
} |
||||||
|
for i in tasks |
||||||
|
] |
||||||
|
assert obj(*tasks).get_define() == expect |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
"obj, expect", |
||||||
|
[ |
||||||
|
(ConditionOperator, "CONDITIONOPERATOR"), |
||||||
|
(And, "AND"), |
||||||
|
(Or, "OR"), |
||||||
|
], |
||||||
|
) |
||||||
|
def test_condition_operator_operator_name(obj: ConditionOperator, expect: str): |
||||||
|
"""Test class ConditionOperator and sub class class function :func:`operator_name`.""" |
||||||
|
assert obj.operator_name() == expect |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
"obj, expect", |
||||||
|
[ |
||||||
|
(ConditionOperator, "CONDITIONOPERATOR"), |
||||||
|
(And, "AND"), |
||||||
|
(Or, "OR"), |
||||||
|
], |
||||||
|
) |
||||||
|
def test_condition_operator_relation(obj: ConditionOperator, expect: str): |
||||||
|
"""Test class ConditionOperator and sub class class property `relation`.""" |
||||||
|
assert obj(1).relation == expect |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
"obj, status_or_operator, match", |
||||||
|
[ |
||||||
|
( |
||||||
|
ConditionOperator, |
||||||
|
[Status(Task("1", TEST_TYPE)), 1], |
||||||
|
".*?operator parameter support ConditionTask and ConditionOperator.*?", |
||||||
|
), |
||||||
|
( |
||||||
|
ConditionOperator, |
||||||
|
[ |
||||||
|
Status(Task("1", TEST_TYPE)), |
||||||
|
1.0, |
||||||
|
], |
||||||
|
".*?operator parameter support ConditionTask and ConditionOperator.*?", |
||||||
|
), |
||||||
|
( |
||||||
|
ConditionOperator, |
||||||
|
[ |
||||||
|
Status(Task("1", TEST_TYPE)), |
||||||
|
ConditionOperator(And(Status(Task("1", TEST_TYPE)))), |
||||||
|
], |
||||||
|
".*?operator parameter only support same type.", |
||||||
|
), |
||||||
|
( |
||||||
|
ConditionOperator, |
||||||
|
[ |
||||||
|
ConditionOperator(And(Status(Task("1", TEST_TYPE)))), |
||||||
|
Status(Task("1", TEST_TYPE)), |
||||||
|
], |
||||||
|
".*?operator parameter only support same type.", |
||||||
|
), |
||||||
|
], |
||||||
|
) |
||||||
|
def test_condition_operator_set_define_attr_not_support_type( |
||||||
|
obj, status_or_operator, match |
||||||
|
): |
||||||
|
"""Test class ConditionOperator parameter error, including parameter not same or type not support.""" |
||||||
|
with pytest.raises(PyDSParamException, match=match): |
||||||
|
op = obj(*status_or_operator) |
||||||
|
op.set_define_attr() |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
"obj, task_num", |
||||||
|
[ |
||||||
|
(ConditionOperator, 1), |
||||||
|
(ConditionOperator, 2), |
||||||
|
(ConditionOperator, 3), |
||||||
|
(And, 1), |
||||||
|
(And, 2), |
||||||
|
(And, 3), |
||||||
|
(Or, 1), |
||||||
|
(Or, 2), |
||||||
|
(Or, 3), |
||||||
|
], |
||||||
|
) |
||||||
|
def test_condition_operator_set_define_attr_status( |
||||||
|
obj: ConditionOperator, task_num: int |
||||||
|
): |
||||||
|
"""Test :func:`set_define_attr` with one or more class status.""" |
||||||
|
attr = "depend_item_list" |
||||||
|
|
||||||
|
tasks = [Task(str(i), TEST_TYPE) for i in range(task_num)] |
||||||
|
status = Status(*tasks) |
||||||
|
|
||||||
|
expect = [ |
||||||
|
{"depTaskCode": task.code, "status": status.status_name()} for task in tasks |
||||||
|
] |
||||||
|
|
||||||
|
co = obj(status) |
||||||
|
co.set_define_attr() |
||||||
|
assert getattr(co, attr) == expect |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
"obj, status", |
||||||
|
[ |
||||||
|
(ConditionOperator, (SUCCESS, SUCCESS)), |
||||||
|
(ConditionOperator, (FAILURE, FAILURE)), |
||||||
|
(ConditionOperator, (SUCCESS, FAILURE)), |
||||||
|
(ConditionOperator, (FAILURE, SUCCESS)), |
||||||
|
(And, (SUCCESS, SUCCESS)), |
||||||
|
(And, (FAILURE, FAILURE)), |
||||||
|
(And, (SUCCESS, FAILURE)), |
||||||
|
(And, (FAILURE, SUCCESS)), |
||||||
|
(Or, (SUCCESS, SUCCESS)), |
||||||
|
(Or, (FAILURE, FAILURE)), |
||||||
|
(Or, (SUCCESS, FAILURE)), |
||||||
|
(Or, (FAILURE, SUCCESS)), |
||||||
|
], |
||||||
|
) |
||||||
|
def test_condition_operator_set_define_attr_mix_status( |
||||||
|
obj: ConditionOperator, status: List[Status] |
||||||
|
): |
||||||
|
"""Test :func:`set_define_attr` with one or more mixed status.""" |
||||||
|
attr = "depend_item_list" |
||||||
|
|
||||||
|
task = Task("test-operator", TEST_TYPE) |
||||||
|
status_list = [] |
||||||
|
expect = [] |
||||||
|
for sta in status: |
||||||
|
status_list.append(sta(task)) |
||||||
|
expect.append({"depTaskCode": task.code, "status": sta.status_name()}) |
||||||
|
|
||||||
|
co = obj(*status_list) |
||||||
|
co.set_define_attr() |
||||||
|
assert getattr(co, attr) == expect |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
"obj, task_num", |
||||||
|
[ |
||||||
|
(ConditionOperator, 1), |
||||||
|
(ConditionOperator, 2), |
||||||
|
(ConditionOperator, 3), |
||||||
|
(And, 1), |
||||||
|
(And, 2), |
||||||
|
(And, 3), |
||||||
|
(Or, 1), |
||||||
|
(Or, 2), |
||||||
|
(Or, 3), |
||||||
|
], |
||||||
|
) |
||||||
|
def test_condition_operator_set_define_attr_operator( |
||||||
|
obj: ConditionOperator, task_num: int |
||||||
|
): |
||||||
|
"""Test :func:`set_define_attr` with one or more class condition operator.""" |
||||||
|
attr = "depend_task_list" |
||||||
|
|
||||||
|
task = Task("test-operator", TEST_TYPE) |
||||||
|
status = Status(task) |
||||||
|
|
||||||
|
expect = [ |
||||||
|
{ |
||||||
|
"relation": obj.operator_name(), |
||||||
|
"dependItemList": [ |
||||||
|
{ |
||||||
|
"depTaskCode": task.code, |
||||||
|
"status": status.status_name(), |
||||||
|
} |
||||||
|
], |
||||||
|
} |
||||||
|
for _ in range(task_num) |
||||||
|
] |
||||||
|
|
||||||
|
co = obj(*[obj(status) for _ in range(task_num)]) |
||||||
|
co.set_define_attr() |
||||||
|
assert getattr(co, attr) == expect |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
"cond, sub_cond", |
||||||
|
[ |
||||||
|
(ConditionOperator, (And, Or)), |
||||||
|
(ConditionOperator, (Or, And)), |
||||||
|
(And, (And, Or)), |
||||||
|
(And, (Or, And)), |
||||||
|
(Or, (And, Or)), |
||||||
|
(Or, (Or, And)), |
||||||
|
], |
||||||
|
) |
||||||
|
def test_condition_operator_set_define_attr_mix_operator( |
||||||
|
cond: ConditionOperator, sub_cond: Tuple[ConditionOperator] |
||||||
|
): |
||||||
|
"""Test :func:`set_define_attr` with one or more class mix condition operator.""" |
||||||
|
attr = "depend_task_list" |
||||||
|
|
||||||
|
task = Task("test-operator", TEST_TYPE) |
||||||
|
|
||||||
|
expect = [] |
||||||
|
sub_condition = [] |
||||||
|
for cond in sub_cond: |
||||||
|
status = Status(task) |
||||||
|
sub_condition.append(cond(status)) |
||||||
|
expect.append( |
||||||
|
{ |
||||||
|
"relation": cond.operator_name(), |
||||||
|
"dependItemList": [ |
||||||
|
{ |
||||||
|
"depTaskCode": task.code, |
||||||
|
"status": status.status_name(), |
||||||
|
} |
||||||
|
], |
||||||
|
} |
||||||
|
) |
||||||
|
co = cond(*sub_condition) |
||||||
|
co.set_define_attr() |
||||||
|
assert getattr(co, attr) == expect |
||||||
|
|
||||||
|
|
||||||
|
@patch( |
||||||
|
"pydolphinscheduler.core.task.Task.gen_code_and_version", |
||||||
|
return_value=(12345, 1), |
||||||
|
) |
||||||
|
@patch( |
||||||
|
"pydolphinscheduler.tasks.condition.Conditions.gen_code_and_version", |
||||||
|
return_value=(123, 1), |
||||||
|
) |
||||||
|
def test_dependent_get_define(mock_condition_code_version, mock_task_code_version): |
||||||
|
"""Test task condition :func:`get_define`.""" |
||||||
|
common_task = Task(name="common_task", task_type="test_task_condition") |
||||||
|
cond_operator = And( |
||||||
|
And( |
||||||
|
SUCCESS(common_task, common_task), |
||||||
|
FAILURE(common_task, common_task), |
||||||
|
), |
||||||
|
Or( |
||||||
|
SUCCESS(common_task, common_task), |
||||||
|
FAILURE(common_task, common_task), |
||||||
|
), |
||||||
|
) |
||||||
|
|
||||||
|
name = "test_condition_get_define" |
||||||
|
expect = { |
||||||
|
"code": 123, |
||||||
|
"name": name, |
||||||
|
"version": 1, |
||||||
|
"description": None, |
||||||
|
"delayTime": 0, |
||||||
|
"taskType": "CONDITIONS", |
||||||
|
"taskParams": { |
||||||
|
"resourceList": [], |
||||||
|
"localParams": [], |
||||||
|
"dependence": { |
||||||
|
"relation": "AND", |
||||||
|
"dependTaskList": [ |
||||||
|
{ |
||||||
|
"relation": "AND", |
||||||
|
"dependItemList": [ |
||||||
|
{"depTaskCode": common_task.code, "status": "SUCCESS"}, |
||||||
|
{"depTaskCode": common_task.code, "status": "SUCCESS"}, |
||||||
|
{"depTaskCode": common_task.code, "status": "FAILURE"}, |
||||||
|
{"depTaskCode": common_task.code, "status": "FAILURE"}, |
||||||
|
], |
||||||
|
}, |
||||||
|
{ |
||||||
|
"relation": "OR", |
||||||
|
"dependItemList": [ |
||||||
|
{"depTaskCode": common_task.code, "status": "SUCCESS"}, |
||||||
|
{"depTaskCode": common_task.code, "status": "SUCCESS"}, |
||||||
|
{"depTaskCode": common_task.code, "status": "FAILURE"}, |
||||||
|
{"depTaskCode": common_task.code, "status": "FAILURE"}, |
||||||
|
], |
||||||
|
}, |
||||||
|
], |
||||||
|
}, |
||||||
|
"conditionResult": {"successNode": [""], "failedNode": [""]}, |
||||||
|
"waitStartTimeout": {}, |
||||||
|
}, |
||||||
|
"flag": "YES", |
||||||
|
"taskPriority": "MEDIUM", |
||||||
|
"workerGroup": "default", |
||||||
|
"failRetryTimes": 0, |
||||||
|
"failRetryInterval": 1, |
||||||
|
"timeoutFlag": "CLOSE", |
||||||
|
"timeoutNotifyStrategy": None, |
||||||
|
"timeout": 0, |
||||||
|
} |
||||||
|
|
||||||
|
task = Conditions(name, condition=cond_operator) |
||||||
|
assert task.get_define() == expect |
||||||
|
|
||||||
|
|
||||||
|
@patch( |
||||||
|
"pydolphinscheduler.core.task.Task.gen_code_and_version", |
||||||
|
return_value=(123, 1), |
||||||
|
) |
||||||
|
def test_condition_set_dep_workflow(mock_task_code_version): |
||||||
|
"""Test task condition set dependence in workflow level.""" |
||||||
|
with ProcessDefinition(name="test-condition-set-dep-workflow") as pd: |
||||||
|
parent = Task(name="parent", task_type=TEST_TYPE) |
||||||
|
condition_success_1 = Task(name="condition_success_1", task_type=TEST_TYPE) |
||||||
|
condition_success_2 = Task(name="condition_success_2", task_type=TEST_TYPE) |
||||||
|
condition_fail = Task(name="condition_fail", task_type=TEST_TYPE) |
||||||
|
cond_operator = And( |
||||||
|
And( |
||||||
|
SUCCESS(condition_success_1, condition_success_2), |
||||||
|
FAILURE(condition_fail), |
||||||
|
), |
||||||
|
) |
||||||
|
|
||||||
|
condition = Conditions(name=TEST_NAME, condition=cond_operator) |
||||||
|
parent >> condition |
||||||
|
# General tasks test |
||||||
|
assert len(pd.tasks) == 5 |
||||||
|
assert sorted(pd.task_list, key=lambda t: t.name) == sorted( |
||||||
|
[ |
||||||
|
parent, |
||||||
|
condition, |
||||||
|
condition_success_1, |
||||||
|
condition_success_2, |
||||||
|
condition_fail, |
||||||
|
], |
||||||
|
key=lambda t: t.name, |
||||||
|
) |
||||||
|
# Task dep test |
||||||
|
assert parent._downstream_task_codes == {condition.code} |
||||||
|
assert condition._upstream_task_codes == {parent.code} |
||||||
|
|
||||||
|
# Condition task dep after ProcessDefinition function get_define called |
||||||
|
assert condition._downstream_task_codes == { |
||||||
|
condition_success_1.code, |
||||||
|
condition_success_2.code, |
||||||
|
condition_fail.code, |
||||||
|
} |
||||||
|
assert all( |
||||||
|
[ |
||||||
|
child._upstream_task_codes == {condition.code} |
||||||
|
for child in [condition_success_1, condition_success_2, condition_fail] |
||||||
|
] |
||||||
|
) |
@ -0,0 +1,124 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""Test Task DataX.""" |
||||||
|
|
||||||
|
from unittest.mock import patch |
||||||
|
|
||||||
|
import pytest |
||||||
|
|
||||||
|
from pydolphinscheduler.tasks.datax import CustomDataX, DataX |
||||||
|
|
||||||
|
|
||||||
|
@patch( |
||||||
|
"pydolphinscheduler.core.database.Database.get_database_info", |
||||||
|
return_value=({"id": 1, "type": "MYSQL"}), |
||||||
|
) |
||||||
|
def test_datax_get_define(mock_datasource): |
||||||
|
"""Test task datax function get_define.""" |
||||||
|
code = 123 |
||||||
|
version = 1 |
||||||
|
name = "test_datax_get_define" |
||||||
|
command = "select name from test_source_table_name" |
||||||
|
datasource_name = "test_datasource" |
||||||
|
datatarget_name = "test_datatarget" |
||||||
|
target_table = "test_target_table_name" |
||||||
|
expect = { |
||||||
|
"code": code, |
||||||
|
"name": name, |
||||||
|
"version": 1, |
||||||
|
"description": None, |
||||||
|
"delayTime": 0, |
||||||
|
"taskType": "DATAX", |
||||||
|
"taskParams": { |
||||||
|
"customConfig": 0, |
||||||
|
"dsType": "MYSQL", |
||||||
|
"dataSource": 1, |
||||||
|
"dtType": "MYSQL", |
||||||
|
"dataTarget": 1, |
||||||
|
"sql": command, |
||||||
|
"targetTable": target_table, |
||||||
|
"jobSpeedByte": 0, |
||||||
|
"jobSpeedRecord": 1000, |
||||||
|
"xms": 1, |
||||||
|
"xmx": 1, |
||||||
|
"preStatements": [], |
||||||
|
"postStatements": [], |
||||||
|
"localParams": [], |
||||||
|
"resourceList": [], |
||||||
|
"dependence": {}, |
||||||
|
"conditionResult": {"successNode": [""], "failedNode": [""]}, |
||||||
|
"waitStartTimeout": {}, |
||||||
|
}, |
||||||
|
"flag": "YES", |
||||||
|
"taskPriority": "MEDIUM", |
||||||
|
"workerGroup": "default", |
||||||
|
"failRetryTimes": 0, |
||||||
|
"failRetryInterval": 1, |
||||||
|
"timeoutFlag": "CLOSE", |
||||||
|
"timeoutNotifyStrategy": None, |
||||||
|
"timeout": 0, |
||||||
|
} |
||||||
|
with patch( |
||||||
|
"pydolphinscheduler.core.task.Task.gen_code_and_version", |
||||||
|
return_value=(code, version), |
||||||
|
): |
||||||
|
task = DataX(name, datasource_name, datatarget_name, command, target_table) |
||||||
|
assert task.get_define() == expect |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("json_template", ["json_template"]) |
||||||
|
def test_custom_datax_get_define(json_template): |
||||||
|
"""Test task custom datax function get_define.""" |
||||||
|
code = 123 |
||||||
|
version = 1 |
||||||
|
name = "test_custom_datax_get_define" |
||||||
|
expect = { |
||||||
|
"code": code, |
||||||
|
"name": name, |
||||||
|
"version": 1, |
||||||
|
"description": None, |
||||||
|
"delayTime": 0, |
||||||
|
"taskType": "DATAX", |
||||||
|
"taskParams": { |
||||||
|
"customConfig": 1, |
||||||
|
"json": json_template, |
||||||
|
"xms": 1, |
||||||
|
"xmx": 1, |
||||||
|
"localParams": [], |
||||||
|
"resourceList": [], |
||||||
|
"dependence": {}, |
||||||
|
"conditionResult": {"successNode": [""], "failedNode": [""]}, |
||||||
|
"waitStartTimeout": {}, |
||||||
|
}, |
||||||
|
"flag": "YES", |
||||||
|
"taskPriority": "MEDIUM", |
||||||
|
"workerGroup": "default", |
||||||
|
"failRetryTimes": 0, |
||||||
|
"failRetryInterval": 1, |
||||||
|
"timeoutFlag": "CLOSE", |
||||||
|
"timeoutNotifyStrategy": None, |
||||||
|
"timeout": 0, |
||||||
|
} |
||||||
|
with patch( |
||||||
|
"pydolphinscheduler.core.task.Task.gen_code_and_version", |
||||||
|
return_value=(code, version), |
||||||
|
): |
||||||
|
task = CustomDataX(name, json_template) |
||||||
|
print(task.get_define()) |
||||||
|
print(expect) |
||||||
|
assert task.get_define() == expect |
@ -0,0 +1,793 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""Test Task dependent.""" |
||||||
|
import itertools |
||||||
|
from typing import Dict, List, Optional, Tuple, Union |
||||||
|
from unittest.mock import patch |
||||||
|
|
||||||
|
import pytest |
||||||
|
|
||||||
|
from pydolphinscheduler.exceptions import PyDSParamException |
||||||
|
from pydolphinscheduler.tasks.dependent import ( |
||||||
|
And, |
||||||
|
Dependent, |
||||||
|
DependentDate, |
||||||
|
DependentItem, |
||||||
|
DependentOperator, |
||||||
|
Or, |
||||||
|
) |
||||||
|
|
||||||
|
TEST_PROJECT = "test-project" |
||||||
|
TEST_PROCESS_DEFINITION = "test-process-definition" |
||||||
|
TEST_TASK = "test-task" |
||||||
|
TEST_PROJECT_CODE, TEST_DEFINITION_CODE, TEST_TASK_CODE = 12345, 123456, 1234567 |
||||||
|
|
||||||
|
TEST_OPERATOR_LIST = ("AND", "OR") |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
"dep_date, dep_cycle", |
||||||
|
[ |
||||||
|
# hour |
||||||
|
(DependentDate.CURRENT_HOUR, "hour"), |
||||||
|
(DependentDate.LAST_ONE_HOUR, "hour"), |
||||||
|
(DependentDate.LAST_TWO_HOURS, "hour"), |
||||||
|
(DependentDate.LAST_THREE_HOURS, "hour"), |
||||||
|
(DependentDate.LAST_TWENTY_FOUR_HOURS, "hour"), |
||||||
|
# day |
||||||
|
(DependentDate.TODAY, "day"), |
||||||
|
(DependentDate.LAST_ONE_DAYS, "day"), |
||||||
|
(DependentDate.LAST_TWO_DAYS, "day"), |
||||||
|
(DependentDate.LAST_THREE_DAYS, "day"), |
||||||
|
(DependentDate.LAST_SEVEN_DAYS, "day"), |
||||||
|
# week |
||||||
|
(DependentDate.THIS_WEEK, "week"), |
||||||
|
(DependentDate.LAST_WEEK, "week"), |
||||||
|
(DependentDate.LAST_MONDAY, "week"), |
||||||
|
(DependentDate.LAST_TUESDAY, "week"), |
||||||
|
(DependentDate.LAST_WEDNESDAY, "week"), |
||||||
|
(DependentDate.LAST_THURSDAY, "week"), |
||||||
|
(DependentDate.LAST_FRIDAY, "week"), |
||||||
|
(DependentDate.LAST_SATURDAY, "week"), |
||||||
|
(DependentDate.LAST_SUNDAY, "week"), |
||||||
|
# month |
||||||
|
(DependentDate.THIS_MONTH, "month"), |
||||||
|
(DependentDate.LAST_MONTH, "month"), |
||||||
|
(DependentDate.LAST_MONTH_BEGIN, "month"), |
||||||
|
(DependentDate.LAST_MONTH_END, "month"), |
||||||
|
], |
||||||
|
) |
||||||
|
@patch( |
||||||
|
"pydolphinscheduler.tasks.dependent.DependentItem.get_code_from_gateway", |
||||||
|
return_value={ |
||||||
|
"projectCode": TEST_PROJECT_CODE, |
||||||
|
"processDefinitionCode": TEST_DEFINITION_CODE, |
||||||
|
"taskDefinitionCode": TEST_TASK_CODE, |
||||||
|
}, |
||||||
|
) |
||||||
|
def test_dependent_item_get_define(mock_task_info, dep_date, dep_cycle): |
||||||
|
"""Test dependent.DependentItem get define. |
||||||
|
|
||||||
|
Here we have test some cases as below. |
||||||
|
```py |
||||||
|
{ |
||||||
|
"projectCode": "project code", |
||||||
|
"definitionCode": "definition code", |
||||||
|
"depTaskCode": "dep task code", |
||||||
|
"cycle": "day", |
||||||
|
"dateValue": "today" |
||||||
|
} |
||||||
|
``` |
||||||
|
""" |
||||||
|
attr = { |
||||||
|
"project_name": TEST_PROJECT, |
||||||
|
"process_definition_name": TEST_PROCESS_DEFINITION, |
||||||
|
"dependent_task_name": TEST_TASK, |
||||||
|
"dependent_date": dep_date, |
||||||
|
} |
||||||
|
expect = { |
||||||
|
"projectCode": TEST_PROJECT_CODE, |
||||||
|
"definitionCode": TEST_DEFINITION_CODE, |
||||||
|
"depTaskCode": TEST_TASK_CODE, |
||||||
|
"cycle": dep_cycle, |
||||||
|
"dateValue": dep_date, |
||||||
|
} |
||||||
|
task = DependentItem(**attr) |
||||||
|
assert expect == task.get_define() |
||||||
|
|
||||||
|
|
||||||
|
def test_dependent_item_date_error(): |
||||||
|
"""Test error when pass None to dependent_date.""" |
||||||
|
with pytest.raises( |
||||||
|
PyDSParamException, match="Parameter dependent_date must provider.*?" |
||||||
|
): |
||||||
|
DependentItem( |
||||||
|
project_name=TEST_PROJECT, |
||||||
|
process_definition_name=TEST_PROCESS_DEFINITION, |
||||||
|
dependent_date=None, |
||||||
|
) |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
"task_name, result", |
||||||
|
[ |
||||||
|
({"dependent_task_name": TEST_TASK}, TEST_TASK), |
||||||
|
({}, None), |
||||||
|
], |
||||||
|
) |
||||||
|
def test_dependent_item_code_parameter(task_name: dict, result: Optional[str]): |
||||||
|
"""Test dependent item property code_parameter.""" |
||||||
|
dependent_item = DependentItem( |
||||||
|
project_name=TEST_PROJECT, |
||||||
|
process_definition_name=TEST_PROCESS_DEFINITION, |
||||||
|
**task_name, |
||||||
|
) |
||||||
|
expect = (TEST_PROJECT, TEST_PROCESS_DEFINITION, result) |
||||||
|
assert dependent_item.code_parameter == expect |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
"arg_list", |
||||||
|
[ |
||||||
|
[1, 2], |
||||||
|
[ |
||||||
|
DependentItem( |
||||||
|
project_name=TEST_PROJECT, |
||||||
|
process_definition_name=TEST_PROCESS_DEFINITION, |
||||||
|
), |
||||||
|
1, |
||||||
|
], |
||||||
|
[ |
||||||
|
And( |
||||||
|
DependentItem( |
||||||
|
project_name=TEST_PROJECT, |
||||||
|
process_definition_name=TEST_PROCESS_DEFINITION, |
||||||
|
) |
||||||
|
), |
||||||
|
1, |
||||||
|
], |
||||||
|
[ |
||||||
|
DependentItem( |
||||||
|
project_name=TEST_PROJECT, |
||||||
|
process_definition_name=TEST_PROCESS_DEFINITION, |
||||||
|
), |
||||||
|
And( |
||||||
|
DependentItem( |
||||||
|
project_name=TEST_PROJECT, |
||||||
|
process_definition_name=TEST_PROCESS_DEFINITION, |
||||||
|
) |
||||||
|
), |
||||||
|
], |
||||||
|
], |
||||||
|
) |
||||||
|
@patch( |
||||||
|
"pydolphinscheduler.tasks.dependent.DependentItem.get_code_from_gateway", |
||||||
|
return_value={ |
||||||
|
"projectCode": TEST_PROJECT_CODE, |
||||||
|
"processDefinitionCode": TEST_DEFINITION_CODE, |
||||||
|
"taskDefinitionCode": TEST_TASK_CODE, |
||||||
|
}, |
||||||
|
) |
||||||
|
def test_dependent_operator_set_define_error(mock_code, arg_list): |
||||||
|
"""Test dependent operator function :func:`set_define` with not support type.""" |
||||||
|
dep_op = DependentOperator(*arg_list) |
||||||
|
with pytest.raises(PyDSParamException, match="Dependent .*? operator.*?"): |
||||||
|
dep_op.set_define_attr() |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
# Test dependent operator, Test dependent item parameters, expect operator define |
||||||
|
"operators, kwargs, expect", |
||||||
|
[ |
||||||
|
# Test dependent operator (And | Or) with single dependent item |
||||||
|
( |
||||||
|
(And, Or), |
||||||
|
( |
||||||
|
{ |
||||||
|
"project_name": TEST_PROJECT, |
||||||
|
"process_definition_name": TEST_PROCESS_DEFINITION, |
||||||
|
"dependent_task_name": TEST_TASK, |
||||||
|
"dependent_date": DependentDate.LAST_MONTH_END, |
||||||
|
}, |
||||||
|
), |
||||||
|
[ |
||||||
|
{ |
||||||
|
"relation": op, |
||||||
|
"dependItemList": [ |
||||||
|
{ |
||||||
|
"projectCode": TEST_PROJECT_CODE, |
||||||
|
"definitionCode": TEST_DEFINITION_CODE, |
||||||
|
"depTaskCode": TEST_TASK_CODE, |
||||||
|
"cycle": "month", |
||||||
|
"dateValue": DependentDate.LAST_MONTH_END, |
||||||
|
}, |
||||||
|
], |
||||||
|
} |
||||||
|
for op in TEST_OPERATOR_LIST |
||||||
|
], |
||||||
|
), |
||||||
|
# Test dependent operator (And | Or) with two dependent item |
||||||
|
( |
||||||
|
(And, Or), |
||||||
|
( |
||||||
|
{ |
||||||
|
"project_name": TEST_PROJECT, |
||||||
|
"process_definition_name": TEST_PROCESS_DEFINITION, |
||||||
|
"dependent_task_name": TEST_TASK, |
||||||
|
"dependent_date": DependentDate.LAST_MONTH_END, |
||||||
|
}, |
||||||
|
{ |
||||||
|
"project_name": TEST_PROJECT, |
||||||
|
"process_definition_name": TEST_PROCESS_DEFINITION, |
||||||
|
"dependent_task_name": TEST_TASK, |
||||||
|
"dependent_date": DependentDate.LAST_WEEK, |
||||||
|
}, |
||||||
|
), |
||||||
|
[ |
||||||
|
{ |
||||||
|
"relation": op, |
||||||
|
"dependItemList": [ |
||||||
|
{ |
||||||
|
"projectCode": TEST_PROJECT_CODE, |
||||||
|
"definitionCode": TEST_DEFINITION_CODE, |
||||||
|
"depTaskCode": TEST_TASK_CODE, |
||||||
|
"cycle": "month", |
||||||
|
"dateValue": DependentDate.LAST_MONTH_END, |
||||||
|
}, |
||||||
|
{ |
||||||
|
"projectCode": TEST_PROJECT_CODE, |
||||||
|
"definitionCode": TEST_DEFINITION_CODE, |
||||||
|
"depTaskCode": TEST_TASK_CODE, |
||||||
|
"cycle": "week", |
||||||
|
"dateValue": DependentDate.LAST_WEEK, |
||||||
|
}, |
||||||
|
], |
||||||
|
} |
||||||
|
for op in TEST_OPERATOR_LIST |
||||||
|
], |
||||||
|
), |
||||||
|
# Test dependent operator (And | Or) with multiply dependent item |
||||||
|
( |
||||||
|
(And, Or), |
||||||
|
( |
||||||
|
{ |
||||||
|
"project_name": TEST_PROJECT, |
||||||
|
"process_definition_name": TEST_PROCESS_DEFINITION, |
||||||
|
"dependent_task_name": TEST_TASK, |
||||||
|
"dependent_date": DependentDate.LAST_MONTH_END, |
||||||
|
}, |
||||||
|
{ |
||||||
|
"project_name": TEST_PROJECT, |
||||||
|
"process_definition_name": TEST_PROCESS_DEFINITION, |
||||||
|
"dependent_task_name": TEST_TASK, |
||||||
|
"dependent_date": DependentDate.LAST_WEEK, |
||||||
|
}, |
||||||
|
{ |
||||||
|
"project_name": TEST_PROJECT, |
||||||
|
"process_definition_name": TEST_PROCESS_DEFINITION, |
||||||
|
"dependent_task_name": TEST_TASK, |
||||||
|
"dependent_date": DependentDate.LAST_ONE_DAYS, |
||||||
|
}, |
||||||
|
), |
||||||
|
[ |
||||||
|
{ |
||||||
|
"relation": op, |
||||||
|
"dependItemList": [ |
||||||
|
{ |
||||||
|
"projectCode": TEST_PROJECT_CODE, |
||||||
|
"definitionCode": TEST_DEFINITION_CODE, |
||||||
|
"depTaskCode": TEST_TASK_CODE, |
||||||
|
"cycle": "month", |
||||||
|
"dateValue": DependentDate.LAST_MONTH_END, |
||||||
|
}, |
||||||
|
{ |
||||||
|
"projectCode": TEST_PROJECT_CODE, |
||||||
|
"definitionCode": TEST_DEFINITION_CODE, |
||||||
|
"depTaskCode": TEST_TASK_CODE, |
||||||
|
"cycle": "week", |
||||||
|
"dateValue": DependentDate.LAST_WEEK, |
||||||
|
}, |
||||||
|
{ |
||||||
|
"projectCode": TEST_PROJECT_CODE, |
||||||
|
"definitionCode": TEST_DEFINITION_CODE, |
||||||
|
"depTaskCode": TEST_TASK_CODE, |
||||||
|
"cycle": "day", |
||||||
|
"dateValue": DependentDate.LAST_ONE_DAYS, |
||||||
|
}, |
||||||
|
], |
||||||
|
} |
||||||
|
for op in TEST_OPERATOR_LIST |
||||||
|
], |
||||||
|
), |
||||||
|
], |
||||||
|
) |
||||||
|
@patch( |
||||||
|
"pydolphinscheduler.tasks.dependent.DependentItem.get_code_from_gateway", |
||||||
|
return_value={ |
||||||
|
"projectCode": TEST_PROJECT_CODE, |
||||||
|
"processDefinitionCode": TEST_DEFINITION_CODE, |
||||||
|
"taskDefinitionCode": TEST_TASK_CODE, |
||||||
|
}, |
||||||
|
) |
||||||
|
def test_operator_dependent_item( |
||||||
|
mock_code_info, |
||||||
|
operators: Tuple[DependentOperator], |
||||||
|
kwargs: Tuple[dict], |
||||||
|
expect: List[Dict], |
||||||
|
): |
||||||
|
"""Test DependentOperator(DependentItem) function get_define. |
||||||
|
|
||||||
|
Here we have test some cases as below, including single dependentItem and multiply dependentItem. |
||||||
|
```py |
||||||
|
{ |
||||||
|
"relation": "AND", |
||||||
|
"dependItemList": [ |
||||||
|
{ |
||||||
|
"projectCode": "project code", |
||||||
|
"definitionCode": "definition code", |
||||||
|
"depTaskCode": "dep task code", |
||||||
|
"cycle": "day", |
||||||
|
"dateValue": "today" |
||||||
|
}, |
||||||
|
... |
||||||
|
] |
||||||
|
} |
||||||
|
``` |
||||||
|
""" |
||||||
|
for idx, operator in enumerate(operators): |
||||||
|
# Use variable to keep one or more dependent item to test dependent operator behavior |
||||||
|
dependent_item_list = [] |
||||||
|
for kwarg in kwargs: |
||||||
|
dependent_item = DependentItem(**kwarg) |
||||||
|
dependent_item_list.append(dependent_item) |
||||||
|
op = operator(*dependent_item_list) |
||||||
|
assert expect[idx] == op.get_define() |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
# Test dependent operator, Test dependent item parameters, expect operator define |
||||||
|
"operators, args, expect", |
||||||
|
[ |
||||||
|
# Test dependent operator (And | Or) with single dependent task list |
||||||
|
( |
||||||
|
(And, Or), |
||||||
|
( |
||||||
|
(And, Or), |
||||||
|
( |
||||||
|
{ |
||||||
|
"project_name": TEST_PROJECT, |
||||||
|
"process_definition_name": TEST_PROCESS_DEFINITION, |
||||||
|
"dependent_task_name": TEST_TASK, |
||||||
|
"dependent_date": DependentDate.LAST_MONTH_END, |
||||||
|
}, |
||||||
|
), |
||||||
|
), |
||||||
|
[ |
||||||
|
{ |
||||||
|
"relation": par_op, |
||||||
|
"dependTaskList": [ |
||||||
|
{ |
||||||
|
"relation": chr_op, |
||||||
|
"dependItemList": [ |
||||||
|
{ |
||||||
|
"projectCode": TEST_PROJECT_CODE, |
||||||
|
"definitionCode": TEST_DEFINITION_CODE, |
||||||
|
"depTaskCode": TEST_TASK_CODE, |
||||||
|
"cycle": "month", |
||||||
|
"dateValue": DependentDate.LAST_MONTH_END, |
||||||
|
}, |
||||||
|
], |
||||||
|
} |
||||||
|
], |
||||||
|
} |
||||||
|
for (par_op, chr_op) in itertools.product( |
||||||
|
TEST_OPERATOR_LIST, TEST_OPERATOR_LIST |
||||||
|
) |
||||||
|
], |
||||||
|
), |
||||||
|
# Test dependent operator (And | Or) with two dependent task list |
||||||
|
( |
||||||
|
(And, Or), |
||||||
|
( |
||||||
|
(And, Or), |
||||||
|
( |
||||||
|
{ |
||||||
|
"project_name": TEST_PROJECT, |
||||||
|
"process_definition_name": TEST_PROCESS_DEFINITION, |
||||||
|
"dependent_task_name": TEST_TASK, |
||||||
|
"dependent_date": DependentDate.LAST_MONTH_END, |
||||||
|
}, |
||||||
|
{ |
||||||
|
"project_name": TEST_PROJECT, |
||||||
|
"process_definition_name": TEST_PROCESS_DEFINITION, |
||||||
|
"dependent_task_name": TEST_TASK, |
||||||
|
"dependent_date": DependentDate.LAST_WEEK, |
||||||
|
}, |
||||||
|
), |
||||||
|
), |
||||||
|
[ |
||||||
|
{ |
||||||
|
"relation": par_op, |
||||||
|
"dependTaskList": [ |
||||||
|
{ |
||||||
|
"relation": chr_op, |
||||||
|
"dependItemList": [ |
||||||
|
{ |
||||||
|
"projectCode": TEST_PROJECT_CODE, |
||||||
|
"definitionCode": TEST_DEFINITION_CODE, |
||||||
|
"depTaskCode": TEST_TASK_CODE, |
||||||
|
"cycle": "month", |
||||||
|
"dateValue": DependentDate.LAST_MONTH_END, |
||||||
|
}, |
||||||
|
{ |
||||||
|
"projectCode": TEST_PROJECT_CODE, |
||||||
|
"definitionCode": TEST_DEFINITION_CODE, |
||||||
|
"depTaskCode": TEST_TASK_CODE, |
||||||
|
"cycle": "week", |
||||||
|
"dateValue": DependentDate.LAST_WEEK, |
||||||
|
}, |
||||||
|
], |
||||||
|
} |
||||||
|
], |
||||||
|
} |
||||||
|
for (par_op, chr_op) in itertools.product( |
||||||
|
TEST_OPERATOR_LIST, TEST_OPERATOR_LIST |
||||||
|
) |
||||||
|
], |
||||||
|
), |
||||||
|
# Test dependent operator (And | Or) with multiply dependent task list |
||||||
|
( |
||||||
|
(And, Or), |
||||||
|
( |
||||||
|
(And, Or), |
||||||
|
( |
||||||
|
{ |
||||||
|
"project_name": TEST_PROJECT, |
||||||
|
"process_definition_name": TEST_PROCESS_DEFINITION, |
||||||
|
"dependent_task_name": TEST_TASK, |
||||||
|
"dependent_date": DependentDate.LAST_MONTH_END, |
||||||
|
}, |
||||||
|
{ |
||||||
|
"project_name": TEST_PROJECT, |
||||||
|
"process_definition_name": TEST_PROCESS_DEFINITION, |
||||||
|
"dependent_task_name": TEST_TASK, |
||||||
|
"dependent_date": DependentDate.LAST_WEEK, |
||||||
|
}, |
||||||
|
{ |
||||||
|
"project_name": TEST_PROJECT, |
||||||
|
"process_definition_name": TEST_PROCESS_DEFINITION, |
||||||
|
"dependent_task_name": TEST_TASK, |
||||||
|
"dependent_date": DependentDate.LAST_ONE_DAYS, |
||||||
|
}, |
||||||
|
), |
||||||
|
), |
||||||
|
[ |
||||||
|
{ |
||||||
|
"relation": par_op, |
||||||
|
"dependTaskList": [ |
||||||
|
{ |
||||||
|
"relation": chr_op, |
||||||
|
"dependItemList": [ |
||||||
|
{ |
||||||
|
"projectCode": TEST_PROJECT_CODE, |
||||||
|
"definitionCode": TEST_DEFINITION_CODE, |
||||||
|
"depTaskCode": TEST_TASK_CODE, |
||||||
|
"cycle": "month", |
||||||
|
"dateValue": DependentDate.LAST_MONTH_END, |
||||||
|
}, |
||||||
|
{ |
||||||
|
"projectCode": TEST_PROJECT_CODE, |
||||||
|
"definitionCode": TEST_DEFINITION_CODE, |
||||||
|
"depTaskCode": TEST_TASK_CODE, |
||||||
|
"cycle": "week", |
||||||
|
"dateValue": DependentDate.LAST_WEEK, |
||||||
|
}, |
||||||
|
{ |
||||||
|
"projectCode": TEST_PROJECT_CODE, |
||||||
|
"definitionCode": TEST_DEFINITION_CODE, |
||||||
|
"depTaskCode": TEST_TASK_CODE, |
||||||
|
"cycle": "day", |
||||||
|
"dateValue": DependentDate.LAST_ONE_DAYS, |
||||||
|
}, |
||||||
|
], |
||||||
|
} |
||||||
|
], |
||||||
|
} |
||||||
|
for (par_op, chr_op) in itertools.product( |
||||||
|
TEST_OPERATOR_LIST, TEST_OPERATOR_LIST |
||||||
|
) |
||||||
|
], |
||||||
|
), |
||||||
|
], |
||||||
|
) |
||||||
|
@patch( |
||||||
|
"pydolphinscheduler.tasks.dependent.DependentItem.get_code_from_gateway", |
||||||
|
return_value={ |
||||||
|
"projectCode": TEST_PROJECT_CODE, |
||||||
|
"processDefinitionCode": TEST_DEFINITION_CODE, |
||||||
|
"taskDefinitionCode": TEST_TASK_CODE, |
||||||
|
}, |
||||||
|
) |
||||||
|
def test_operator_dependent_task_list_multi_dependent_item( |
||||||
|
mock_code_info, |
||||||
|
operators: Tuple[DependentOperator], |
||||||
|
args: Tuple[Union[Tuple, dict]], |
||||||
|
expect: List[Dict], |
||||||
|
): |
||||||
|
"""Test DependentOperator(DependentOperator(DependentItem)) single operator function get_define. |
||||||
|
|
||||||
|
Here we have test some cases as below. This test case only test single DependTaskList with one or |
||||||
|
multiply dependItemList. |
||||||
|
```py |
||||||
|
{ |
||||||
|
"relation": "OR", |
||||||
|
"dependTaskList": [ |
||||||
|
{ |
||||||
|
"relation": "AND", |
||||||
|
"dependItemList": [ |
||||||
|
{ |
||||||
|
"projectCode": "project code", |
||||||
|
"definitionCode": "definition code", |
||||||
|
"depTaskCode": "dep task code", |
||||||
|
"cycle": "day", |
||||||
|
"dateValue": "today" |
||||||
|
}, |
||||||
|
... |
||||||
|
] |
||||||
|
}, |
||||||
|
] |
||||||
|
} |
||||||
|
``` |
||||||
|
""" |
||||||
|
# variable expect_idx record idx should be use to get specific expect |
||||||
|
expect_idx = 0 |
||||||
|
|
||||||
|
for op_idx, operator in enumerate(operators): |
||||||
|
dependent_operator = args[0] |
||||||
|
dependent_item_kwargs = args[1] |
||||||
|
|
||||||
|
for dop_idx, dpt_op in enumerate(dependent_operator): |
||||||
|
dependent_item_list = [] |
||||||
|
for dpt_kwargs in dependent_item_kwargs: |
||||||
|
dpti = DependentItem(**dpt_kwargs) |
||||||
|
dependent_item_list.append(dpti) |
||||||
|
child_dep_op = dpt_op(*dependent_item_list) |
||||||
|
op = operator(child_dep_op) |
||||||
|
assert expect[expect_idx] == op.get_define() |
||||||
|
expect_idx += 1 |
||||||
|
|
||||||
|
|
||||||
|
def get_dep_task_list(*operator): |
||||||
|
"""Return dependent task list from given operators list.""" |
||||||
|
result = [] |
||||||
|
for op in operator: |
||||||
|
result.append( |
||||||
|
{ |
||||||
|
"relation": op.operator_name(), |
||||||
|
"dependItemList": [ |
||||||
|
{ |
||||||
|
"projectCode": TEST_PROJECT_CODE, |
||||||
|
"definitionCode": TEST_DEFINITION_CODE, |
||||||
|
"depTaskCode": TEST_TASK_CODE, |
||||||
|
"cycle": "month", |
||||||
|
"dateValue": DependentDate.LAST_MONTH_END, |
||||||
|
}, |
||||||
|
], |
||||||
|
} |
||||||
|
) |
||||||
|
return result |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
# Test dependent operator, Test dependent item parameters, expect operator define |
||||||
|
"operators, args, expect", |
||||||
|
[ |
||||||
|
# Test dependent operator (And | Or) with two dependent task list |
||||||
|
( |
||||||
|
(And, Or), |
||||||
|
( |
||||||
|
((And, And), (And, Or), (Or, And), (Or, Or)), |
||||||
|
{ |
||||||
|
"project_name": TEST_PROJECT, |
||||||
|
"process_definition_name": TEST_PROCESS_DEFINITION, |
||||||
|
"dependent_task_name": TEST_TASK, |
||||||
|
"dependent_date": DependentDate.LAST_MONTH_END, |
||||||
|
}, |
||||||
|
), |
||||||
|
[ |
||||||
|
{ |
||||||
|
"relation": parent_op.operator_name(), |
||||||
|
"dependTaskList": get_dep_task_list(*child_ops), |
||||||
|
} |
||||||
|
for parent_op in (And, Or) |
||||||
|
for child_ops in ((And, And), (And, Or), (Or, And), (Or, Or)) |
||||||
|
], |
||||||
|
), |
||||||
|
# Test dependent operator (And | Or) with multiple dependent task list |
||||||
|
( |
||||||
|
(And, Or), |
||||||
|
( |
||||||
|
((And, And, And), (And, And, And, And), (And, And, And, And, And)), |
||||||
|
{ |
||||||
|
"project_name": TEST_PROJECT, |
||||||
|
"process_definition_name": TEST_PROCESS_DEFINITION, |
||||||
|
"dependent_task_name": TEST_TASK, |
||||||
|
"dependent_date": DependentDate.LAST_MONTH_END, |
||||||
|
}, |
||||||
|
), |
||||||
|
[ |
||||||
|
{ |
||||||
|
"relation": parent_op.operator_name(), |
||||||
|
"dependTaskList": get_dep_task_list(*child_ops), |
||||||
|
} |
||||||
|
for parent_op in (And, Or) |
||||||
|
for child_ops in ( |
||||||
|
(And, And, And), |
||||||
|
(And, And, And, And), |
||||||
|
(And, And, And, And, And), |
||||||
|
) |
||||||
|
], |
||||||
|
), |
||||||
|
], |
||||||
|
) |
||||||
|
@patch( |
||||||
|
"pydolphinscheduler.tasks.dependent.DependentItem.get_code_from_gateway", |
||||||
|
return_value={ |
||||||
|
"projectCode": TEST_PROJECT_CODE, |
||||||
|
"processDefinitionCode": TEST_DEFINITION_CODE, |
||||||
|
"taskDefinitionCode": TEST_TASK_CODE, |
||||||
|
}, |
||||||
|
) |
||||||
|
def test_operator_dependent_task_list_multi_dependent_list( |
||||||
|
mock_code_info, |
||||||
|
operators: Tuple[DependentOperator], |
||||||
|
args: Tuple[Union[Tuple, dict]], |
||||||
|
expect: List[Dict], |
||||||
|
): |
||||||
|
"""Test DependentOperator(DependentOperator(DependentItem)) multiply operator function get_define. |
||||||
|
|
||||||
|
Here we have test some cases as below. This test case only test single DependTaskList with one or |
||||||
|
multiply dependTaskList. |
||||||
|
```py |
||||||
|
{ |
||||||
|
"relation": "OR", |
||||||
|
"dependTaskList": [ |
||||||
|
{ |
||||||
|
"relation": "AND", |
||||||
|
"dependItemList": [ |
||||||
|
{ |
||||||
|
"projectCode": "project code", |
||||||
|
"definitionCode": "definition code", |
||||||
|
"depTaskCode": "dep task code", |
||||||
|
"cycle": "day", |
||||||
|
"dateValue": "today" |
||||||
|
} |
||||||
|
] |
||||||
|
}, |
||||||
|
... |
||||||
|
] |
||||||
|
} |
||||||
|
``` |
||||||
|
""" |
||||||
|
# variable expect_idx record idx should be use to get specific expect |
||||||
|
expect_idx = 0 |
||||||
|
for op_idx, operator in enumerate(operators): |
||||||
|
dependent_operator = args[0] |
||||||
|
dependent_item_kwargs = args[1] |
||||||
|
|
||||||
|
for dop_idx, dpt_ops in enumerate(dependent_operator): |
||||||
|
dependent_task_list = [ |
||||||
|
dpt_op(DependentItem(**dependent_item_kwargs)) for dpt_op in dpt_ops |
||||||
|
] |
||||||
|
op = operator(*dependent_task_list) |
||||||
|
assert ( |
||||||
|
expect[expect_idx] == op.get_define() |
||||||
|
), f"Failed with operator syntax {operator}.{dpt_ops}" |
||||||
|
expect_idx += 1 |
||||||
|
|
||||||
|
|
||||||
|
@patch( |
||||||
|
"pydolphinscheduler.tasks.dependent.DependentItem.get_code_from_gateway", |
||||||
|
return_value={ |
||||||
|
"projectCode": TEST_PROJECT_CODE, |
||||||
|
"processDefinitionCode": TEST_DEFINITION_CODE, |
||||||
|
"taskDefinitionCode": TEST_TASK_CODE, |
||||||
|
}, |
||||||
|
) |
||||||
|
@patch( |
||||||
|
"pydolphinscheduler.core.task.Task.gen_code_and_version", |
||||||
|
return_value=(123, 1), |
||||||
|
) |
||||||
|
def test_dependent_get_define(mock_code_version, mock_dep_code): |
||||||
|
"""Test task dependent function get_define.""" |
||||||
|
project_name = "test-dep-project" |
||||||
|
process_definition_name = "test-dep-definition" |
||||||
|
dependent_task_name = "test-dep-task" |
||||||
|
dep_operator = And( |
||||||
|
Or( |
||||||
|
# test dependence with add tasks |
||||||
|
DependentItem( |
||||||
|
project_name=project_name, |
||||||
|
process_definition_name=process_definition_name, |
||||||
|
) |
||||||
|
), |
||||||
|
And( |
||||||
|
# test dependence with specific task |
||||||
|
DependentItem( |
||||||
|
project_name=project_name, |
||||||
|
process_definition_name=process_definition_name, |
||||||
|
dependent_task_name=dependent_task_name, |
||||||
|
) |
||||||
|
), |
||||||
|
) |
||||||
|
|
||||||
|
name = "test_dependent_get_define" |
||||||
|
expect = { |
||||||
|
"code": 123, |
||||||
|
"name": name, |
||||||
|
"version": 1, |
||||||
|
"description": None, |
||||||
|
"delayTime": 0, |
||||||
|
"taskType": "DEPENDENT", |
||||||
|
"taskParams": { |
||||||
|
"resourceList": [], |
||||||
|
"localParams": [], |
||||||
|
"dependence": { |
||||||
|
"relation": "AND", |
||||||
|
"dependTaskList": [ |
||||||
|
{ |
||||||
|
"relation": "OR", |
||||||
|
"dependItemList": [ |
||||||
|
{ |
||||||
|
"projectCode": TEST_PROJECT_CODE, |
||||||
|
"definitionCode": TEST_DEFINITION_CODE, |
||||||
|
"depTaskCode": "0", |
||||||
|
"cycle": "day", |
||||||
|
"dateValue": "today", |
||||||
|
} |
||||||
|
], |
||||||
|
}, |
||||||
|
{ |
||||||
|
"relation": "AND", |
||||||
|
"dependItemList": [ |
||||||
|
{ |
||||||
|
"projectCode": TEST_PROJECT_CODE, |
||||||
|
"definitionCode": TEST_DEFINITION_CODE, |
||||||
|
"depTaskCode": TEST_TASK_CODE, |
||||||
|
"cycle": "day", |
||||||
|
"dateValue": "today", |
||||||
|
} |
||||||
|
], |
||||||
|
}, |
||||||
|
], |
||||||
|
}, |
||||||
|
"conditionResult": {"successNode": [""], "failedNode": [""]}, |
||||||
|
"waitStartTimeout": {}, |
||||||
|
}, |
||||||
|
"flag": "YES", |
||||||
|
"taskPriority": "MEDIUM", |
||||||
|
"workerGroup": "default", |
||||||
|
"failRetryTimes": 0, |
||||||
|
"failRetryInterval": 1, |
||||||
|
"timeoutFlag": "CLOSE", |
||||||
|
"timeoutNotifyStrategy": None, |
||||||
|
"timeout": 0, |
||||||
|
} |
||||||
|
|
||||||
|
task = Dependent(name, dependence=dep_operator) |
||||||
|
assert task.get_define() == expect |
@ -0,0 +1,144 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""Test Task HTTP.""" |
||||||
|
|
||||||
|
from unittest.mock import patch |
||||||
|
|
||||||
|
import pytest |
||||||
|
|
||||||
|
from pydolphinscheduler.exceptions import PyDSParamException |
||||||
|
from pydolphinscheduler.tasks.http import Http, HttpCheckCondition, HttpMethod |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
"class_name, attrs", |
||||||
|
[ |
||||||
|
(HttpMethod, ("GET", "POST", "HEAD", "PUT", "DELETE")), |
||||||
|
( |
||||||
|
HttpCheckCondition, |
||||||
|
( |
||||||
|
"STATUS_CODE_DEFAULT", |
||||||
|
"STATUS_CODE_CUSTOM", |
||||||
|
"BODY_CONTAINS", |
||||||
|
"BODY_NOT_CONTAINS", |
||||||
|
), |
||||||
|
), |
||||||
|
], |
||||||
|
) |
||||||
|
def test_attr_exists(class_name, attrs): |
||||||
|
"""Test weather class HttpMethod and HttpCheckCondition contain specific attribute.""" |
||||||
|
assert all(hasattr(class_name, attr) for attr in attrs) |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
"attr, expect", |
||||||
|
[ |
||||||
|
( |
||||||
|
{"url": "https://www.apache.org"}, |
||||||
|
{ |
||||||
|
"url": "https://www.apache.org", |
||||||
|
"httpMethod": "GET", |
||||||
|
"httpParams": [], |
||||||
|
"httpCheckCondition": "STATUS_CODE_DEFAULT", |
||||||
|
"condition": None, |
||||||
|
"connectTimeout": 60000, |
||||||
|
"socketTimeout": 60000, |
||||||
|
"localParams": [], |
||||||
|
"resourceList": [], |
||||||
|
"dependence": {}, |
||||||
|
"waitStartTimeout": {}, |
||||||
|
"conditionResult": {"successNode": [""], "failedNode": [""]}, |
||||||
|
}, |
||||||
|
) |
||||||
|
], |
||||||
|
) |
||||||
|
@patch( |
||||||
|
"pydolphinscheduler.core.task.Task.gen_code_and_version", |
||||||
|
return_value=(123, 1), |
||||||
|
) |
||||||
|
def test_property_task_params(mock_code_version, attr, expect): |
||||||
|
"""Test task http property.""" |
||||||
|
task = Http("test-http-task-params", **attr) |
||||||
|
assert expect == task.task_params |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
"param", |
||||||
|
[ |
||||||
|
{"http_method": "http_method"}, |
||||||
|
{"http_check_condition": "http_check_condition"}, |
||||||
|
{"http_check_condition": HttpCheckCondition.STATUS_CODE_CUSTOM}, |
||||||
|
{ |
||||||
|
"http_check_condition": HttpCheckCondition.STATUS_CODE_CUSTOM, |
||||||
|
"condition": None, |
||||||
|
}, |
||||||
|
], |
||||||
|
) |
||||||
|
@patch( |
||||||
|
"pydolphinscheduler.core.task.Task.gen_code_and_version", |
||||||
|
return_value=(123, 1), |
||||||
|
) |
||||||
|
def test_http_task_param_not_support_param(mock_code, param): |
||||||
|
"""Test HttpTaskParams not support parameter.""" |
||||||
|
url = "https://www.apache.org" |
||||||
|
with pytest.raises(PyDSParamException, match="Parameter .*?"): |
||||||
|
Http("test-no-supprot-param", url, **param) |
||||||
|
|
||||||
|
|
||||||
|
def test_http_get_define(): |
||||||
|
"""Test task HTTP function get_define.""" |
||||||
|
code = 123 |
||||||
|
version = 1 |
||||||
|
name = "test_http_get_define" |
||||||
|
url = "https://www.apache.org" |
||||||
|
expect = { |
||||||
|
"code": code, |
||||||
|
"name": name, |
||||||
|
"version": 1, |
||||||
|
"description": None, |
||||||
|
"delayTime": 0, |
||||||
|
"taskType": "HTTP", |
||||||
|
"taskParams": { |
||||||
|
"localParams": [], |
||||||
|
"httpParams": [], |
||||||
|
"url": url, |
||||||
|
"httpMethod": "GET", |
||||||
|
"httpCheckCondition": "STATUS_CODE_DEFAULT", |
||||||
|
"condition": None, |
||||||
|
"connectTimeout": 60000, |
||||||
|
"socketTimeout": 60000, |
||||||
|
"dependence": {}, |
||||||
|
"resourceList": [], |
||||||
|
"conditionResult": {"successNode": [""], "failedNode": [""]}, |
||||||
|
"waitStartTimeout": {}, |
||||||
|
}, |
||||||
|
"flag": "YES", |
||||||
|
"taskPriority": "MEDIUM", |
||||||
|
"workerGroup": "default", |
||||||
|
"failRetryTimes": 0, |
||||||
|
"failRetryInterval": 1, |
||||||
|
"timeoutFlag": "CLOSE", |
||||||
|
"timeoutNotifyStrategy": None, |
||||||
|
"timeout": 0, |
||||||
|
} |
||||||
|
with patch( |
||||||
|
"pydolphinscheduler.core.task.Task.gen_code_and_version", |
||||||
|
return_value=(code, version), |
||||||
|
): |
||||||
|
http = Http(name, url) |
||||||
|
assert http.get_define() == expect |
@ -0,0 +1,106 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""Test Task Procedure.""" |
||||||
|
|
||||||
|
from unittest.mock import patch |
||||||
|
|
||||||
|
import pytest |
||||||
|
|
||||||
|
from pydolphinscheduler.tasks.procedure import Procedure |
||||||
|
|
||||||
|
TEST_PROCEDURE_SQL = ( |
||||||
|
'create procedure HelloWorld() selece "hello world"; call HelloWorld();' |
||||||
|
) |
||||||
|
TEST_PROCEDURE_DATASOURCE_NAME = "test_datasource" |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
"attr, expect", |
||||||
|
[ |
||||||
|
( |
||||||
|
{ |
||||||
|
"name": "test-procedure-task-params", |
||||||
|
"datasource_name": TEST_PROCEDURE_DATASOURCE_NAME, |
||||||
|
"method": TEST_PROCEDURE_SQL, |
||||||
|
}, |
||||||
|
{ |
||||||
|
"method": TEST_PROCEDURE_SQL, |
||||||
|
"type": "MYSQL", |
||||||
|
"datasource": 1, |
||||||
|
"localParams": [], |
||||||
|
"resourceList": [], |
||||||
|
"dependence": {}, |
||||||
|
"waitStartTimeout": {}, |
||||||
|
"conditionResult": {"successNode": [""], "failedNode": [""]}, |
||||||
|
}, |
||||||
|
) |
||||||
|
], |
||||||
|
) |
||||||
|
@patch( |
||||||
|
"pydolphinscheduler.core.task.Task.gen_code_and_version", |
||||||
|
return_value=(123, 1), |
||||||
|
) |
||||||
|
@patch( |
||||||
|
"pydolphinscheduler.core.database.Database.get_database_info", |
||||||
|
return_value=({"id": 1, "type": "MYSQL"}), |
||||||
|
) |
||||||
|
def test_property_task_params(mock_datasource, mock_code_version, attr, expect): |
||||||
|
"""Test task sql task property.""" |
||||||
|
task = Procedure(**attr) |
||||||
|
assert expect == task.task_params |
||||||
|
|
||||||
|
|
||||||
|
@patch( |
||||||
|
"pydolphinscheduler.core.task.Task.gen_code_and_version", |
||||||
|
return_value=(123, 1), |
||||||
|
) |
||||||
|
@patch( |
||||||
|
"pydolphinscheduler.core.database.Database.get_database_info", |
||||||
|
return_value=({"id": 1, "type": "MYSQL"}), |
||||||
|
) |
||||||
|
def test_sql_get_define(mock_datasource, mock_code_version): |
||||||
|
"""Test task procedure function get_define.""" |
||||||
|
name = "test_procedure_get_define" |
||||||
|
expect = { |
||||||
|
"code": 123, |
||||||
|
"name": name, |
||||||
|
"version": 1, |
||||||
|
"description": None, |
||||||
|
"delayTime": 0, |
||||||
|
"taskType": "PROCEDURE", |
||||||
|
"taskParams": { |
||||||
|
"type": "MYSQL", |
||||||
|
"datasource": 1, |
||||||
|
"method": TEST_PROCEDURE_SQL, |
||||||
|
"localParams": [], |
||||||
|
"resourceList": [], |
||||||
|
"dependence": {}, |
||||||
|
"conditionResult": {"successNode": [""], "failedNode": [""]}, |
||||||
|
"waitStartTimeout": {}, |
||||||
|
}, |
||||||
|
"flag": "YES", |
||||||
|
"taskPriority": "MEDIUM", |
||||||
|
"workerGroup": "default", |
||||||
|
"failRetryTimes": 0, |
||||||
|
"failRetryInterval": 1, |
||||||
|
"timeoutFlag": "CLOSE", |
||||||
|
"timeoutNotifyStrategy": None, |
||||||
|
"timeout": 0, |
||||||
|
} |
||||||
|
task = Procedure(name, TEST_PROCEDURE_DATASOURCE_NAME, TEST_PROCEDURE_SQL) |
||||||
|
assert task.get_define() == expect |
@ -0,0 +1,122 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""Test Task python.""" |
||||||
|
|
||||||
|
|
||||||
|
from unittest.mock import patch |
||||||
|
|
||||||
|
import pytest |
||||||
|
|
||||||
|
from pydolphinscheduler.exceptions import PyDSParamException |
||||||
|
from pydolphinscheduler.tasks.python import Python |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
"attr, expect", |
||||||
|
[ |
||||||
|
( |
||||||
|
{"code": "print(1)"}, |
||||||
|
{ |
||||||
|
"rawScript": "print(1)", |
||||||
|
"localParams": [], |
||||||
|
"resourceList": [], |
||||||
|
"dependence": {}, |
||||||
|
"waitStartTimeout": {}, |
||||||
|
"conditionResult": {"successNode": [""], "failedNode": [""]}, |
||||||
|
}, |
||||||
|
) |
||||||
|
], |
||||||
|
) |
||||||
|
@patch( |
||||||
|
"pydolphinscheduler.core.task.Task.gen_code_and_version", |
||||||
|
return_value=(123, 1), |
||||||
|
) |
||||||
|
def test_property_task_params(mock_code_version, attr, expect): |
||||||
|
"""Test task python property.""" |
||||||
|
task = Python("test-python-task-params", **attr) |
||||||
|
assert expect == task.task_params |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
"script_code", |
||||||
|
[ |
||||||
|
123, |
||||||
|
("print", "hello world"), |
||||||
|
], |
||||||
|
) |
||||||
|
@patch( |
||||||
|
"pydolphinscheduler.core.task.Task.gen_code_and_version", |
||||||
|
return_value=(123, 1), |
||||||
|
) |
||||||
|
def test_python_task_not_support_code(mock_code, script_code): |
||||||
|
"""Test python task parameters.""" |
||||||
|
name = "not_support_code_type" |
||||||
|
with pytest.raises(PyDSParamException, match="Parameter code do not support .*?"): |
||||||
|
task = Python(name, script_code) |
||||||
|
task.raw_script |
||||||
|
|
||||||
|
|
||||||
|
def foo(): # noqa: D103 |
||||||
|
print("hello world.") |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
"name, script_code, raw", |
||||||
|
[ |
||||||
|
("string_define", 'print("hello world.")', 'print("hello world.")'), |
||||||
|
( |
||||||
|
"function_define", |
||||||
|
foo, |
||||||
|
'def foo(): # noqa: D103\n print("hello world.")\n', |
||||||
|
), |
||||||
|
], |
||||||
|
) |
||||||
|
def test_python_get_define(name, script_code, raw): |
||||||
|
"""Test task python function get_define.""" |
||||||
|
code = 123 |
||||||
|
version = 1 |
||||||
|
expect = { |
||||||
|
"code": code, |
||||||
|
"name": name, |
||||||
|
"version": 1, |
||||||
|
"description": None, |
||||||
|
"delayTime": 0, |
||||||
|
"taskType": "PYTHON", |
||||||
|
"taskParams": { |
||||||
|
"resourceList": [], |
||||||
|
"localParams": [], |
||||||
|
"rawScript": raw, |
||||||
|
"dependence": {}, |
||||||
|
"conditionResult": {"successNode": [""], "failedNode": [""]}, |
||||||
|
"waitStartTimeout": {}, |
||||||
|
}, |
||||||
|
"flag": "YES", |
||||||
|
"taskPriority": "MEDIUM", |
||||||
|
"workerGroup": "default", |
||||||
|
"failRetryTimes": 0, |
||||||
|
"failRetryInterval": 1, |
||||||
|
"timeoutFlag": "CLOSE", |
||||||
|
"timeoutNotifyStrategy": None, |
||||||
|
"timeout": 0, |
||||||
|
} |
||||||
|
with patch( |
||||||
|
"pydolphinscheduler.core.task.Task.gen_code_and_version", |
||||||
|
return_value=(code, version), |
||||||
|
): |
||||||
|
shell = Python(name, script_code) |
||||||
|
assert shell.get_define() == expect |
@ -0,0 +1,89 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""Test Task shell.""" |
||||||
|
|
||||||
|
|
||||||
|
from unittest.mock import patch |
||||||
|
|
||||||
|
import pytest |
||||||
|
|
||||||
|
from pydolphinscheduler.tasks.shell import Shell |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
"attr, expect", |
||||||
|
[ |
||||||
|
( |
||||||
|
{"command": "test script"}, |
||||||
|
{ |
||||||
|
"rawScript": "test script", |
||||||
|
"localParams": [], |
||||||
|
"resourceList": [], |
||||||
|
"dependence": {}, |
||||||
|
"waitStartTimeout": {}, |
||||||
|
"conditionResult": {"successNode": [""], "failedNode": [""]}, |
||||||
|
}, |
||||||
|
) |
||||||
|
], |
||||||
|
) |
||||||
|
@patch( |
||||||
|
"pydolphinscheduler.core.task.Task.gen_code_and_version", |
||||||
|
return_value=(123, 1), |
||||||
|
) |
||||||
|
def test_property_task_params(mock_code_version, attr, expect): |
||||||
|
"""Test task shell task property.""" |
||||||
|
task = Shell("test-shell-task-params", **attr) |
||||||
|
assert expect == task.task_params |
||||||
|
|
||||||
|
|
||||||
|
def test_shell_get_define(): |
||||||
|
"""Test task shell function get_define.""" |
||||||
|
code = 123 |
||||||
|
version = 1 |
||||||
|
name = "test_shell_get_define" |
||||||
|
command = "echo test shell" |
||||||
|
expect = { |
||||||
|
"code": code, |
||||||
|
"name": name, |
||||||
|
"version": 1, |
||||||
|
"description": None, |
||||||
|
"delayTime": 0, |
||||||
|
"taskType": "SHELL", |
||||||
|
"taskParams": { |
||||||
|
"resourceList": [], |
||||||
|
"localParams": [], |
||||||
|
"rawScript": command, |
||||||
|
"dependence": {}, |
||||||
|
"conditionResult": {"successNode": [""], "failedNode": [""]}, |
||||||
|
"waitStartTimeout": {}, |
||||||
|
}, |
||||||
|
"flag": "YES", |
||||||
|
"taskPriority": "MEDIUM", |
||||||
|
"workerGroup": "default", |
||||||
|
"failRetryTimes": 0, |
||||||
|
"failRetryInterval": 1, |
||||||
|
"timeoutFlag": "CLOSE", |
||||||
|
"timeoutNotifyStrategy": None, |
||||||
|
"timeout": 0, |
||||||
|
} |
||||||
|
with patch( |
||||||
|
"pydolphinscheduler.core.task.Task.gen_code_and_version", |
||||||
|
return_value=(code, version), |
||||||
|
): |
||||||
|
shell = Shell(name, command) |
||||||
|
assert shell.get_define() == expect |
@ -0,0 +1,149 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""Test Task Sql.""" |
||||||
|
|
||||||
|
|
||||||
|
from unittest.mock import patch |
||||||
|
|
||||||
|
import pytest |
||||||
|
|
||||||
|
from pydolphinscheduler.tasks.sql import Sql, SqlType |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
"sql, sql_type", |
||||||
|
[ |
||||||
|
("select 1", SqlType.SELECT), |
||||||
|
(" select 1", SqlType.SELECT), |
||||||
|
(" select 1 ", SqlType.SELECT), |
||||||
|
(" select 'insert' ", SqlType.SELECT), |
||||||
|
(" select 'insert ' ", SqlType.SELECT), |
||||||
|
("with tmp as (select 1) select * from tmp ", SqlType.SELECT), |
||||||
|
("insert into table_name(col1, col2) value (val1, val2)", SqlType.NOT_SELECT), |
||||||
|
( |
||||||
|
"insert into table_name(select, col2) value ('select', val2)", |
||||||
|
SqlType.NOT_SELECT, |
||||||
|
), |
||||||
|
("update table_name SET col1=val1 where col1=val2", SqlType.NOT_SELECT), |
||||||
|
("update table_name SET col1='select' where col1=val2", SqlType.NOT_SELECT), |
||||||
|
("delete from table_name where id < 10", SqlType.NOT_SELECT), |
||||||
|
("delete from table_name where id < 10", SqlType.NOT_SELECT), |
||||||
|
("alter table table_name add column col1 int", SqlType.NOT_SELECT), |
||||||
|
], |
||||||
|
) |
||||||
|
@patch( |
||||||
|
"pydolphinscheduler.core.task.Task.gen_code_and_version", |
||||||
|
return_value=(123, 1), |
||||||
|
) |
||||||
|
@patch( |
||||||
|
"pydolphinscheduler.core.database.Database.get_database_info", |
||||||
|
return_value=({"id": 1, "type": "mock_type"}), |
||||||
|
) |
||||||
|
def test_get_sql_type(mock_datasource, mock_code_version, sql, sql_type): |
||||||
|
"""Test property sql_type could return correct type.""" |
||||||
|
name = "test_get_sql_type" |
||||||
|
datasource_name = "test_datasource" |
||||||
|
task = Sql(name, datasource_name, sql) |
||||||
|
assert ( |
||||||
|
sql_type == task.sql_type |
||||||
|
), f"Sql {sql} expect sql type is {sql_type} but got {task.sql_type}" |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
"attr, expect", |
||||||
|
[ |
||||||
|
( |
||||||
|
{"datasource_name": "datasource_name", "sql": "select 1"}, |
||||||
|
{ |
||||||
|
"sql": "select 1", |
||||||
|
"type": "MYSQL", |
||||||
|
"datasource": 1, |
||||||
|
"sqlType": SqlType.SELECT, |
||||||
|
"preStatements": [], |
||||||
|
"postStatements": [], |
||||||
|
"displayRows": 10, |
||||||
|
"localParams": [], |
||||||
|
"resourceList": [], |
||||||
|
"dependence": {}, |
||||||
|
"waitStartTimeout": {}, |
||||||
|
"conditionResult": {"successNode": [""], "failedNode": [""]}, |
||||||
|
}, |
||||||
|
) |
||||||
|
], |
||||||
|
) |
||||||
|
@patch( |
||||||
|
"pydolphinscheduler.core.task.Task.gen_code_and_version", |
||||||
|
return_value=(123, 1), |
||||||
|
) |
||||||
|
@patch( |
||||||
|
"pydolphinscheduler.core.database.Database.get_database_info", |
||||||
|
return_value=({"id": 1, "type": "MYSQL"}), |
||||||
|
) |
||||||
|
def test_property_task_params(mock_datasource, mock_code_version, attr, expect): |
||||||
|
"""Test task sql task property.""" |
||||||
|
task = Sql("test-sql-task-params", **attr) |
||||||
|
assert expect == task.task_params |
||||||
|
|
||||||
|
|
||||||
|
@patch( |
||||||
|
"pydolphinscheduler.core.database.Database.get_database_info", |
||||||
|
return_value=({"id": 1, "type": "MYSQL"}), |
||||||
|
) |
||||||
|
def test_sql_get_define(mock_datasource): |
||||||
|
"""Test task sql function get_define.""" |
||||||
|
code = 123 |
||||||
|
version = 1 |
||||||
|
name = "test_sql_get_define" |
||||||
|
command = "select 1" |
||||||
|
datasource_name = "test_datasource" |
||||||
|
expect = { |
||||||
|
"code": code, |
||||||
|
"name": name, |
||||||
|
"version": 1, |
||||||
|
"description": None, |
||||||
|
"delayTime": 0, |
||||||
|
"taskType": "SQL", |
||||||
|
"taskParams": { |
||||||
|
"type": "MYSQL", |
||||||
|
"datasource": 1, |
||||||
|
"sql": command, |
||||||
|
"sqlType": SqlType.SELECT, |
||||||
|
"displayRows": 10, |
||||||
|
"preStatements": [], |
||||||
|
"postStatements": [], |
||||||
|
"localParams": [], |
||||||
|
"resourceList": [], |
||||||
|
"dependence": {}, |
||||||
|
"conditionResult": {"successNode": [""], "failedNode": [""]}, |
||||||
|
"waitStartTimeout": {}, |
||||||
|
}, |
||||||
|
"flag": "YES", |
||||||
|
"taskPriority": "MEDIUM", |
||||||
|
"workerGroup": "default", |
||||||
|
"failRetryTimes": 0, |
||||||
|
"failRetryInterval": 1, |
||||||
|
"timeoutFlag": "CLOSE", |
||||||
|
"timeoutNotifyStrategy": None, |
||||||
|
"timeout": 0, |
||||||
|
} |
||||||
|
with patch( |
||||||
|
"pydolphinscheduler.core.task.Task.gen_code_and_version", |
||||||
|
return_value=(code, version), |
||||||
|
): |
||||||
|
task = Sql(name, datasource_name, command) |
||||||
|
assert task.get_define() == expect |
@ -0,0 +1,114 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""Test Task sub_process.""" |
||||||
|
|
||||||
|
|
||||||
|
from unittest.mock import patch |
||||||
|
|
||||||
|
import pytest |
||||||
|
|
||||||
|
from pydolphinscheduler.core.process_definition import ProcessDefinition |
||||||
|
from pydolphinscheduler.tasks.sub_process import SubProcess |
||||||
|
|
||||||
|
TEST_SUB_PROCESS_DEFINITION_NAME = "sub-test-process-definition" |
||||||
|
TEST_SUB_PROCESS_DEFINITION_CODE = "3643589832320" |
||||||
|
TEST_PROCESS_DEFINITION_NAME = "simple-test-process-definition" |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
"attr, expect", |
||||||
|
[ |
||||||
|
( |
||||||
|
{"process_definition_name": TEST_SUB_PROCESS_DEFINITION_NAME}, |
||||||
|
{ |
||||||
|
"processDefinitionCode": TEST_SUB_PROCESS_DEFINITION_CODE, |
||||||
|
"localParams": [], |
||||||
|
"resourceList": [], |
||||||
|
"dependence": {}, |
||||||
|
"waitStartTimeout": {}, |
||||||
|
"conditionResult": {"successNode": [""], "failedNode": [""]}, |
||||||
|
}, |
||||||
|
) |
||||||
|
], |
||||||
|
) |
||||||
|
@patch( |
||||||
|
"pydolphinscheduler.tasks.sub_process.SubProcess.get_process_definition_info", |
||||||
|
return_value=( |
||||||
|
{ |
||||||
|
"id": 1, |
||||||
|
"name": TEST_SUB_PROCESS_DEFINITION_NAME, |
||||||
|
"code": TEST_SUB_PROCESS_DEFINITION_CODE, |
||||||
|
} |
||||||
|
), |
||||||
|
) |
||||||
|
@patch( |
||||||
|
"pydolphinscheduler.core.task.Task.gen_code_and_version", |
||||||
|
return_value=(123, 1), |
||||||
|
) |
||||||
|
def test_property_task_params(mock_code_version, mock_pd_info, attr, expect): |
||||||
|
"""Test task sub process property.""" |
||||||
|
task = SubProcess("test-sub-process-task-params", **attr) |
||||||
|
assert expect == task.task_params |
||||||
|
|
||||||
|
|
||||||
|
@patch( |
||||||
|
"pydolphinscheduler.tasks.sub_process.SubProcess.get_process_definition_info", |
||||||
|
return_value=( |
||||||
|
{ |
||||||
|
"id": 1, |
||||||
|
"name": TEST_SUB_PROCESS_DEFINITION_NAME, |
||||||
|
"code": TEST_SUB_PROCESS_DEFINITION_CODE, |
||||||
|
} |
||||||
|
), |
||||||
|
) |
||||||
|
def test_sub_process_get_define(mock_process_definition): |
||||||
|
"""Test task sub_process function get_define.""" |
||||||
|
code = 123 |
||||||
|
version = 1 |
||||||
|
name = "test_sub_process_get_define" |
||||||
|
expect = { |
||||||
|
"code": code, |
||||||
|
"name": name, |
||||||
|
"version": 1, |
||||||
|
"description": None, |
||||||
|
"delayTime": 0, |
||||||
|
"taskType": "SUB_PROCESS", |
||||||
|
"taskParams": { |
||||||
|
"resourceList": [], |
||||||
|
"localParams": [], |
||||||
|
"processDefinitionCode": TEST_SUB_PROCESS_DEFINITION_CODE, |
||||||
|
"dependence": {}, |
||||||
|
"conditionResult": {"successNode": [""], "failedNode": [""]}, |
||||||
|
"waitStartTimeout": {}, |
||||||
|
}, |
||||||
|
"flag": "YES", |
||||||
|
"taskPriority": "MEDIUM", |
||||||
|
"workerGroup": "default", |
||||||
|
"failRetryTimes": 0, |
||||||
|
"failRetryInterval": 1, |
||||||
|
"timeoutFlag": "CLOSE", |
||||||
|
"timeoutNotifyStrategy": None, |
||||||
|
"timeout": 0, |
||||||
|
} |
||||||
|
with patch( |
||||||
|
"pydolphinscheduler.core.task.Task.gen_code_and_version", |
||||||
|
return_value=(code, version), |
||||||
|
): |
||||||
|
with ProcessDefinition(TEST_PROCESS_DEFINITION_NAME): |
||||||
|
sub_process = SubProcess(name, TEST_SUB_PROCESS_DEFINITION_NAME) |
||||||
|
assert sub_process.get_define() == expect |
@ -0,0 +1,300 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""Test Task switch.""" |
||||||
|
|
||||||
|
from typing import Optional, Tuple |
||||||
|
from unittest.mock import patch |
||||||
|
|
||||||
|
import pytest |
||||||
|
|
||||||
|
from pydolphinscheduler.core.process_definition import ProcessDefinition |
||||||
|
from pydolphinscheduler.exceptions import PyDSParamException |
||||||
|
from pydolphinscheduler.tasks.switch import ( |
||||||
|
Branch, |
||||||
|
Default, |
||||||
|
Switch, |
||||||
|
SwitchBranch, |
||||||
|
SwitchCondition, |
||||||
|
) |
||||||
|
from tests.testing.task import Task |
||||||
|
|
||||||
|
TEST_NAME = "test-task" |
||||||
|
TEST_TYPE = "test-type" |
||||||
|
|
||||||
|
|
||||||
|
def task_switch_arg_wrapper(obj, task: Task, exp: Optional[str] = None) -> SwitchBranch: |
||||||
|
"""Wrap task switch and its subclass.""" |
||||||
|
if obj is Default: |
||||||
|
return obj(task) |
||||||
|
elif obj is Branch: |
||||||
|
return obj(exp, task) |
||||||
|
else: |
||||||
|
return obj(task, exp) |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
"obj", |
||||||
|
[ |
||||||
|
SwitchBranch, |
||||||
|
Branch, |
||||||
|
Default, |
||||||
|
], |
||||||
|
) |
||||||
|
def test_switch_branch_attr_next_node(obj: SwitchBranch): |
||||||
|
"""Test get attribute from class switch branch.""" |
||||||
|
task = Task(name=TEST_NAME, task_type=TEST_TYPE) |
||||||
|
switch_branch = task_switch_arg_wrapper(obj, task=task, exp="unittest") |
||||||
|
assert switch_branch.next_node == task.code |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
"obj", |
||||||
|
[ |
||||||
|
SwitchBranch, |
||||||
|
Default, |
||||||
|
], |
||||||
|
) |
||||||
|
def test_switch_branch_get_define_without_condition(obj: SwitchBranch): |
||||||
|
"""Test function :func:`get_define` with None value of attribute condition from class switch branch.""" |
||||||
|
task = Task(name=TEST_NAME, task_type=TEST_TYPE) |
||||||
|
expect = {"nextNode": task.code} |
||||||
|
switch_branch = task_switch_arg_wrapper(obj, task=task) |
||||||
|
assert switch_branch.get_define() == expect |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
"obj", |
||||||
|
[ |
||||||
|
SwitchBranch, |
||||||
|
Branch, |
||||||
|
], |
||||||
|
) |
||||||
|
def test_switch_branch_get_define_condition(obj: SwitchBranch): |
||||||
|
"""Test function :func:`get_define` with specific attribute condition from class switch branch.""" |
||||||
|
task = Task(name=TEST_NAME, task_type=TEST_TYPE) |
||||||
|
exp = "${var} == 1" |
||||||
|
expect = { |
||||||
|
"nextNode": task.code, |
||||||
|
"condition": exp, |
||||||
|
} |
||||||
|
switch_branch = task_switch_arg_wrapper(obj, task=task, exp=exp) |
||||||
|
assert switch_branch.get_define() == expect |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
"args, msg", |
||||||
|
[ |
||||||
|
( |
||||||
|
(1,), |
||||||
|
".*?parameter only support SwitchBranch but got.*?", |
||||||
|
), |
||||||
|
( |
||||||
|
(Default(Task(TEST_NAME, TEST_TYPE)), 2), |
||||||
|
".*?parameter only support SwitchBranch but got.*?", |
||||||
|
), |
||||||
|
( |
||||||
|
(Default(Task(TEST_NAME, TEST_TYPE)), Default(Task(TEST_NAME, TEST_TYPE))), |
||||||
|
".*?parameter only support exactly one default branch", |
||||||
|
), |
||||||
|
( |
||||||
|
( |
||||||
|
Branch(condition="unittest", task=Task(TEST_NAME, TEST_TYPE)), |
||||||
|
Default(Task(TEST_NAME, TEST_TYPE)), |
||||||
|
Default(Task(TEST_NAME, TEST_TYPE)), |
||||||
|
), |
||||||
|
".*?parameter only support exactly one default branch", |
||||||
|
), |
||||||
|
], |
||||||
|
) |
||||||
|
def test_switch_condition_set_define_attr_error(args: Tuple, msg: str): |
||||||
|
"""Test error case on :class:`SwitchCondition`.""" |
||||||
|
switch_condition = SwitchCondition(*args) |
||||||
|
with pytest.raises(PyDSParamException, match=msg): |
||||||
|
switch_condition.set_define_attr() |
||||||
|
|
||||||
|
|
||||||
|
def test_switch_condition_set_define_attr_default(): |
||||||
|
"""Test set :class:`Default` to attribute on :class:`SwitchCondition`.""" |
||||||
|
task = Task(TEST_NAME, TEST_TYPE) |
||||||
|
switch_condition = SwitchCondition(Default(task)) |
||||||
|
switch_condition.set_define_attr() |
||||||
|
assert getattr(switch_condition, "next_node") == task.code |
||||||
|
assert getattr(switch_condition, "depend_task_list") == [] |
||||||
|
|
||||||
|
|
||||||
|
def test_switch_condition_set_define_attr_branch(): |
||||||
|
"""Test set :class:`Branch` to attribute on :class:`SwitchCondition`.""" |
||||||
|
task = Task(TEST_NAME, TEST_TYPE) |
||||||
|
switch_condition = SwitchCondition( |
||||||
|
Branch("unittest1", task), Branch("unittest2", task) |
||||||
|
) |
||||||
|
expect = [ |
||||||
|
{"condition": "unittest1", "nextNode": task.code}, |
||||||
|
{"condition": "unittest2", "nextNode": task.code}, |
||||||
|
] |
||||||
|
|
||||||
|
switch_condition.set_define_attr() |
||||||
|
assert getattr(switch_condition, "next_node") == "" |
||||||
|
assert getattr(switch_condition, "depend_task_list") == expect |
||||||
|
|
||||||
|
|
||||||
|
def test_switch_condition_set_define_attr_mix_branch_and_default(): |
||||||
|
"""Test set bot :class:`Branch` and :class:`Default` to attribute on :class:`SwitchCondition`.""" |
||||||
|
task = Task(TEST_NAME, TEST_TYPE) |
||||||
|
switch_condition = SwitchCondition( |
||||||
|
Branch("unittest1", task), Branch("unittest2", task), Default(task) |
||||||
|
) |
||||||
|
expect = [ |
||||||
|
{"condition": "unittest1", "nextNode": task.code}, |
||||||
|
{"condition": "unittest2", "nextNode": task.code}, |
||||||
|
] |
||||||
|
|
||||||
|
switch_condition.set_define_attr() |
||||||
|
assert getattr(switch_condition, "next_node") == task.code |
||||||
|
assert getattr(switch_condition, "depend_task_list") == expect |
||||||
|
|
||||||
|
|
||||||
|
def test_switch_condition_get_define_default(): |
||||||
|
"""Test function :func:`get_define` with :class:`Default` in :class:`SwitchCondition`.""" |
||||||
|
task = Task(TEST_NAME, TEST_TYPE) |
||||||
|
switch_condition = SwitchCondition(Default(task)) |
||||||
|
expect = { |
||||||
|
"dependTaskList": [], |
||||||
|
"nextNode": task.code, |
||||||
|
} |
||||||
|
assert switch_condition.get_define() == expect |
||||||
|
|
||||||
|
|
||||||
|
def test_switch_condition_get_define_branch(): |
||||||
|
"""Test function :func:`get_define` with :class:`Branch` in :class:`SwitchCondition`.""" |
||||||
|
task = Task(TEST_NAME, TEST_TYPE) |
||||||
|
switch_condition = SwitchCondition( |
||||||
|
Branch("unittest1", task), Branch("unittest2", task) |
||||||
|
) |
||||||
|
expect = { |
||||||
|
"dependTaskList": [ |
||||||
|
{"condition": "unittest1", "nextNode": task.code}, |
||||||
|
{"condition": "unittest2", "nextNode": task.code}, |
||||||
|
], |
||||||
|
"nextNode": "", |
||||||
|
} |
||||||
|
assert switch_condition.get_define() == expect |
||||||
|
|
||||||
|
|
||||||
|
def test_switch_condition_get_define_mix_branch_and_default(): |
||||||
|
"""Test function :func:`get_define` with both :class:`Branch` and :class:`Default`.""" |
||||||
|
task = Task(TEST_NAME, TEST_TYPE) |
||||||
|
switch_condition = SwitchCondition( |
||||||
|
Branch("unittest1", task), Branch("unittest2", task), Default(task) |
||||||
|
) |
||||||
|
expect = { |
||||||
|
"dependTaskList": [ |
||||||
|
{"condition": "unittest1", "nextNode": task.code}, |
||||||
|
{"condition": "unittest2", "nextNode": task.code}, |
||||||
|
], |
||||||
|
"nextNode": task.code, |
||||||
|
} |
||||||
|
assert switch_condition.get_define() == expect |
||||||
|
|
||||||
|
|
||||||
|
@patch( |
||||||
|
"pydolphinscheduler.core.task.Task.gen_code_and_version", |
||||||
|
return_value=(123, 1), |
||||||
|
) |
||||||
|
def test_switch_get_define(mock_task_code_version): |
||||||
|
"""Test task switch :func:`get_define`.""" |
||||||
|
task = Task(name=TEST_NAME, task_type=TEST_TYPE) |
||||||
|
switch_condition = SwitchCondition( |
||||||
|
Branch(condition="${var1} > 1", task=task), |
||||||
|
Branch(condition="${var1} <= 1", task=task), |
||||||
|
Default(task), |
||||||
|
) |
||||||
|
|
||||||
|
name = "test_switch_get_define" |
||||||
|
expect = { |
||||||
|
"code": 123, |
||||||
|
"name": name, |
||||||
|
"version": 1, |
||||||
|
"description": None, |
||||||
|
"delayTime": 0, |
||||||
|
"taskType": "SWITCH", |
||||||
|
"taskParams": { |
||||||
|
"resourceList": [], |
||||||
|
"localParams": [], |
||||||
|
"dependence": {}, |
||||||
|
"conditionResult": {"successNode": [""], "failedNode": [""]}, |
||||||
|
"waitStartTimeout": {}, |
||||||
|
"switchResult": { |
||||||
|
"dependTaskList": [ |
||||||
|
{"condition": "${var1} > 1", "nextNode": task.code}, |
||||||
|
{"condition": "${var1} <= 1", "nextNode": task.code}, |
||||||
|
], |
||||||
|
"nextNode": task.code, |
||||||
|
}, |
||||||
|
}, |
||||||
|
"flag": "YES", |
||||||
|
"taskPriority": "MEDIUM", |
||||||
|
"workerGroup": "default", |
||||||
|
"failRetryTimes": 0, |
||||||
|
"failRetryInterval": 1, |
||||||
|
"timeoutFlag": "CLOSE", |
||||||
|
"timeoutNotifyStrategy": None, |
||||||
|
"timeout": 0, |
||||||
|
} |
||||||
|
|
||||||
|
task = Switch(name, condition=switch_condition) |
||||||
|
assert task.get_define() == expect |
||||||
|
|
||||||
|
|
||||||
|
@patch( |
||||||
|
"pydolphinscheduler.core.task.Task.gen_code_and_version", |
||||||
|
return_value=(123, 1), |
||||||
|
) |
||||||
|
def test_switch_set_dep_workflow(mock_task_code_version): |
||||||
|
"""Test task switch set dependence in workflow level.""" |
||||||
|
with ProcessDefinition(name="test-switch-set-dep-workflow") as pd: |
||||||
|
parent = Task(name="parent", task_type=TEST_TYPE) |
||||||
|
switch_child_1 = Task(name="switch_child_1", task_type=TEST_TYPE) |
||||||
|
switch_child_2 = Task(name="switch_child_2", task_type=TEST_TYPE) |
||||||
|
switch_condition = SwitchCondition( |
||||||
|
Branch(condition="${var} > 1", task=switch_child_1), |
||||||
|
Default(task=switch_child_2), |
||||||
|
) |
||||||
|
|
||||||
|
switch = Switch(name=TEST_NAME, condition=switch_condition) |
||||||
|
parent >> switch |
||||||
|
# General tasks test |
||||||
|
assert len(pd.tasks) == 4 |
||||||
|
assert sorted(pd.task_list, key=lambda t: t.name) == sorted( |
||||||
|
[parent, switch, switch_child_1, switch_child_2], key=lambda t: t.name |
||||||
|
) |
||||||
|
# Task dep test |
||||||
|
assert parent._downstream_task_codes == {switch.code} |
||||||
|
assert switch._upstream_task_codes == {parent.code} |
||||||
|
|
||||||
|
# Switch task dep after ProcessDefinition function get_define called |
||||||
|
assert switch._downstream_task_codes == { |
||||||
|
switch_child_1.code, |
||||||
|
switch_child_2.code, |
||||||
|
} |
||||||
|
assert all( |
||||||
|
[ |
||||||
|
child._upstream_task_codes == {switch.code} |
||||||
|
for child in [switch_child_1, switch_child_2] |
||||||
|
] |
||||||
|
) |
@ -0,0 +1,52 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""Test pydolphinscheduler java gateway.""" |
||||||
|
|
||||||
|
|
||||||
|
from py4j.java_gateway import JavaGateway, java_import |
||||||
|
|
||||||
|
|
||||||
|
def test_gateway_connect(): |
||||||
|
"""Test weather client could connect java gate way or not.""" |
||||||
|
gateway = JavaGateway() |
||||||
|
app = gateway.entry_point |
||||||
|
assert app.ping() == "PONG" |
||||||
|
|
||||||
|
|
||||||
|
def test_jvm_simple(): |
||||||
|
"""Test use JVM build-in object and operator from java gateway.""" |
||||||
|
gateway = JavaGateway() |
||||||
|
smaller = gateway.jvm.java.lang.Integer.MIN_VALUE |
||||||
|
bigger = gateway.jvm.java.lang.Integer.MAX_VALUE |
||||||
|
assert bigger > smaller |
||||||
|
|
||||||
|
|
||||||
|
def test_python_client_java_import_single(): |
||||||
|
"""Test import single class from java gateway.""" |
||||||
|
gateway = JavaGateway() |
||||||
|
java_import(gateway.jvm, "org.apache.dolphinscheduler.common.utils.FileUtils") |
||||||
|
assert hasattr(gateway.jvm, "FileUtils") |
||||||
|
|
||||||
|
|
||||||
|
def test_python_client_java_import_package(): |
||||||
|
"""Test import package contain multiple class from java gateway.""" |
||||||
|
gateway = JavaGateway() |
||||||
|
java_import(gateway.jvm, "org.apache.dolphinscheduler.common.utils.*") |
||||||
|
# test if jvm view have some common utils |
||||||
|
for util in ("FileUtils", "OSUtils", "DateUtils"): |
||||||
|
assert hasattr(gateway.jvm, util) |
@ -0,0 +1,18 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""Init testing package, it provider easy way for pydolphinscheduler test.""" |
@ -0,0 +1,32 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""Mock class Task for other test.""" |
||||||
|
|
||||||
|
import uuid |
||||||
|
|
||||||
|
from pydolphinscheduler.core.task import Task as SourceTask |
||||||
|
|
||||||
|
|
||||||
|
class Task(SourceTask): |
||||||
|
"""Mock class :class:`pydolphinscheduler.core.task.Task` for unittest.""" |
||||||
|
|
||||||
|
DEFAULT_VERSION = 1 |
||||||
|
|
||||||
|
def gen_code_and_version(self): |
||||||
|
"""Mock java gateway code and version, convenience method for unittest.""" |
||||||
|
return uuid.uuid1().time, self.DEFAULT_VERSION |
@ -0,0 +1,18 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""Init tests for utils package.""" |
@ -0,0 +1,78 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""Test utils.date module.""" |
||||||
|
|
||||||
|
from datetime import datetime |
||||||
|
|
||||||
|
import pytest |
||||||
|
|
||||||
|
from pydolphinscheduler.utils.date import FMT_STD, conv_from_str, conv_to_schedule |
||||||
|
|
||||||
|
curr_date = datetime.now() |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
"src,expect", |
||||||
|
[ |
||||||
|
(curr_date, curr_date.strftime(FMT_STD)), |
||||||
|
(datetime(2021, 1, 1), "2021-01-01 00:00:00"), |
||||||
|
(datetime(2021, 1, 1, 1), "2021-01-01 01:00:00"), |
||||||
|
(datetime(2021, 1, 1, 1, 1), "2021-01-01 01:01:00"), |
||||||
|
(datetime(2021, 1, 1, 1, 1, 1), "2021-01-01 01:01:01"), |
||||||
|
(datetime(2021, 1, 1, 1, 1, 1, 1), "2021-01-01 01:01:01"), |
||||||
|
], |
||||||
|
) |
||||||
|
def test_conv_to_schedule(src: datetime, expect: str) -> None: |
||||||
|
"""Test function conv_to_schedule.""" |
||||||
|
assert expect == conv_to_schedule(src) |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
"src,expect", |
||||||
|
[ |
||||||
|
("2021-01-01", datetime(2021, 1, 1)), |
||||||
|
("2021/01/01", datetime(2021, 1, 1)), |
||||||
|
("20210101", datetime(2021, 1, 1)), |
||||||
|
("2021-01-01 01:01:01", datetime(2021, 1, 1, 1, 1, 1)), |
||||||
|
("2021/01/01 01:01:01", datetime(2021, 1, 1, 1, 1, 1)), |
||||||
|
("20210101 010101", datetime(2021, 1, 1, 1, 1, 1)), |
||||||
|
], |
||||||
|
) |
||||||
|
def test_conv_from_str_success(src: str, expect: datetime) -> None: |
||||||
|
"""Test function conv_from_str success case.""" |
||||||
|
assert expect == conv_from_str( |
||||||
|
src |
||||||
|
), f"Function conv_from_str convert {src} not expect to {expect}." |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
"src", |
||||||
|
[ |
||||||
|
"2021-01-01 010101", |
||||||
|
"2021:01:01", |
||||||
|
"202111", |
||||||
|
"20210101010101", |
||||||
|
"2021:01:01 01:01:01", |
||||||
|
], |
||||||
|
) |
||||||
|
def test_conv_from_str_not_impl(src: str) -> None: |
||||||
|
"""Test function conv_from_str fail case.""" |
||||||
|
with pytest.raises( |
||||||
|
NotImplementedError, match=".*? could not be convert to datetime for now." |
||||||
|
): |
||||||
|
conv_from_str(src) |
@ -0,0 +1,87 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""Test utils.string module.""" |
||||||
|
|
||||||
|
import pytest |
||||||
|
|
||||||
|
from pydolphinscheduler.utils.string import attr2camel, class_name2camel, snake2camel |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
"snake, expect", |
||||||
|
[ |
||||||
|
("snake_case", "snakeCase"), |
||||||
|
("snake_123case", "snake123Case"), |
||||||
|
("snake_c_a_s_e", "snakeCASE"), |
||||||
|
("snake__case", "snakeCase"), |
||||||
|
("snake_case_case", "snakeCaseCase"), |
||||||
|
("_snake_case", "SnakeCase"), |
||||||
|
("__snake_case", "SnakeCase"), |
||||||
|
("Snake_case", "SnakeCase"), |
||||||
|
], |
||||||
|
) |
||||||
|
def test_snake2camel(snake: str, expect: str): |
||||||
|
"""Test function snake2camel, this is a base function for utils.string.""" |
||||||
|
assert expect == snake2camel( |
||||||
|
snake |
||||||
|
), f"Test case {snake} do no return expect result {expect}." |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
"attr, expects", |
||||||
|
[ |
||||||
|
# source attribute, (true expect, false expect), |
||||||
|
("snake_case", ("snakeCase", "snakeCase")), |
||||||
|
("snake_123case", ("snake123Case", "snake123Case")), |
||||||
|
("snake_c_a_s_e", ("snakeCASE", "snakeCASE")), |
||||||
|
("snake__case", ("snakeCase", "snakeCase")), |
||||||
|
("snake_case_case", ("snakeCaseCase", "snakeCaseCase")), |
||||||
|
("_snake_case", ("snakeCase", "SnakeCase")), |
||||||
|
("__snake_case", ("snakeCase", "SnakeCase")), |
||||||
|
("Snake_case", ("SnakeCase", "SnakeCase")), |
||||||
|
], |
||||||
|
) |
||||||
|
def test_attr2camel(attr: str, expects: tuple): |
||||||
|
"""Test function attr2camel.""" |
||||||
|
for idx, expect in enumerate(expects): |
||||||
|
include_private = idx % 2 == 0 |
||||||
|
assert expect == attr2camel( |
||||||
|
attr, include_private |
||||||
|
), f"Test case {attr} do no return expect result {expect} when include_private is {include_private}." |
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize( |
||||||
|
"class_name, expect", |
||||||
|
[ |
||||||
|
("snake_case", "snakeCase"), |
||||||
|
("snake_123case", "snake123Case"), |
||||||
|
("snake_c_a_s_e", "snakeCASE"), |
||||||
|
("snake__case", "snakeCase"), |
||||||
|
("snake_case_case", "snakeCaseCase"), |
||||||
|
("_snake_case", "snakeCase"), |
||||||
|
("_Snake_case", "snakeCase"), |
||||||
|
("__snake_case", "snakeCase"), |
||||||
|
("__Snake_case", "snakeCase"), |
||||||
|
("Snake_case", "snakeCase"), |
||||||
|
], |
||||||
|
) |
||||||
|
def test_class_name2camel(class_name: str, expect: str): |
||||||
|
"""Test function class_name2camel.""" |
||||||
|
assert expect == class_name2camel( |
||||||
|
class_name |
||||||
|
), f"Test case {class_name} do no return expect result {expect}." |
Loading…
Reference in new issue