Browse Source
* Init DS python SDK pydolphinscheduler: python code definition * Doc first * Add quick start and developer doc * Java documentation change * Add LICENSE-py4j.txt * Add py4j to release-docs/LICENSE * Move dependency version to parent pom * Remove outdated code * Add tenant parameter to tutorial3.0.0/version-upgrade
Jiajie Zhong
3 years ago
committed by
GitHub
52 changed files with 2363 additions and 3 deletions
@ -0,0 +1,26 @@
|
||||
Copyright (c) 2009-2018, Barthelemy Dagenais and individual contributors. All |
||||
rights reserved. |
||||
|
||||
Redistribution and use in source and binary forms, with or without |
||||
modification, are permitted provided that the following conditions are met: |
||||
|
||||
- Redistributions of source code must retain the above copyright notice, this |
||||
list of conditions and the following disclaimer. |
||||
|
||||
- Redistributions in binary form must reproduce the above copyright notice, |
||||
this list of conditions and the following disclaimer in the documentation |
||||
and/or other materials provided with the distribution. |
||||
|
||||
- The name of the author may not be used to endorse or promote products |
||||
derived from this software without specific prior written permission. |
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" |
||||
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
||||
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
||||
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE |
||||
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL |
||||
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR |
||||
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER |
||||
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, |
||||
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||||
@ -0,0 +1,60 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?> |
||||
<!-- |
||||
~ Licensed to the Apache Software Foundation (ASF) under one or more |
||||
~ contributor license agreements. See the NOTICE file distributed with |
||||
~ this work for additional information regarding copyright ownership. |
||||
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
~ (the "License"); you may not use this file except in compliance with |
||||
~ the License. You may obtain a copy of the License at |
||||
~ |
||||
~ http://www.apache.org/licenses/LICENSE-2.0 |
||||
~ |
||||
~ Unless required by applicable law or agreed to in writing, software |
||||
~ distributed under the License is distributed on an "AS IS" BASIS, |
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
~ See the License for the specific language governing permissions and |
||||
~ limitations under the License. |
||||
--> |
||||
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" |
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> |
||||
<modelVersion>4.0.0</modelVersion> |
||||
<parent> |
||||
<groupId>org.apache.dolphinscheduler</groupId> |
||||
<artifactId>dolphinscheduler</artifactId> |
||||
<version>2.0.0-SNAPSHOT</version> |
||||
</parent> |
||||
<artifactId>dolphinscheduler-python</artifactId> |
||||
<name>${project.artifactId}</name> |
||||
<packaging>jar</packaging> |
||||
|
||||
<dependencies> |
||||
<!-- dolphinscheduler --> |
||||
<dependency> |
||||
<groupId>org.apache.dolphinscheduler</groupId> |
||||
<artifactId>dolphinscheduler-api</artifactId> |
||||
</dependency> |
||||
|
||||
<!--springboot--> |
||||
<dependency> |
||||
<groupId>org.springframework.boot</groupId> |
||||
<artifactId>spring-boot-starter-web</artifactId> |
||||
<exclusions> |
||||
<exclusion> |
||||
<groupId>org.springframework.boot</groupId> |
||||
<artifactId>spring-boot-starter-tomcat</artifactId> |
||||
</exclusion> |
||||
<exclusion> |
||||
<artifactId>log4j-to-slf4j</artifactId> |
||||
<groupId>org.apache.logging.log4j</groupId> |
||||
</exclusion> |
||||
</exclusions> |
||||
</dependency> |
||||
|
||||
<dependency> |
||||
<groupId>net.sf.py4j</groupId> |
||||
<artifactId>py4j</artifactId> |
||||
</dependency> |
||||
|
||||
</dependencies> |
||||
</project> |
||||
@ -0,0 +1,103 @@
|
||||
<!-- |
||||
Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
Unless required by applicable law or agreed to in writing, |
||||
software distributed under the License is distributed on an |
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
KIND, either express or implied. See the License for the |
||||
specific language governing permissions and limitations |
||||
under the License. |
||||
--> |
||||
|
||||
# pydolphinscheduler |
||||
|
||||
pydolphinscheduler is python API for Apache DolphinScheduler, which allow you definition |
||||
your workflow by python code, aka workflow-as-codes. |
||||
|
||||
## Quick Start |
||||
|
||||
> **_Notice:_** For now, due to pydolphinscheduler without release to any binary tarball or [PyPI][pypi], you |
||||
> have to clone Apache DolphinScheduler code from GitHub to ensure quick start setup |
||||
|
||||
Here we show you how to install and run a simple example of pydolphinscheduler |
||||
|
||||
### Prepare |
||||
|
||||
```shell |
||||
# Clone code from github |
||||
git clone git@github.com:apache/dolphinscheduler.git |
||||
|
||||
# Install pydolphinscheduler from source |
||||
cd dolphinscheduler-python/pydolphinscheduler |
||||
pip setup.py install |
||||
``` |
||||
|
||||
### Start Server And Run Example |
||||
|
||||
Before you run an example, you have to start backend server. You could follow [development setup][dev-setup] |
||||
section "DolphinScheduler Standalone Quick Start" to set up developer environment. You have to start backend |
||||
and frontend server in this step, which mean that you could view DolphinScheduler UI in your browser with URL |
||||
http://localhost:12345/dolphinscheduler |
||||
|
||||
After backend server is being start, all requests from `pydolphinscheduler` would be sends to backend server. |
||||
And for now we could run a simple example by: |
||||
|
||||
```shell |
||||
cd dolphinscheduler-python/pydolphinscheduler |
||||
python example/tutorial.py |
||||
``` |
||||
|
||||
> **_NOTICE:_** Since Apache DolphinScheduler's tenant is requests while running command, you might need to change |
||||
> tenant value in `example/tutorial.py`. For now the value is `tenant_exists`, please change it to username exists |
||||
> in you environment. |
||||
|
||||
After command execute, you could see a new project with single process definition named *tutorial* in the [UI][ui-project]. |
||||
|
||||
Until now, we finish quick start by an example of pydolphinscheduler and run it. If you want to inspect or join |
||||
pydolphinscheduler develop, you could take a look at [develop](#develop) |
||||
|
||||
## Develop |
||||
|
||||
pydolphinscheduler is python API for Apache DolphinScheduler, it just defines what workflow look like instead of |
||||
store or execute it. We here use [py4j][py4j] to dynamically access Java Virtual Machine. |
||||
|
||||
### Setup Develop Environment |
||||
|
||||
We already clone the code in [quick start](#quick-start), so next step we have to open pydolphinscheduler project |
||||
in you editor. We recommend you use [pycharm][pycharm] instead of [IntelliJ IDEA][idea] to open it. And you could |
||||
just open directory `dolphinscheduler-python/pydolphinscheduler` instead of `dolphinscheduler-python`. |
||||
|
||||
### Brief Concept |
||||
|
||||
Apache DolphinScheduler is design to define workflow by UI, and pydolphinscheduler try to define it by code. When |
||||
define by code, user usually do not care user, tanant, or queue exists or not. All user care about is create |
||||
a new workflow by the code his/her definition. So we have some **side object** in `pydolphinscheduler/side` |
||||
directory, their only check object exists or not, and create them if not exists. |
||||
|
||||
#### Process Definition |
||||
|
||||
pydolphinscheduler workflow object name, process definition is also same name as Java object(maybe would be change to |
||||
other word for more simple). |
||||
|
||||
#### Tasks |
||||
|
||||
pydolphinscheduler tasks object, we use tasks to define exact job we want DolphinScheduler do for us. For now, |
||||
we only support `shell` task to execute shell task. [This link][all-task] list all tasks support in DolphinScheduler |
||||
and would be implement in the further. |
||||
|
||||
|
||||
[pypi]: https://pypi.org/ |
||||
[dev-setup]: https://dolphinscheduler.apache.org/en-us/development/development-environment-setup.html |
||||
[ui-project]: http://8.142.34.29:12345/dolphinscheduler/ui/#/projects/list |
||||
[py4j]: https://www.py4j.org/index.html |
||||
[pycharm]: https://www.jetbrains.com/pycharm |
||||
[idea]: https://www.jetbrains.com/idea/ |
||||
[all-task]: https://dolphinscheduler.apache.org/en-us/docs/dev/user_doc/guide/task/shell.html |
||||
@ -0,0 +1,34 @@
|
||||
<!-- |
||||
Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
Unless required by applicable law or agreed to in writing, |
||||
software distributed under the License is distributed on an |
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
KIND, either express or implied. See the License for the |
||||
specific language governing permissions and limitations |
||||
under the License. |
||||
--> |
||||
|
||||
## Roadmap |
||||
|
||||
### v0.0.3 |
||||
|
||||
Add other features, tasks, parameters in DS, keep code coverage up to 90% |
||||
|
||||
### v0.0.2 |
||||
|
||||
Add docs about how to use and develop package, code coverage up to 90%, add CI/CD |
||||
for package |
||||
|
||||
### v0.0.1(current) |
||||
|
||||
Setup up POC, for defining DAG with python code, running DAG manually, |
||||
releasing to pypi |
||||
@ -0,0 +1,43 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
r""" |
||||
After tutorial.py file submit to Apache DolphinScheduler server a DAG would be create, |
||||
and workflow DAG graph as below: |
||||
|
||||
--> task_child_one |
||||
/ \ |
||||
task_parent --> --> task_union |
||||
\ / |
||||
--> task_child_two |
||||
""" |
||||
|
||||
from pydolphinscheduler.core.process_definition import ProcessDefinition |
||||
from pydolphinscheduler.tasks.shell import Shell |
||||
|
||||
with ProcessDefinition(name="tutorial", tenant="tenant_exists") as pd: |
||||
task_parent = Shell(name="task_parent", command="echo hello pydolphinscheduler") |
||||
task_child_one = Shell(name="task_child_one", command="echo 'child one'") |
||||
task_child_two = Shell(name="task_child_two", command="echo 'child two'") |
||||
task_union = Shell(name="task_union", command="echo union") |
||||
|
||||
task_group = [task_child_one, task_child_two] |
||||
task_parent.set_downstream(task_group) |
||||
|
||||
task_union << task_group |
||||
|
||||
pd.run() |
||||
@ -0,0 +1,18 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
py4j~=0.10.9.2 |
||||
@ -0,0 +1,24 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
# testting |
||||
pytest~=6.2.5 |
||||
# code linting and formatting |
||||
flake8-black~=0.2.3 |
||||
# flake8 |
||||
# flake8-docstrings |
||||
# flake8-black |
||||
@ -0,0 +1,16 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
@ -0,0 +1,90 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
|
||||
import sys |
||||
from os.path import dirname, join |
||||
|
||||
from setuptools import find_packages, setup |
||||
|
||||
version = '0.0.1.dev0' |
||||
|
||||
if sys.version_info[0] < 3: |
||||
raise Exception("pydolphinscheduler does not support Python 2. Please upgrade to Python 3.") |
||||
|
||||
|
||||
def read(*names, **kwargs): |
||||
return open( |
||||
join(dirname(__file__), *names), encoding=kwargs.get("encoding", "utf8") |
||||
).read() |
||||
|
||||
|
||||
setup( |
||||
name="pydolphinscheduler", |
||||
version=version, |
||||
license="Apache License 2.0", |
||||
description="Apache DolphinScheduler python SDK", |
||||
long_description=read("README.md"), |
||||
# Make sure pypi is expecting markdown |
||||
long_description_content_type="text/markdown", |
||||
author="Apache Software Foundation", |
||||
author_email="dev@dolphinscheduler.apache.org", |
||||
url="https://dolphinscheduler.apache.org/", |
||||
python_requires=">=3.6", |
||||
keywords=[ |
||||
"dolphinscheduler", |
||||
"workflow", |
||||
"scheduler", |
||||
"taskflow", |
||||
], |
||||
project_urls={ |
||||
"Homepage": "https://dolphinscheduler.apache.org", |
||||
"Documentation": "https://dolphinscheduler.apache.org/en-us/docs/latest/user_doc/quick-start.html", |
||||
"Source": "https://github.com/apache/dolphinscheduler", |
||||
"Issue Tracker": "https://github.com/apache/dolphinscheduler/issues", |
||||
"Discussion": "https://github.com/apache/dolphinscheduler/discussions", |
||||
"Twitter": "https://twitter.com/dolphinschedule", |
||||
}, |
||||
packages=find_packages(where="src"), |
||||
package_dir={"": "src"}, |
||||
include_package_data=True, |
||||
classifiers=[ |
||||
# complete classifier list: http://pypi.python.org/pypi?%3Aaction=list_classifiers |
||||
"Development Status :: 1 - Planning", |
||||
"Environment :: Console", |
||||
"Intended Audience :: Developers", |
||||
"License :: OSI Approved :: Apache Software License", |
||||
"Operating System :: Unix", |
||||
"Operating System :: POSIX", |
||||
"Operating System :: Microsoft :: Windows", |
||||
"Programming Language :: Python", |
||||
"Programming Language :: Python :: 3", |
||||
"Programming Language :: Python :: 3.6", |
||||
"Programming Language :: Python :: 3.7", |
||||
"Programming Language :: Python :: 3.8", |
||||
"Programming Language :: Python :: 3.9", |
||||
"Programming Language :: Python :: Implementation :: CPython", |
||||
"Programming Language :: Python :: Implementation :: PyPy", |
||||
"Topic :: Software Development :: User Interfaces", |
||||
], |
||||
install_requires=[ |
||||
# Core |
||||
"py4j~=0.10", |
||||
# Dev |
||||
"pytest~=6.2", |
||||
] |
||||
) |
||||
@ -0,0 +1,16 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
@ -0,0 +1,16 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
@ -0,0 +1,74 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
class ProcessDefinitionReleaseState: |
||||
""" |
||||
ProcessDefinition release state |
||||
""" |
||||
ONLINE: str = "ONLINE" |
||||
OFFLINE: str = "OFFLINE" |
||||
|
||||
|
||||
class ProcessDefinitionDefault: |
||||
""" |
||||
ProcessDefinition default values |
||||
""" |
||||
PROJECT: str = "project-pydolphin" |
||||
TENANT: str = "tenant_pydolphin" |
||||
USER: str = "userPythonGateway" |
||||
# TODO simple set password same as username |
||||
USER_PWD: str = "userPythonGateway" |
||||
USER_EMAIL: str = "userPythonGateway@dolphinscheduler.com" |
||||
USER_PHONE: str = "11111111111" |
||||
USER_STATE: int = 1 |
||||
QUEUE: str = "queuePythonGateway" |
||||
WORKER_GROUP: str = "default" |
||||
|
||||
|
||||
class TaskPriority(str): |
||||
HIGHEST = "HIGHEST" |
||||
HIGH = "HIGH" |
||||
MEDIUM = "MEDIUM" |
||||
LOW = "LOW" |
||||
LOWEST = "LOWEST" |
||||
|
||||
|
||||
class TaskFlag(str): |
||||
YES = "YES" |
||||
NO = "NO" |
||||
|
||||
|
||||
class TaskTimeoutFlag(str): |
||||
CLOSE = "CLOSE" |
||||
|
||||
|
||||
class TaskType(str): |
||||
SHELL = "SHELL" |
||||
|
||||
|
||||
class DefaultTaskCodeNum(str): |
||||
DEFAULT = 1 |
||||
|
||||
|
||||
class JavaGatewayDefault(str): |
||||
RESULT_MESSAGE_KEYWORD = "msg" |
||||
RESULT_MESSAGE_SUCCESS = "success" |
||||
|
||||
RESULT_STATUS_KEYWORD = "status" |
||||
RESULT_STATUS_SUCCESS = "SUCCESS" |
||||
|
||||
RESULT_DATA = "data" |
||||
@ -0,0 +1,16 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
@ -0,0 +1,72 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
from typing import Optional, Dict |
||||
|
||||
# from pydolphinscheduler.side.user import User |
||||
from pydolphinscheduler.utils.string import attr2camel |
||||
|
||||
|
||||
class Base: |
||||
""" |
||||
Base |
||||
""" |
||||
|
||||
_KEY_ATTR: set = { |
||||
"name", |
||||
"description" |
||||
} |
||||
|
||||
_TO_DICT_ATTR: set = set() |
||||
|
||||
DEFAULT_ATTR: Dict = {} |
||||
|
||||
def __init__( |
||||
self, |
||||
name: str, |
||||
description: Optional[str] = None |
||||
): |
||||
self.name = name |
||||
self.description = description |
||||
|
||||
def __repr__(self) -> str: |
||||
return f'<{type(self).__name__}: name="{self.name}">' |
||||
|
||||
def __eq__(self, other): |
||||
return type(self) == type(other) and \ |
||||
all(getattr(self, a, None) == getattr(other, a, None) for a in self._KEY_ATTR) |
||||
|
||||
# TODO check how Redash do |
||||
# TODO DRY |
||||
def to_dict(self, camel_attr=True) -> Dict: |
||||
# content = {} |
||||
# for attr, value in self.__dict__.items(): |
||||
# # Don't publish private variables |
||||
# if attr.startswith("_"): |
||||
# continue |
||||
# else: |
||||
# content[snake2camel(attr)] = value |
||||
# content.update(self.DEFAULT_ATTR) |
||||
# return content |
||||
content = {} |
||||
for attr in self._TO_DICT_ATTR: |
||||
val = getattr(self, attr, None) |
||||
if camel_attr: |
||||
content[attr2camel(attr)] = val |
||||
else: |
||||
content[attr] = val |
||||
return content |
||||
@ -0,0 +1,43 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
from typing import Optional |
||||
|
||||
from pydolphinscheduler.constants import ProcessDefinitionDefault |
||||
from pydolphinscheduler.core.base import Base |
||||
|
||||
|
||||
class BaseSide(Base): |
||||
def __init__( |
||||
self, |
||||
name: str, |
||||
description: Optional[str] = None |
||||
): |
||||
super().__init__(name, description) |
||||
|
||||
@classmethod |
||||
def create_if_not_exists( |
||||
cls, |
||||
# TODO comment for avoiding cycle import |
||||
# user: Optional[User] = ProcessDefinitionDefault.USER |
||||
user=ProcessDefinitionDefault.USER |
||||
): |
||||
""" |
||||
Create Base if not exists |
||||
""" |
||||
|
||||
raise NotImplementedError |
||||
@ -0,0 +1,249 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
import json |
||||
from typing import Optional, List, Dict, Set |
||||
|
||||
from pydolphinscheduler.constants import ProcessDefinitionReleaseState, ProcessDefinitionDefault |
||||
from pydolphinscheduler.core.base import Base |
||||
from pydolphinscheduler.java_gateway import launch_gateway |
||||
from pydolphinscheduler.side import Tenant, Project, User |
||||
|
||||
|
||||
class ProcessDefinitionContext: |
||||
_context_managed_process_definition: Optional["ProcessDefinition"] = None |
||||
|
||||
@classmethod |
||||
def set(cls, pd: "ProcessDefinition") -> None: |
||||
cls._context_managed_process_definition = pd |
||||
|
||||
@classmethod |
||||
def get(cls) -> Optional["ProcessDefinition"]: |
||||
return cls._context_managed_process_definition |
||||
|
||||
@classmethod |
||||
def delete(cls) -> None: |
||||
cls._context_managed_process_definition = None |
||||
|
||||
|
||||
class ProcessDefinition(Base): |
||||
""" |
||||
ProcessDefinition |
||||
TODO :ref: comment may not correct ref |
||||
TODO: maybe we should rename this class, currently use DS object name |
||||
""" |
||||
|
||||
# key attribute for identify ProcessDefinition object |
||||
_KEY_ATTR = { |
||||
"name", |
||||
"project", |
||||
"tenant", |
||||
"release_state", |
||||
"param", |
||||
} |
||||
|
||||
_TO_DICT_ATTR = { |
||||
"name", |
||||
"description", |
||||
"_project", |
||||
"_tenant", |
||||
"timeout", |
||||
"release_state", |
||||
"param", |
||||
"tasks", |
||||
"task_definition_json", |
||||
"task_relation_json", |
||||
} |
||||
|
||||
def __init__( |
||||
self, |
||||
name: str, |
||||
description: Optional[str] = None, |
||||
user: Optional[str] = ProcessDefinitionDefault.USER, |
||||
project: Optional[str] = ProcessDefinitionDefault.PROJECT, |
||||
tenant: Optional[str] = ProcessDefinitionDefault.TENANT, |
||||
queue: Optional[str] = ProcessDefinitionDefault.QUEUE, |
||||
timeout: Optional[int] = 0, |
||||
release_state: Optional[str] = ProcessDefinitionReleaseState.ONLINE, |
||||
param: Optional[List] = None |
||||
): |
||||
super().__init__(name, description) |
||||
self._user = user |
||||
self._project = project |
||||
self._tenant = tenant |
||||
self._queue = queue |
||||
self.timeout = timeout |
||||
self.release_state = release_state |
||||
self.param = param |
||||
self.tasks: dict = {} |
||||
# TODO how to fix circle import |
||||
self._task_relations: set["TaskRelation"] = set() |
||||
self._process_definition_code = None |
||||
|
||||
def __enter__(self) -> "ProcessDefinition": |
||||
ProcessDefinitionContext.set(self) |
||||
return self |
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb) -> None: |
||||
ProcessDefinitionContext.delete() |
||||
|
||||
@property |
||||
def tenant(self) -> Tenant: |
||||
return Tenant(self._tenant) |
||||
|
||||
@tenant.setter |
||||
def tenant(self, tenant: Tenant) -> None: |
||||
self._tenant = tenant.name |
||||
|
||||
@property |
||||
def project(self) -> Project: |
||||
return Project(self._project) |
||||
|
||||
@project.setter |
||||
def project(self, project: Project) -> None: |
||||
self._project = project.name |
||||
|
||||
@property |
||||
def user(self) -> User: |
||||
return User(self._user, |
||||
ProcessDefinitionDefault.USER_PWD, |
||||
ProcessDefinitionDefault.USER_EMAIL, |
||||
ProcessDefinitionDefault.USER_PHONE, |
||||
ProcessDefinitionDefault.TENANT, |
||||
ProcessDefinitionDefault.QUEUE, |
||||
ProcessDefinitionDefault.USER_STATE) |
||||
|
||||
@property |
||||
def task_definition_json(self) -> List[Dict]: |
||||
if not self.tasks: |
||||
return [self.tasks] |
||||
else: |
||||
return [task.to_dict() for task in self.tasks.values()] |
||||
|
||||
@property |
||||
def task_relation_json(self) -> List[Dict]: |
||||
if not self.tasks: |
||||
return [self.tasks] |
||||
else: |
||||
self._handle_root_relation() |
||||
return [tr.to_dict() for tr in self._task_relations] |
||||
|
||||
# TODO inti DAG's tasks are in the same place |
||||
@property |
||||
def task_location(self) -> List[Dict]: |
||||
if not self.tasks: |
||||
return [self.tasks] |
||||
else: |
||||
return [{"taskCode": task_code, "x": 0, "y": 0} for task_code in self.tasks] |
||||
|
||||
@property |
||||
def task_list(self) -> List["Task"]: |
||||
return list(self.tasks.values()) |
||||
|
||||
def _handle_root_relation(self): |
||||
from pydolphinscheduler.core.task import TaskRelation |
||||
post_relation_code = set() |
||||
for relation in self._task_relations: |
||||
post_relation_code.add(relation.post_task_code) |
||||
for task in self.task_list: |
||||
if task.code not in post_relation_code: |
||||
root_relation = TaskRelation(pre_task_code=0, post_task_code=task.code) |
||||
self._task_relations.add(root_relation) |
||||
|
||||
def add_task(self, task: "Task") -> None: |
||||
self.tasks[task.code] = task |
||||
task._process_definition = self |
||||
|
||||
def add_tasks(self, tasks: List["Task"]) -> None: |
||||
for task in tasks: |
||||
self.add_task(task) |
||||
|
||||
def get_task(self, code: str) -> "Task": |
||||
if code not in self.tasks: |
||||
raise ValueError("Task with code %s can not found in process definition %", (code, self.name)) |
||||
return self.tasks[code] |
||||
|
||||
# TODO which tying should return in this case |
||||
def get_tasks_by_name(self, name: str) -> Set["Task"]: |
||||
find = set() |
||||
for task in self.tasks.values(): |
||||
if task.name == name: |
||||
find.add(task) |
||||
return find |
||||
|
||||
def get_one_task_by_name(self, name: str) -> "Task": |
||||
tasks = self.get_tasks_by_name(name) |
||||
if not tasks: |
||||
raise ValueError(f"Can not find task with name {name}.") |
||||
return tasks.pop() |
||||
|
||||
def run(self): |
||||
""" |
||||
Run ProcessDefinition instance, a shortcut for :ref: submit and :ref: start |
||||
Only support manual for now, schedule run will coming soon |
||||
:return: |
||||
""" |
||||
self.submit() |
||||
self.start() |
||||
|
||||
def _ensure_side_model_exists(self): |
||||
""" |
||||
Ensure side model exists which including :ref: Project, Tenant, User. |
||||
If those model not exists, would create default value in :ref: ProcessDefinitionDefault |
||||
""" |
||||
# TODO used metaclass for more pythonic |
||||
self.tenant.create_if_not_exists(self._queue) |
||||
# model User have to create after Tenant created |
||||
self.user.create_if_not_exists() |
||||
# Project model need User object exists |
||||
self.project.create_if_not_exists(self._user) |
||||
|
||||
def submit(self) -> int: |
||||
""" |
||||
Submit ProcessDefinition instance to java gateway |
||||
:return: |
||||
""" |
||||
self._ensure_side_model_exists() |
||||
gateway = launch_gateway() |
||||
self._process_definition_code = gateway.entry_point.createOrUpdateProcessDefinition( |
||||
self._user, |
||||
self._project, |
||||
self.name, |
||||
str(self.description) if self.description else "", |
||||
str(self.param) if self.param else None, |
||||
json.dumps(self.task_location), |
||||
self.timeout, |
||||
self._tenant, |
||||
# TODO add serialization function |
||||
json.dumps(self.task_relation_json), |
||||
json.dumps(self.task_definition_json), |
||||
) |
||||
return self._process_definition_code |
||||
|
||||
def start(self) -> None: |
||||
""" |
||||
Start ProcessDefinition instance which post to `start-process-instance` to java gateway |
||||
:return: |
||||
""" |
||||
gateway = launch_gateway() |
||||
gateway.entry_point.execProcessInstance( |
||||
self._user, |
||||
self._project, |
||||
self.name, |
||||
"", |
||||
"default", |
||||
24 * 3600, |
||||
) |
||||
@ -0,0 +1,237 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
from typing import Optional, List, Dict, Set, Union, Sequence, Tuple |
||||
|
||||
from pydolphinscheduler.constants import TaskPriority, ProcessDefinitionDefault, TaskFlag, TaskTimeoutFlag, \ |
||||
DefaultTaskCodeNum, JavaGatewayDefault |
||||
from pydolphinscheduler.core.base import Base |
||||
from pydolphinscheduler.core.process_definition import ProcessDefinition |
||||
from pydolphinscheduler.core.process_definition import ProcessDefinitionContext |
||||
from pydolphinscheduler.java_gateway import launch_gateway, gateway_result_checker |
||||
from pydolphinscheduler.utils.string import snake2camel, class_name2camel |
||||
|
||||
|
||||
class ObjectJsonBase: |
||||
DEFAULT_ATTR = {} |
||||
|
||||
def __int__(self, *args, **kwargs): |
||||
pass |
||||
|
||||
def __str__(self) -> str: |
||||
content = [] |
||||
for attribute, value in self.__dict__.items(): |
||||
content.append(f"\"{snake2camel(attribute)}\": {value}") |
||||
content = ",".join(content) |
||||
return f"\"{class_name2camel(type(self).__name__)}\":{{{content}}}" |
||||
|
||||
# TODO check how Redash do |
||||
# TODO DRY |
||||
def to_dict(self) -> Dict: |
||||
content = {snake2camel(attr): value for attr, value in self.__dict__.items()} |
||||
content.update(self.DEFAULT_ATTR) |
||||
return content |
||||
|
||||
|
||||
class TaskParams(ObjectJsonBase): |
||||
DEFAULT_CONDITION_RESULT = { |
||||
"successNode": [ |
||||
"" |
||||
], |
||||
"failedNode": [ |
||||
"" |
||||
] |
||||
} |
||||
|
||||
def __init__( |
||||
self, |
||||
raw_script: str, |
||||
local_params: Optional[List] = None, |
||||
resource_list: Optional[List] = None, |
||||
dependence: Optional[Dict] = None, |
||||
wait_start_timeout: Optional[Dict] = None, |
||||
condition_result: Optional[Dict] = None, |
||||
): |
||||
super().__init__() |
||||
self.raw_script = raw_script |
||||
self.local_params = local_params or [] |
||||
self.resource_list = resource_list or [] |
||||
self.dependence = dependence or {} |
||||
self.wait_start_timeout = wait_start_timeout or {} |
||||
# TODO need better way to handle it, this code just for POC |
||||
self.condition_result = condition_result or self.DEFAULT_CONDITION_RESULT |
||||
|
||||
|
||||
class TaskRelation(ObjectJsonBase): |
||||
DEFAULT_ATTR = { |
||||
"name": "", |
||||
"preTaskVersion": 1, |
||||
"postTaskVersion": 1, |
||||
"conditionType": 0, |
||||
"conditionParams": {} |
||||
} |
||||
|
||||
def __init__( |
||||
self, |
||||
pre_task_code: int, |
||||
post_task_code: int, |
||||
): |
||||
super().__init__() |
||||
self.pre_task_code = pre_task_code |
||||
self.post_task_code = post_task_code |
||||
|
||||
def __hash__(self): |
||||
return hash(f"{self.post_task_code}, {self.post_task_code}") |
||||
|
||||
|
||||
class Task(Base): |
||||
|
||||
DEFAULT_DEPS_ATTR = { |
||||
"name": "", |
||||
"preTaskVersion": 1, |
||||
"postTaskVersion": 1, |
||||
"conditionType": 0, |
||||
"conditionParams": {} |
||||
} |
||||
|
||||
def __init__( |
||||
self, |
||||
name: str, |
||||
task_type: str, |
||||
task_params: TaskParams, |
||||
description: Optional[str] = None, |
||||
flag: Optional[str] = TaskFlag.YES, |
||||
task_priority: Optional[str] = TaskPriority.MEDIUM, |
||||
worker_group: Optional[str] = ProcessDefinitionDefault.WORKER_GROUP, |
||||
delay_time: Optional[int] = 0, |
||||
fail_retry_times: Optional[int] = 0, |
||||
fail_retry_interval: Optional[int] = 1, |
||||
timeout_flag: Optional[int] = TaskTimeoutFlag.CLOSE, |
||||
timeout_notify_strategy: Optional = None, |
||||
timeout: Optional[int] = 0, |
||||
process_definition: Optional[ProcessDefinition] = None, |
||||
): |
||||
|
||||
super().__init__(name, description) |
||||
self.task_type = task_type |
||||
self.task_params = task_params |
||||
self.flag = flag |
||||
self.task_priority = task_priority |
||||
self.worker_group = worker_group |
||||
self.fail_retry_times = fail_retry_times |
||||
self.fail_retry_interval = fail_retry_interval |
||||
self.delay_time = delay_time |
||||
self.timeout_flag = timeout_flag |
||||
self.timeout_notify_strategy = timeout_notify_strategy |
||||
self.timeout = timeout |
||||
self._process_definition = None |
||||
self.process_definition: ProcessDefinition = process_definition or ProcessDefinitionContext.get() |
||||
self._upstream_task_codes: Set[int] = set() |
||||
self._downstream_task_codes: Set[int] = set() |
||||
self._task_relation: Set[TaskRelation] = set() |
||||
# move attribute code and version after _process_definition and process_definition declare |
||||
self.code, self.version = self.gen_code_and_version() |
||||
# Add task to process definition, maybe we could put into property process_definition latter |
||||
if self.process_definition is not None and self.code not in self.process_definition.tasks: |
||||
self.process_definition.add_task(self) |
||||
|
||||
@property |
||||
def process_definition(self) -> Optional[ProcessDefinition]: |
||||
if self._process_definition: |
||||
return self._process_definition |
||||
else: |
||||
raise ValueError(f'Task {self} has not been assigned to a ProcessDefinition yet') |
||||
|
||||
@process_definition.setter |
||||
def process_definition(self, process_definition: Optional[ProcessDefinition]): |
||||
self._process_definition = process_definition |
||||
|
||||
def __hash__(self): |
||||
return hash(self.code) |
||||
|
||||
def __lshift__(self, other: Union["Task", Sequence["Task"]]): |
||||
"""Implements Task << Task""" |
||||
self.set_upstream(other) |
||||
return other |
||||
|
||||
def __rshift__(self, other: Union["Task", Sequence["Task"]]): |
||||
"""Implements Task >> Task""" |
||||
self.set_downstream(other) |
||||
return other |
||||
|
||||
def __rrshift__(self, other: Union["Task", Sequence["Task"]]): |
||||
"""Called for Task >> [Task] because list don't have __rshift__ operators.""" |
||||
self.__lshift__(other) |
||||
return self |
||||
|
||||
def __rlshift__(self, other: Union["Task", Sequence["Task"]]): |
||||
"""Called for Task << [Task] because list don't have __lshift__ operators.""" |
||||
self.__rshift__(other) |
||||
return self |
||||
|
||||
def _set_deps(self, tasks: Union["Task", Sequence["Task"]], upstream: bool = True) -> None: |
||||
if not isinstance(tasks, Sequence): |
||||
tasks = [tasks] |
||||
|
||||
for task in tasks: |
||||
if upstream: |
||||
self._upstream_task_codes.add(task.code) |
||||
task._downstream_task_codes.add(self.code) |
||||
|
||||
if self._process_definition: |
||||
task_relation = TaskRelation( |
||||
pre_task_code=task.code, |
||||
post_task_code=self.code, |
||||
) |
||||
self.process_definition._task_relations.add(task_relation) |
||||
else: |
||||
self._downstream_task_codes.add(task.code) |
||||
task._upstream_task_codes.add(self.code) |
||||
|
||||
if self._process_definition: |
||||
task_relation = TaskRelation( |
||||
pre_task_code=self.code, |
||||
post_task_code=task.code, |
||||
) |
||||
self.process_definition._task_relations.add(task_relation) |
||||
|
||||
def set_upstream(self, tasks: Union["Task", Sequence["Task"]]) -> None: |
||||
self._set_deps(tasks, upstream=True) |
||||
|
||||
def set_downstream(self, tasks: Union["Task", Sequence["Task"]]) -> None: |
||||
self._set_deps(tasks, upstream=False) |
||||
|
||||
# TODO code should better generate in bulk mode when :ref: processDefinition run submit or start |
||||
def gen_code_and_version(self) -> Tuple: |
||||
# TODO get code from specific project process definition and task name |
||||
gateway = launch_gateway() |
||||
result = gateway.entry_point.getCodeAndVersion(self.process_definition._project, self.name) |
||||
# result = gateway.entry_point.genTaskCodeList(DefaultTaskCodeNum.DEFAULT) |
||||
# gateway_result_checker(result) |
||||
return result.get("code"), result.get("version") |
||||
|
||||
def to_dict(self, camel_attr=True) -> Dict: |
||||
content = {} |
||||
for attr, value in self.__dict__.items(): |
||||
# Don't publish private variables |
||||
if attr.startswith("_"): |
||||
continue |
||||
elif isinstance(value, TaskParams): |
||||
content[snake2camel(attr)] = value.to_dict() |
||||
else: |
||||
content[snake2camel(attr)] = value |
||||
return content |
||||
@ -0,0 +1,43 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
from typing import Any, Optional |
||||
|
||||
from py4j.java_collections import JavaMap |
||||
from py4j.java_gateway import JavaGateway, GatewayParameters |
||||
|
||||
from pydolphinscheduler.constants import JavaGatewayDefault |
||||
|
||||
|
||||
def launch_gateway() -> JavaGateway: |
||||
# TODO Note that automatic conversion makes calling Java methods slightly less efficient because |
||||
# in the worst case, Py4J needs to go through all registered converters for all parameters. |
||||
# This is why automatic conversion is disabled by default. |
||||
gateway = JavaGateway(gateway_parameters=GatewayParameters(auto_convert=True)) |
||||
return gateway |
||||
|
||||
|
||||
def gateway_result_checker( |
||||
result: JavaMap, |
||||
msg_check: Optional[str] = JavaGatewayDefault.RESULT_MESSAGE_SUCCESS |
||||
) -> Any: |
||||
if result[JavaGatewayDefault.RESULT_STATUS_KEYWORD].toString() != \ |
||||
JavaGatewayDefault.RESULT_STATUS_SUCCESS: |
||||
raise RuntimeError(f"Failed when try to got result for java gateway") |
||||
if msg_check is not None and result[JavaGatewayDefault.RESULT_MESSAGE_KEYWORD] != msg_check: |
||||
raise ValueError(f"Get result state not success.") |
||||
return result |
||||
@ -0,0 +1,20 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
from pydolphinscheduler.side.project import Project |
||||
from pydolphinscheduler.side.tenant import Tenant |
||||
from pydolphinscheduler.side.user import User |
||||
@ -0,0 +1,45 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
from typing import Optional |
||||
|
||||
from pydolphinscheduler.core.base_side import BaseSide |
||||
from pydolphinscheduler.constants import ProcessDefinitionDefault |
||||
from pydolphinscheduler.java_gateway import launch_gateway, gateway_result_checker |
||||
|
||||
|
||||
class Project(BaseSide): |
||||
""" |
||||
Project |
||||
""" |
||||
|
||||
def __init__( |
||||
self, |
||||
name: str = ProcessDefinitionDefault.PROJECT, |
||||
description: Optional[str] = None |
||||
): |
||||
super().__init__(name, description) |
||||
|
||||
def create_if_not_exists(self, user=ProcessDefinitionDefault.USER) -> None: |
||||
""" |
||||
Create Project if not exists |
||||
""" |
||||
gateway = launch_gateway() |
||||
result = gateway.entry_point.createProject(user, self.name, self.description) |
||||
# TODO recover result checker |
||||
# gateway_result_checker(result, None) |
||||
|
||||
@ -0,0 +1,44 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
from typing import Optional |
||||
|
||||
from pydolphinscheduler.constants import ProcessDefinitionDefault |
||||
from pydolphinscheduler.core.base_side import BaseSide |
||||
from pydolphinscheduler.java_gateway import launch_gateway, gateway_result_checker |
||||
|
||||
|
||||
class Queue(BaseSide): |
||||
""" |
||||
Queue |
||||
""" |
||||
|
||||
def __init__( |
||||
self, |
||||
name: str = ProcessDefinitionDefault.QUEUE, |
||||
description: Optional[str] = "" |
||||
): |
||||
super().__init__(name, description) |
||||
|
||||
def create_if_not_exists(self, user=ProcessDefinitionDefault.USER) -> None: |
||||
""" |
||||
Create Queue if not exists |
||||
""" |
||||
gateway = launch_gateway() |
||||
# Here we set Queue.name and Queue.queueName same as self.name |
||||
result = gateway.entry_point.createProject(user, self.name, self.name) |
||||
gateway_result_checker(result, None) |
||||
@ -0,0 +1,45 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
from typing import Optional |
||||
|
||||
from pydolphinscheduler.constants import ProcessDefinitionDefault |
||||
from pydolphinscheduler.core.base_side import BaseSide |
||||
from pydolphinscheduler.java_gateway import launch_gateway, gateway_result_checker |
||||
|
||||
|
||||
class Tenant(BaseSide): |
||||
""" |
||||
Tenant |
||||
""" |
||||
|
||||
def __init__( |
||||
self, |
||||
name: str = ProcessDefinitionDefault.TENANT, |
||||
queue: str = ProcessDefinitionDefault.QUEUE, |
||||
description: Optional[str] = None |
||||
): |
||||
super().__init__(name, description) |
||||
self.queue = queue |
||||
|
||||
def create_if_not_exists(self, queue_name: str, user=ProcessDefinitionDefault.USER) -> None: |
||||
""" |
||||
Create Tenant if not exists |
||||
""" |
||||
gateway = launch_gateway() |
||||
result = gateway.entry_point.createTenant(self.name, self.description, queue_name) |
||||
# gateway_result_checker(result, None) |
||||
@ -0,0 +1,68 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
from typing import Optional |
||||
|
||||
from pydolphinscheduler.core.base_side import BaseSide |
||||
from pydolphinscheduler.java_gateway import launch_gateway, gateway_result_checker |
||||
|
||||
|
||||
class User(BaseSide): |
||||
_KEY_ATTR = { |
||||
"name", |
||||
"password", |
||||
"email", |
||||
"phone", |
||||
"tenant", |
||||
"queue", |
||||
"status", |
||||
} |
||||
|
||||
def __init__( |
||||
self, |
||||
name: str, |
||||
password: str, |
||||
email: str, |
||||
phone: str, |
||||
tenant: str, |
||||
queue: Optional[str] = None, |
||||
status: Optional[int] = 1, |
||||
): |
||||
super().__init__(name) |
||||
self.password = password |
||||
self.email = email |
||||
self.phone = phone |
||||
self.tenant = tenant |
||||
self.queue = queue |
||||
self.status = status |
||||
|
||||
def create_if_not_exists(self, **kwargs): |
||||
""" |
||||
Create User if not exists |
||||
""" |
||||
gateway = launch_gateway() |
||||
result = gateway.entry_point.createUser( |
||||
self.name, |
||||
self.password, |
||||
self.email, |
||||
self.phone, |
||||
self.tenant, |
||||
self.queue, |
||||
self.status |
||||
) |
||||
# TODO recover result checker |
||||
# gateway_result_checker(result, None) |
||||
@ -0,0 +1,35 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
from typing import Optional |
||||
|
||||
from pydolphinscheduler.core.base_side import BaseSide |
||||
|
||||
|
||||
class WorkerGroup(BaseSide): |
||||
""" |
||||
Worker Group |
||||
""" |
||||
|
||||
def __init__( |
||||
self, |
||||
name: str, |
||||
address: str, |
||||
description: Optional[str] = None |
||||
): |
||||
super().__init__(name, description) |
||||
self.address = address |
||||
@ -0,0 +1,16 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
@ -0,0 +1,34 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
from pydolphinscheduler.constants import TaskType |
||||
from pydolphinscheduler.core.task import Task, TaskParams |
||||
|
||||
|
||||
class Shell(Task): |
||||
# TODO maybe we could use instance name to replace attribute `name` |
||||
# which is simplify as `task_shell = Shell(command = "echo 1")` and |
||||
# task.name assign to `task_shell` |
||||
def __init__( |
||||
self, |
||||
name: str, |
||||
command: str, |
||||
task_type: str = TaskType.SHELL, |
||||
*args, **kwargs |
||||
): |
||||
task_params = TaskParams(raw_script=command) |
||||
super().__init__(name, task_type, task_params, *args, **kwargs) |
||||
@ -0,0 +1,16 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
@ -0,0 +1,30 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
def attr2camel(attr: str, include_private=True): |
||||
if include_private: |
||||
attr = attr.lstrip("_") |
||||
return snake2camel(attr) |
||||
|
||||
|
||||
def snake2camel(snake: str): |
||||
components = snake.split("_") |
||||
return components[0] + "".join(x.title() for x in components[1:]) |
||||
|
||||
|
||||
def class_name2camel(class_name: str): |
||||
return class_name[0].lower() + class_name[1:] |
||||
@ -0,0 +1,16 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
@ -0,0 +1,16 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
@ -0,0 +1,118 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
import pytest |
||||
|
||||
from pydolphinscheduler.constants import ProcessDefinitionDefault, ProcessDefinitionReleaseState |
||||
from pydolphinscheduler.core.process_definition import ProcessDefinition |
||||
from pydolphinscheduler.core.task import Task, TaskParams |
||||
from pydolphinscheduler.side import Tenant, Project, User |
||||
|
||||
TEST_PROCESS_DEFINITION_NAME = "simple-test-process-definition" |
||||
|
||||
|
||||
@pytest.mark.parametrize( |
||||
"func", |
||||
[ |
||||
"run", "submit", "start" |
||||
] |
||||
) |
||||
def test_process_definition_key_attr(func): |
||||
with ProcessDefinition(TEST_PROCESS_DEFINITION_NAME) as pd: |
||||
assert hasattr(pd, func), f"ProcessDefinition instance don't have attribute `{func}`" |
||||
|
||||
|
||||
@pytest.mark.parametrize( |
||||
"name,value", |
||||
[ |
||||
("project", Project(ProcessDefinitionDefault.PROJECT)), |
||||
("tenant", Tenant(ProcessDefinitionDefault.TENANT)), |
||||
("user", User(ProcessDefinitionDefault.USER, |
||||
ProcessDefinitionDefault.USER_PWD, |
||||
ProcessDefinitionDefault.USER_EMAIL, |
||||
ProcessDefinitionDefault.USER_PHONE, |
||||
ProcessDefinitionDefault.TENANT, |
||||
ProcessDefinitionDefault.QUEUE, |
||||
ProcessDefinitionDefault.USER_STATE)), |
||||
("release_state", ProcessDefinitionReleaseState.ONLINE), |
||||
], |
||||
) |
||||
def test_process_definition_default_value(name, value): |
||||
with ProcessDefinition(TEST_PROCESS_DEFINITION_NAME) as pd: |
||||
assert getattr(pd, name) == value, \ |
||||
f"ProcessDefinition instance attribute `{name}` have not except default value `{getattr(pd, name)}`" |
||||
|
||||
|
||||
@pytest.mark.parametrize( |
||||
"name,cls,expect", |
||||
[ |
||||
("project", Project, "project"), |
||||
("tenant", Tenant, "tenant"), |
||||
], |
||||
) |
||||
def test_process_definition_set_attr(name, cls, expect): |
||||
with ProcessDefinition(TEST_PROCESS_DEFINITION_NAME) as pd: |
||||
setattr(pd, name, cls(expect)) |
||||
assert getattr(pd, name) == cls( |
||||
expect), f"ProcessDefinition set attribute `{name}` do not work expect" |
||||
|
||||
|
||||
def test_process_definition_to_dict_without_task(): |
||||
expect = { |
||||
"name": TEST_PROCESS_DEFINITION_NAME, |
||||
"description": None, |
||||
"project": ProcessDefinitionDefault.PROJECT, |
||||
"tenant": ProcessDefinitionDefault.TENANT, |
||||
"timeout": 0, |
||||
"releaseState": ProcessDefinitionReleaseState.ONLINE, |
||||
"param": None, |
||||
"tasks": {}, |
||||
"taskDefinitionJson": [{}], |
||||
"taskRelationJson": [{}], |
||||
} |
||||
with ProcessDefinition(TEST_PROCESS_DEFINITION_NAME) as pd: |
||||
assert pd.to_dict() == expect |
||||
|
||||
|
||||
def test_process_definition_simple(): |
||||
expect_tasks_num = 5 |
||||
with ProcessDefinition(TEST_PROCESS_DEFINITION_NAME) as pd: |
||||
for i in range(expect_tasks_num): |
||||
task_params = TaskParams(raw_script=f"test-raw-script-{i}") |
||||
curr_task = Task(name=f"task-{i}", task_type=f"type-{i}", task_params=task_params) |
||||
# Set deps task i as i-1 parent |
||||
if i > 0: |
||||
pre_task = pd.get_one_task_by_name(f"task-{i - 1}") |
||||
curr_task.set_upstream(pre_task) |
||||
assert len(pd.tasks) == expect_tasks_num |
||||
|
||||
# Test if task process_definition same as origin one |
||||
task: Task = pd.get_one_task_by_name("task-0") |
||||
assert pd is task.process_definition |
||||
|
||||
# Test if all tasks with expect deps |
||||
for i in range(expect_tasks_num): |
||||
task: Task = pd.get_one_task_by_name(f"task-{i}") |
||||
if i == 0: |
||||
assert task._upstream_task_codes == set() |
||||
assert task._downstream_task_codes == {pd.get_one_task_by_name("task-1").code} |
||||
elif i == expect_tasks_num - 1: |
||||
assert task._upstream_task_codes == {pd.get_one_task_by_name(f"task-{i - 1}").code} |
||||
assert task._downstream_task_codes == set() |
||||
else: |
||||
assert task._upstream_task_codes == {pd.get_one_task_by_name(f"task-{i - 1}").code} |
||||
assert task._downstream_task_codes == {pd.get_one_task_by_name(f"task-{i + 1}").code} |
||||
@ -0,0 +1,96 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
|
||||
from unittest.mock import patch |
||||
|
||||
from pydolphinscheduler.core.task import TaskParams, TaskRelation, Task |
||||
|
||||
|
||||
def test_task_params_to_dict(): |
||||
raw_script = "test_task_params_to_dict" |
||||
expect = { |
||||
"resourceList": [], |
||||
"localParams": [], |
||||
"rawScript": raw_script, |
||||
"dependence": {}, |
||||
"conditionResult": TaskParams.DEFAULT_CONDITION_RESULT, |
||||
"waitStartTimeout": {} |
||||
} |
||||
task_param = TaskParams(raw_script=raw_script) |
||||
assert task_param.to_dict() == expect |
||||
|
||||
|
||||
def test_task_relation_to_dict(): |
||||
pre_task_code = 123 |
||||
post_task_code = 456 |
||||
expect = { |
||||
"name": "", |
||||
"preTaskCode": pre_task_code, |
||||
"postTaskCode": post_task_code, |
||||
"preTaskVersion": 1, |
||||
"postTaskVersion": 1, |
||||
"conditionType": 0, |
||||
"conditionParams": {} |
||||
} |
||||
task_param = TaskRelation(pre_task_code=pre_task_code, post_task_code=post_task_code) |
||||
assert task_param.to_dict() == expect |
||||
|
||||
|
||||
def test_task_to_dict(): |
||||
code = "123" |
||||
name = "test_task_to_dict" |
||||
task_type = "test_task_to_dict_type" |
||||
raw_script = "test_task_params_to_dict" |
||||
expect = { |
||||
"code": code, |
||||
"name": name, |
||||
"version": 1, |
||||
"description": None, |
||||
"delayTime": 0, |
||||
"taskType": task_type, |
||||
"taskParams": { |
||||
"resourceList": [], |
||||
"localParams": [], |
||||
"rawScript": raw_script, |
||||
"dependence": {}, |
||||
"conditionResult": { |
||||
"successNode": [ |
||||
"" |
||||
], |
||||
"failedNode": [ |
||||
"" |
||||
] |
||||
}, |
||||
"waitStartTimeout": {} |
||||
}, |
||||
"flag": "YES", |
||||
"taskPriority": "MEDIUM", |
||||
"workerGroup": "worker-group-pydolphin", |
||||
"failRetryTimes": 0, |
||||
"failRetryInterval": 1, |
||||
"timeoutFlag": "CLOSE", |
||||
"timeoutNotifyStrategy": None, |
||||
"timeout": 0 |
||||
} |
||||
with patch('pydolphinscheduler.core.task.Task.gen_code', return_value=code): |
||||
task = Task( |
||||
name=name, |
||||
task_type=task_type, |
||||
task_params=TaskParams(raw_script) |
||||
) |
||||
assert task.to_dict() == expect |
||||
@ -0,0 +1,16 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
@ -0,0 +1,16 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
@ -0,0 +1,61 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
|
||||
from unittest.mock import patch |
||||
|
||||
from pydolphinscheduler.tasks.shell import Shell |
||||
|
||||
|
||||
def test_shell_to_dict(): |
||||
code = "123" |
||||
name = "test_shell_to_dict" |
||||
command = "echo test shell" |
||||
expect = { |
||||
"code": code, |
||||
"name": name, |
||||
"version": 1, |
||||
"description": None, |
||||
"delayTime": 0, |
||||
"taskType": "SHELL", |
||||
"taskParams": { |
||||
"resourceList": [], |
||||
"localParams": [], |
||||
"rawScript": command, |
||||
"dependence": {}, |
||||
"conditionResult": { |
||||
"successNode": [ |
||||
"" |
||||
], |
||||
"failedNode": [ |
||||
"" |
||||
] |
||||
}, |
||||
"waitStartTimeout": {} |
||||
}, |
||||
"flag": "YES", |
||||
"taskPriority": "MEDIUM", |
||||
"workerGroup": "worker-group-pydolphin", |
||||
"failRetryTimes": 0, |
||||
"failRetryInterval": 1, |
||||
"timeoutFlag": "CLOSE", |
||||
"timeoutNotifyStrategy": None, |
||||
"timeout": 0 |
||||
} |
||||
with patch('pydolphinscheduler.core.task.Task.gen_code', return_value=code): |
||||
shell = Shell(name, command) |
||||
assert shell.to_dict() == expect |
||||
@ -0,0 +1,46 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
|
||||
from py4j.java_gateway import java_import, JavaGateway |
||||
|
||||
|
||||
def test_gateway_connect(): |
||||
gateway = JavaGateway() |
||||
app = gateway.entry_point |
||||
assert app.ping() == "PONG" |
||||
|
||||
|
||||
def test_jvm_simple(): |
||||
gateway = JavaGateway() |
||||
smaller = gateway.jvm.java.lang.Integer.MIN_VALUE |
||||
bigger = gateway.jvm.java.lang.Integer.MAX_VALUE |
||||
assert bigger > smaller |
||||
|
||||
|
||||
def test_python_client_java_import_single(): |
||||
gateway = JavaGateway() |
||||
java_import(gateway.jvm, "org.apache.dolphinscheduler.common.utils.FileUtils") |
||||
assert hasattr(gateway.jvm, "FileUtils") |
||||
|
||||
|
||||
def test_python_client_java_import_package(): |
||||
gateway = JavaGateway() |
||||
java_import(gateway.jvm, "org.apache.dolphinscheduler.common.utils.*") |
||||
# test if jvm view have some common utils |
||||
for util in ("FileUtils", "OSUtils", "DateUtils"): |
||||
assert hasattr(gateway.jvm, util) |
||||
@ -0,0 +1,310 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.server; |
||||
|
||||
import org.apache.dolphinscheduler.api.enums.Status; |
||||
import org.apache.dolphinscheduler.api.service.ExecutorService; |
||||
import org.apache.dolphinscheduler.api.service.ProcessDefinitionService; |
||||
import org.apache.dolphinscheduler.api.service.ProjectService; |
||||
import org.apache.dolphinscheduler.api.service.QueueService; |
||||
import org.apache.dolphinscheduler.api.service.TaskDefinitionService; |
||||
import org.apache.dolphinscheduler.api.service.TenantService; |
||||
import org.apache.dolphinscheduler.api.service.UsersService; |
||||
import org.apache.dolphinscheduler.api.utils.Result; |
||||
import org.apache.dolphinscheduler.common.Constants; |
||||
import org.apache.dolphinscheduler.common.enums.FailureStrategy; |
||||
import org.apache.dolphinscheduler.common.enums.Priority; |
||||
import org.apache.dolphinscheduler.common.enums.ReleaseState; |
||||
import org.apache.dolphinscheduler.common.enums.RunMode; |
||||
import org.apache.dolphinscheduler.common.enums.TaskDependType; |
||||
import org.apache.dolphinscheduler.common.enums.UserType; |
||||
import org.apache.dolphinscheduler.common.enums.WarningType; |
||||
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils; |
||||
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; |
||||
import org.apache.dolphinscheduler.dao.entity.Project; |
||||
import org.apache.dolphinscheduler.dao.entity.Queue; |
||||
import org.apache.dolphinscheduler.dao.entity.TaskDefinition; |
||||
import org.apache.dolphinscheduler.dao.entity.Tenant; |
||||
import org.apache.dolphinscheduler.dao.entity.User; |
||||
import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; |
||||
import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; |
||||
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; |
||||
|
||||
import java.util.HashMap; |
||||
import java.util.List; |
||||
import java.util.Map; |
||||
import java.util.Objects; |
||||
|
||||
import javax.annotation.PostConstruct; |
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired; |
||||
import org.springframework.boot.SpringApplication; |
||||
import org.springframework.boot.web.servlet.support.SpringBootServletInitializer; |
||||
import org.springframework.context.annotation.ComponentScan; |
||||
import org.springframework.context.annotation.FilterType; |
||||
|
||||
import py4j.GatewayServer; |
||||
|
||||
@ComponentScan(value = "org.apache.dolphinscheduler", excludeFilters = { |
||||
@ComponentScan.Filter(type = FilterType.REGEX, pattern = { |
||||
"org.apache.dolphinscheduler.server.master.*", |
||||
"org.apache.dolphinscheduler.server.worker.*", |
||||
"org.apache.dolphinscheduler.server.monitor.*", |
||||
"org.apache.dolphinscheduler.server.log.*" |
||||
}) |
||||
}) |
||||
public class PythonGatewayServer extends SpringBootServletInitializer { |
||||
@Autowired |
||||
private ProcessDefinitionMapper processDefinitionMapper; |
||||
|
||||
@Autowired |
||||
private ProjectService projectService; |
||||
|
||||
@Autowired |
||||
private TenantService tenantService; |
||||
|
||||
@Autowired |
||||
private ExecutorService executorService; |
||||
|
||||
@Autowired |
||||
private ProcessDefinitionService processDefinitionService; |
||||
|
||||
@Autowired |
||||
private TaskDefinitionService taskDefinitionService; |
||||
|
||||
@Autowired |
||||
private UsersService usersService; |
||||
|
||||
@Autowired |
||||
private QueueService queueService; |
||||
|
||||
@Autowired |
||||
private ProjectMapper projectMapper; |
||||
|
||||
@Autowired |
||||
private TaskDefinitionMapper taskDefinitionMapper; |
||||
|
||||
// TODO replace this user to build in admin user if we make sure build in one could not be change
|
||||
private final User dummyAdminUser = new User() { |
||||
{ |
||||
setId(Integer.MAX_VALUE); |
||||
setUserName("dummyUser"); |
||||
setUserType(UserType.ADMIN_USER); |
||||
} |
||||
}; |
||||
|
||||
private final Queue queuePythonGateway = new Queue() { |
||||
{ |
||||
setId(Integer.MAX_VALUE); |
||||
setQueueName("queuePythonGateway"); |
||||
} |
||||
}; |
||||
|
||||
public String ping() { |
||||
return "PONG"; |
||||
} |
||||
|
||||
// TODO Should we import package in python client side? utils package can but service can not, why
|
||||
// Core api
|
||||
public Map<String, Object> genTaskCodeList(Integer genNum) { |
||||
return taskDefinitionService.genTaskCodeList(genNum); |
||||
} |
||||
|
||||
public Map<String, Long> getCodeAndVersion(String projectName, String taskName) throws SnowFlakeUtils.SnowFlakeException { |
||||
Project project = projectMapper.queryByName(projectName); |
||||
Map<String, Long> result = new HashMap<>(); |
||||
// project do not exists, mean task not exists too, so we should directly return init value
|
||||
if (project == null) { |
||||
result.put("code", SnowFlakeUtils.getInstance().nextId()); |
||||
result.put("version", 0L); |
||||
return result; |
||||
} |
||||
TaskDefinition taskDefinition = taskDefinitionMapper.queryByName(project.getCode(), taskName); |
||||
if (taskDefinition == null) { |
||||
result.put("code", SnowFlakeUtils.getInstance().nextId()); |
||||
result.put("version", 0L); |
||||
} else { |
||||
result.put("code", taskDefinition.getCode()); |
||||
result.put("version", (long) taskDefinition.getVersion()); |
||||
} |
||||
return result; |
||||
} |
||||
|
||||
/** |
||||
* create or update process definition. |
||||
* If process definition do not exists in Project=`projectCode` would create a new one |
||||
* If process definition already exists in Project=`projectCode` would update it |
||||
* All requests |
||||
* <p> |
||||
* |
||||
* @param name process definition name |
||||
* @param description description |
||||
* @param globalParams global params |
||||
* @param locations locations for nodes |
||||
* @param timeout timeout |
||||
* @param tenantCode tenantCode |
||||
* @param taskRelationJson relation json for nodes |
||||
* @param taskDefinitionJson taskDefinitionJson |
||||
* @return create result code |
||||
*/ |
||||
public Long createOrUpdateProcessDefinition(String userName, |
||||
String projectName, |
||||
String name, |
||||
String description, |
||||
String globalParams, |
||||
String locations, |
||||
int timeout, |
||||
String tenantCode, |
||||
String taskRelationJson, |
||||
String taskDefinitionJson) { |
||||
User user = usersService.queryUser(userName); |
||||
Project project = (Project) projectService.queryByName(user, projectName).get(Constants.DATA_LIST); |
||||
long projectCode = project.getCode(); |
||||
Map<String, Object> verifyProcessDefinitionExists = processDefinitionService.verifyProcessDefinitionName(user, projectCode, name); |
||||
|
||||
if (verifyProcessDefinitionExists.get(Constants.STATUS) != Status.SUCCESS) { |
||||
// update process definition
|
||||
ProcessDefinition processDefinition = processDefinitionMapper.queryByDefineName(projectCode, name); |
||||
long processDefinitionCode = processDefinition.getCode(); |
||||
// make sure process definition offline which could edit
|
||||
processDefinitionService.releaseProcessDefinition(user, projectCode, processDefinitionCode, ReleaseState.OFFLINE); |
||||
Map<String, Object> result = processDefinitionService.updateProcessDefinition(user, projectCode, name, processDefinitionCode, description, globalParams, |
||||
locations, timeout, tenantCode, taskRelationJson, taskDefinitionJson); |
||||
return processDefinitionCode; |
||||
} else { |
||||
// create process definition
|
||||
Map<String, Object> result = processDefinitionService.createProcessDefinition(user, projectCode, name, description, globalParams, |
||||
locations, timeout, tenantCode, taskRelationJson, taskDefinitionJson); |
||||
ProcessDefinition processDefinition = (ProcessDefinition) result.get(Constants.DATA_LIST); |
||||
return processDefinition.getCode(); |
||||
} |
||||
} |
||||
|
||||
public void execProcessInstance(String userName, |
||||
String projectName, |
||||
String processDefinitionName, |
||||
String cronTime, |
||||
String workerGroup, |
||||
Integer timeout |
||||
) { |
||||
User user = usersService.queryUser(userName); |
||||
Project project = projectMapper.queryByName(projectName); |
||||
ProcessDefinition processDefinition = processDefinitionMapper.queryByDefineName(project.getCode(), processDefinitionName); |
||||
|
||||
// temp default value
|
||||
FailureStrategy failureStrategy = FailureStrategy.CONTINUE; |
||||
TaskDependType taskDependType = TaskDependType.TASK_POST; |
||||
WarningType warningType = WarningType.NONE; |
||||
RunMode runMode = RunMode.RUN_MODE_SERIAL; |
||||
Priority priority = Priority.MEDIUM; |
||||
int warningGroupId = 0; |
||||
Long environmentCode = -1L; |
||||
Map<String, String> startParams = null; |
||||
Integer expectedParallelismNumber = null; |
||||
String startNodeList = null; |
||||
|
||||
// make sure process definition online
|
||||
processDefinitionService.releaseProcessDefinition(user, project.getCode(), processDefinition.getCode(), ReleaseState.ONLINE); |
||||
|
||||
executorService.execProcessInstance(user, |
||||
project.getCode(), |
||||
processDefinition.getCode(), |
||||
cronTime, |
||||
null, |
||||
failureStrategy, |
||||
startNodeList, |
||||
taskDependType, |
||||
warningType, |
||||
warningGroupId, |
||||
runMode, |
||||
priority, |
||||
workerGroup, |
||||
environmentCode, |
||||
timeout, |
||||
startParams, |
||||
expectedParallelismNumber, |
||||
0 |
||||
); |
||||
} |
||||
|
||||
// side object
|
||||
public Map<String, Object> createProject(String userName, String name, String desc) { |
||||
User user = usersService.queryUser(userName); |
||||
return projectService.createProject(user, name, desc); |
||||
} |
||||
|
||||
public Map<String, Object> createQueue(String name, String queueName) { |
||||
Result<Object> verifyQueueExists = queueService.verifyQueue(name, queueName); |
||||
if (verifyQueueExists.getCode() == 0) { |
||||
return queueService.createQueue(dummyAdminUser, name, queueName); |
||||
} else { |
||||
Map<String, Object> result = new HashMap<>(); |
||||
// TODO function putMsg do not work here
|
||||
result.put(Constants.STATUS, Status.SUCCESS); |
||||
result.put(Constants.MSG, Status.SUCCESS.getMsg()); |
||||
return result; |
||||
} |
||||
} |
||||
|
||||
public Map<String, Object> createTenant(String tenantCode, String desc, String queueName) throws Exception { |
||||
if (tenantService.checkTenantExists(tenantCode)) { |
||||
Map<String, Object> result = new HashMap<>(); |
||||
// TODO function putMsg do not work here
|
||||
result.put(Constants.STATUS, Status.SUCCESS); |
||||
result.put(Constants.MSG, Status.SUCCESS.getMsg()); |
||||
return result; |
||||
} else { |
||||
Result<Object> verifyQueueExists = queueService.verifyQueue(queueName, queueName); |
||||
if (verifyQueueExists.getCode() == 0) { |
||||
// TODO why create do not return id?
|
||||
queueService.createQueue(dummyAdminUser, queueName, queueName); |
||||
} |
||||
Map<String, Object> result = queueService.queryQueueName(queueName); |
||||
List<Queue> queueList = (List<Queue>) result.get(Constants.DATA_LIST); |
||||
Queue queue = queueList.get(0); |
||||
return tenantService.createTenant(dummyAdminUser, tenantCode, queue.getId(), desc); |
||||
} |
||||
} |
||||
|
||||
public void createUser(String userName, |
||||
String userPassword, |
||||
String email, |
||||
String phone, |
||||
String tenantCode, |
||||
String queue, |
||||
int state) { |
||||
User user = usersService.queryUser(userName); |
||||
if (Objects.isNull(user)) { |
||||
Map<String, Object> tenantResult = tenantService.queryByTenantCode(tenantCode); |
||||
Tenant tenant = (Tenant) tenantResult.get(Constants.DATA_LIST); |
||||
usersService.createUser(userName, userPassword, email, tenant.getId(), phone, queue, state); |
||||
} |
||||
} |
||||
|
||||
@PostConstruct |
||||
public void run() { |
||||
GatewayServer server = new GatewayServer(this); |
||||
GatewayServer.turnLoggingOn(); |
||||
// Start server to accept python client RPC
|
||||
server.start(); |
||||
} |
||||
|
||||
public static void main(String[] args) { |
||||
SpringApplication.run(PythonGatewayServer.class, args); |
||||
} |
||||
} |
||||
Loading…
Reference in new issue