From 2e7036529c356b35ed99ea46f7ffefe66ec2a916 Mon Sep 17 00:00:00 2001 From: Jiajie Zhong Date: Thu, 4 Nov 2021 11:19:11 +0800 Subject: [PATCH] Add CI test for python API (#6636) * Add CI for python API * Fix task assign without process definition * Fix ci * Add header * Add build badge for pydolphinscheduler * Add content to README.md * Remove unnecessary blank line --- .github/workflows/py-tests.yml | 53 +++++++++++++++++++ .../pydolphinscheduler/README.md | 23 ++++++-- .../pydolphinscheduler/pytest.ini | 22 ++++++++ .../src/pydolphinscheduler/core/task.py | 5 +- .../pydolphinscheduler/test/tasks/__init__.py | 16 ------ .../{src => tests}/__init__.py | 0 .../{test => tests/core}/__init__.py | 0 .../core/test_process_definition.py | 3 +- .../{test => tests}/core/test_task.py | 9 ++-- .../{test/core => tests/tasks}/__init__.py | 0 .../{test => tests}/tasks/test_shell.py | 7 +-- .../{test => tests}/test_java_gateway.py | 0 .../example => tests/testing}/__init__.py | 0 .../pydolphinscheduler/tests/testing/task.py | 27 ++++++++++ 14 files changed, 134 insertions(+), 31 deletions(-) create mode 100644 .github/workflows/py-tests.yml create mode 100644 dolphinscheduler-python/pydolphinscheduler/pytest.ini delete mode 100644 dolphinscheduler-python/pydolphinscheduler/test/tasks/__init__.py rename dolphinscheduler-python/pydolphinscheduler/{src => tests}/__init__.py (100%) rename dolphinscheduler-python/pydolphinscheduler/{test => tests/core}/__init__.py (100%) rename dolphinscheduler-python/pydolphinscheduler/{test => tests}/core/test_process_definition.py (98%) rename dolphinscheduler-python/pydolphinscheduler/{test => tests}/core/test_task.py (93%) rename dolphinscheduler-python/pydolphinscheduler/{test/core => tests/tasks}/__init__.py (100%) rename dolphinscheduler-python/pydolphinscheduler/{test => tests}/tasks/test_shell.py (91%) rename dolphinscheduler-python/pydolphinscheduler/{test => tests}/test_java_gateway.py (100%) rename dolphinscheduler-python/pydolphinscheduler/{test/example => tests/testing}/__init__.py (100%) create mode 100644 dolphinscheduler-python/pydolphinscheduler/tests/testing/task.py diff --git a/.github/workflows/py-tests.yml b/.github/workflows/py-tests.yml new file mode 100644 index 0000000000..e37cab3d85 --- /dev/null +++ b/.github/workflows/py-tests.yml @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Python API Tests + +on: + push: + branches: + - dev + paths: + - 'dolphinscheduler-python/**' + pull_request: + paths: + - 'dolphinscheduler-python/**' + +defaults: + run: + working-directory: dolphinscheduler-python/pydolphinscheduler + +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: [3.6, 3.7, 3.8, 3.9] + os: [ubuntu-18.04, macOS-latest, windows-latest] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies & pydolphinscheduler + run: | + pip install -r requirements.txt -r requirements_dev.txt + pip install -e . + - name: Run tests + run: | + pytest diff --git a/dolphinscheduler-python/pydolphinscheduler/README.md b/dolphinscheduler-python/pydolphinscheduler/README.md index a987483360..a6609844e9 100644 --- a/dolphinscheduler-python/pydolphinscheduler/README.md +++ b/dolphinscheduler-python/pydolphinscheduler/README.md @@ -19,6 +19,8 @@ # pydolphinscheduler +[![GitHub Build][ga-py-test]][ga] + pydolphinscheduler is python API for Apache DolphinScheduler, which allow you definition your workflow by python code, aka workflow-as-codes. @@ -47,7 +49,7 @@ section "DolphinScheduler Standalone Quick Start" to set up developer environmen and frontend server in this step, which mean that you could view DolphinScheduler UI in your browser with URL http://localhost:12345/dolphinscheduler -After backend server is being start, all requests from `pydolphinscheduler` would be sends to backend server. +After backend server is being start, all requests from `pydolphinscheduler` would be sent to backend server. And for now we could run a simple example by: ```shell @@ -78,7 +80,7 @@ just open directory `dolphinscheduler-python/pydolphinscheduler` instead of `dol ### Brief Concept Apache DolphinScheduler is design to define workflow by UI, and pydolphinscheduler try to define it by code. When -define by code, user usually do not care user, tanant, or queue exists or not. All user care about is create +define by code, user usually do not care user, tenant, or queue exists or not. All user care about is created a new workflow by the code his/her definition. So we have some **side object** in `pydolphinscheduler/side` directory, their only check object exists or not, and create them if not exists. @@ -91,9 +93,20 @@ other word for more simple). pydolphinscheduler tasks object, we use tasks to define exact job we want DolphinScheduler do for us. For now, we only support `shell` task to execute shell task. [This link][all-task] list all tasks support in DolphinScheduler -and would be implement in the further. +and would be implemented in the further. + +### Testing + +pydolphinscheduler using [pytest][pytest] to test our codebase. GitHub Action will run our test when you create +pull request or commit to dev branch, with python version `3.6|3.7|3.8|3.9` and operating system `linux|macOS|windows`. +To test locally, you could directly run pytest after set `PYTHONPATH` + +```shell +PYTHONPATH=src/ pytest +``` + [pypi]: https://pypi.org/ [dev-setup]: https://dolphinscheduler.apache.org/en-us/development/development-environment-setup.html [ui-project]: http://8.142.34.29:12345/dolphinscheduler/ui/#/projects/list @@ -101,3 +114,7 @@ and would be implement in the further. [pycharm]: https://www.jetbrains.com/pycharm [idea]: https://www.jetbrains.com/idea/ [all-task]: https://dolphinscheduler.apache.org/en-us/docs/dev/user_doc/guide/task/shell.html +[pytest]: https://docs.pytest.org/en/latest/ + +[ga-py-test]: https://github.com/apache/dolphinscheduler/actions/workflows/py-tests.yml/badge.svg?branch=dev +[ga]: https://github.com/apache/dolphinscheduler/actions diff --git a/dolphinscheduler-python/pydolphinscheduler/pytest.ini b/dolphinscheduler-python/pydolphinscheduler/pytest.ini new file mode 100644 index 0000000000..f2c7ae6489 --- /dev/null +++ b/dolphinscheduler-python/pydolphinscheduler/pytest.ini @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[pytest] +# Do not test test_java_gateway.py due to we can not mock java gateway for now +addopts = --ignore=tests/test_java_gateway.py + +# add path here to skip pytest scan it +norecursedirs = + tests/testing diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py index fa97c76b66..51ad74bed6 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py @@ -151,10 +151,7 @@ class Task(Base): @property def process_definition(self) -> Optional[ProcessDefinition]: - if self._process_definition: - return self._process_definition - else: - raise ValueError(f'Task {self} has not been assigned to a ProcessDefinition yet') + return self._process_definition @process_definition.setter def process_definition(self, process_definition: Optional[ProcessDefinition]): diff --git a/dolphinscheduler-python/pydolphinscheduler/test/tasks/__init__.py b/dolphinscheduler-python/pydolphinscheduler/test/tasks/__init__.py deleted file mode 100644 index 13a83393a9..0000000000 --- a/dolphinscheduler-python/pydolphinscheduler/test/tasks/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/dolphinscheduler-python/pydolphinscheduler/src/__init__.py b/dolphinscheduler-python/pydolphinscheduler/tests/__init__.py similarity index 100% rename from dolphinscheduler-python/pydolphinscheduler/src/__init__.py rename to dolphinscheduler-python/pydolphinscheduler/tests/__init__.py diff --git a/dolphinscheduler-python/pydolphinscheduler/test/__init__.py b/dolphinscheduler-python/pydolphinscheduler/tests/core/__init__.py similarity index 100% rename from dolphinscheduler-python/pydolphinscheduler/test/__init__.py rename to dolphinscheduler-python/pydolphinscheduler/tests/core/__init__.py diff --git a/dolphinscheduler-python/pydolphinscheduler/test/core/test_process_definition.py b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_process_definition.py similarity index 98% rename from dolphinscheduler-python/pydolphinscheduler/test/core/test_process_definition.py rename to dolphinscheduler-python/pydolphinscheduler/tests/core/test_process_definition.py index 7155447d66..83a4c2b965 100644 --- a/dolphinscheduler-python/pydolphinscheduler/test/core/test_process_definition.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_process_definition.py @@ -19,8 +19,9 @@ import pytest from pydolphinscheduler.constants import ProcessDefinitionDefault, ProcessDefinitionReleaseState from pydolphinscheduler.core.process_definition import ProcessDefinition -from pydolphinscheduler.core.task import Task, TaskParams +from pydolphinscheduler.core.task import TaskParams from pydolphinscheduler.side import Tenant, Project, User +from tests.testing.task import Task TEST_PROCESS_DEFINITION_NAME = "simple-test-process-definition" diff --git a/dolphinscheduler-python/pydolphinscheduler/test/core/test_task.py b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py similarity index 93% rename from dolphinscheduler-python/pydolphinscheduler/test/core/test_task.py rename to dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py index 1d89b0778a..6e0342806d 100644 --- a/dolphinscheduler-python/pydolphinscheduler/test/core/test_task.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py @@ -52,14 +52,15 @@ def test_task_relation_to_dict(): def test_task_to_dict(): - code = "123" + code = 123 + version = 1 name = "test_task_to_dict" task_type = "test_task_to_dict_type" raw_script = "test_task_params_to_dict" expect = { "code": code, "name": name, - "version": 1, + "version": version, "description": None, "delayTime": 0, "taskType": task_type, @@ -80,14 +81,14 @@ def test_task_to_dict(): }, "flag": "YES", "taskPriority": "MEDIUM", - "workerGroup": "worker-group-pydolphin", + "workerGroup": "default", "failRetryTimes": 0, "failRetryInterval": 1, "timeoutFlag": "CLOSE", "timeoutNotifyStrategy": None, "timeout": 0 } - with patch('pydolphinscheduler.core.task.Task.gen_code', return_value=code): + with patch('pydolphinscheduler.core.task.Task.gen_code_and_version', return_value=(code, version)): task = Task( name=name, task_type=task_type, diff --git a/dolphinscheduler-python/pydolphinscheduler/test/core/__init__.py b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/__init__.py similarity index 100% rename from dolphinscheduler-python/pydolphinscheduler/test/core/__init__.py rename to dolphinscheduler-python/pydolphinscheduler/tests/tasks/__init__.py diff --git a/dolphinscheduler-python/pydolphinscheduler/test/tasks/test_shell.py b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_shell.py similarity index 91% rename from dolphinscheduler-python/pydolphinscheduler/test/tasks/test_shell.py rename to dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_shell.py index ff1b7b5465..91cc431ffc 100644 --- a/dolphinscheduler-python/pydolphinscheduler/test/tasks/test_shell.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_shell.py @@ -22,7 +22,8 @@ from pydolphinscheduler.tasks.shell import Shell def test_shell_to_dict(): - code = "123" + code = 123 + version = 1 name = "test_shell_to_dict" command = "echo test shell" expect = { @@ -49,13 +50,13 @@ def test_shell_to_dict(): }, "flag": "YES", "taskPriority": "MEDIUM", - "workerGroup": "worker-group-pydolphin", + "workerGroup": "default", "failRetryTimes": 0, "failRetryInterval": 1, "timeoutFlag": "CLOSE", "timeoutNotifyStrategy": None, "timeout": 0 } - with patch('pydolphinscheduler.core.task.Task.gen_code', return_value=code): + with patch('pydolphinscheduler.core.task.Task.gen_code_and_version', return_value=(code, version)): shell = Shell(name, command) assert shell.to_dict() == expect diff --git a/dolphinscheduler-python/pydolphinscheduler/test/test_java_gateway.py b/dolphinscheduler-python/pydolphinscheduler/tests/test_java_gateway.py similarity index 100% rename from dolphinscheduler-python/pydolphinscheduler/test/test_java_gateway.py rename to dolphinscheduler-python/pydolphinscheduler/tests/test_java_gateway.py diff --git a/dolphinscheduler-python/pydolphinscheduler/test/example/__init__.py b/dolphinscheduler-python/pydolphinscheduler/tests/testing/__init__.py similarity index 100% rename from dolphinscheduler-python/pydolphinscheduler/test/example/__init__.py rename to dolphinscheduler-python/pydolphinscheduler/tests/testing/__init__.py diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/testing/task.py b/dolphinscheduler-python/pydolphinscheduler/tests/testing/task.py new file mode 100644 index 0000000000..32d3ffaf73 --- /dev/null +++ b/dolphinscheduler-python/pydolphinscheduler/tests/testing/task.py @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import uuid + +from pydolphinscheduler.core.task import Task as SourceTask + + +class Task(SourceTask): + DEFAULT_VERSION = 1 + + def gen_code_and_version(self): + return uuid.uuid1().time, self.DEFAULT_VERSION