diff --git a/.github/workflows/py-ci.yml b/.github/workflows/py-ci.yml index fb8324ebcd..3ab10f7d2e 100644 --- a/.github/workflows/py-ci.yml +++ b/.github/workflows/py-ci.yml @@ -50,6 +50,8 @@ jobs: python-version: 3.7 - name: Install Development Dependences run: pip install -r requirements_dev.txt + - name: Run Isort Checking + run: isort --check . - name: Run Black Checking run: black --check . - name: Run Flake8 Checking diff --git a/dolphinscheduler-python/pydolphinscheduler/.isort.cfg b/dolphinscheduler-python/pydolphinscheduler/.isort.cfg new file mode 100644 index 0000000000..70fa2e05bd --- /dev/null +++ b/dolphinscheduler-python/pydolphinscheduler/.isort.cfg @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[settings] +profile=black diff --git a/dolphinscheduler-python/pydolphinscheduler/README.md b/dolphinscheduler-python/pydolphinscheduler/README.md index a0cb7486b4..b06632cc00 100644 --- a/dolphinscheduler-python/pydolphinscheduler/README.md +++ b/dolphinscheduler-python/pydolphinscheduler/README.md @@ -21,6 +21,7 @@ [![GitHub Build][ga-py-test]][ga] [![Code style: black][black-shield]][black-gh] +[![Imports: isort][isort-shield]][isort-gh] pydolphinscheduler is python API for Apache DolphinScheduler, which allow you definition your workflow by python code, aka workflow-as-codes. @@ -104,16 +105,20 @@ and would be implemented in the further. ### Code Style -We use [Black][black] for code formatter and [Flake8][flake8] for pep8 checker. If you use [pycharm][pycharm] -or [IntelliJ IDEA][idea], maybe you could follow [Black-integration][black-editor] to configure them in your environment. +We use [isort][isort] to automatically keep Python imports alphabetically, and use [Black][black] for code +formatter and [Flake8][flake8] for pep8 checker. If you use [pycharm][pycharm]or [IntelliJ IDEA][idea], +maybe you could follow [Black-integration][black-editor] to configure them in your environment. -Our Python API CI would automatically run unittest when you submit pull request in GitHub, you could also run -static check locally. +Our Python API CI would automatically run code style checker and unittest when you submit pull request in +GitHub, you could also run static check locally. ```shell -# We recommend you run Black before Flake8, because Black could auto fix some code style issue +# We recommend you run isort and Black before Flake8, because Black could auto fix some code style issue # but Flake8 just hint when code style not match pep8 +# Run Isort +isort . + # Run Black black . @@ -158,8 +163,11 @@ this command output. [flake8]: https://flake8.pycqa.org/en/latest/index.html [black-editor]: https://black.readthedocs.io/en/stable/integrations/editors.html#pycharm-intellij-idea [coverage]: https://coverage.readthedocs.io/en/stable/ +[isort]: https://pycqa.github.io/isort/index.html [ga-py-test]: https://github.com/apache/dolphinscheduler/actions/workflows/py-ci.yml/badge.svg?branch=dev [ga]: https://github.com/apache/dolphinscheduler/actions [black-shield]: https://img.shields.io/badge/code%20style-black-000000.svg [black-gh]: https://github.com/psf/black +[isort-shield]: https://img.shields.io/badge/%20imports-isort-%231674b1?style=flat&labelColor=ef8336 +[isort-gh]: https://pycqa.github.io/isort/ diff --git a/dolphinscheduler-python/pydolphinscheduler/examples/tutorial.py b/dolphinscheduler-python/pydolphinscheduler/examples/tutorial.py index 451bb75b22..d58bd753b0 100644 --- a/dolphinscheduler-python/pydolphinscheduler/examples/tutorial.py +++ b/dolphinscheduler-python/pydolphinscheduler/examples/tutorial.py @@ -33,7 +33,6 @@ it will instantiate and run all the task it have. from pydolphinscheduler.core.process_definition import ProcessDefinition from pydolphinscheduler.tasks.shell import Shell - with ProcessDefinition( name="tutorial", schedule="0 0 0 * * ? *", diff --git a/dolphinscheduler-python/pydolphinscheduler/requirements_dev.txt b/dolphinscheduler-python/pydolphinscheduler/requirements_dev.txt index fa40e3caa8..d31a56dc44 100644 --- a/dolphinscheduler-python/pydolphinscheduler/requirements_dev.txt +++ b/dolphinscheduler-python/pydolphinscheduler/requirements_dev.txt @@ -24,3 +24,4 @@ coverage flake8 flake8-docstrings flake8-black +isort diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/base.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/base.py index ce71a4a064..d636d512e1 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/base.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/base.py @@ -17,7 +17,7 @@ """DolphinScheduler Base object.""" -from typing import Optional, Dict +from typing import Dict, Optional # from pydolphinscheduler.side.user import User from pydolphinscheduler.utils.string import attr2camel diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/process_definition.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/process_definition.py index 152119bf5d..e5769f518d 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/process_definition.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/process_definition.py @@ -19,16 +19,16 @@ import json from datetime import datetime -from typing import Optional, List, Dict, Set, Any +from typing import Any, Dict, List, Optional, Set from pydolphinscheduler.constants import ( - ProcessDefinitionReleaseState, ProcessDefinitionDefault, + ProcessDefinitionReleaseState, ) from pydolphinscheduler.core.base import Base from pydolphinscheduler.java_gateway import launch_gateway -from pydolphinscheduler.side import Tenant, Project, User -from pydolphinscheduler.utils.date import conv_from_str, conv_to_schedule, MAX_DATETIME +from pydolphinscheduler.side import Project, Tenant, User +from pydolphinscheduler.utils.date import MAX_DATETIME, conv_from_str, conv_to_schedule class ProcessDefinitionContext: diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py index 6f9e454ef0..c8eb54ad43 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py @@ -17,19 +17,21 @@ """DolphinScheduler ObjectJsonBase, TaskParams and Task object.""" -from typing import Optional, List, Dict, Set, Union, Sequence, Tuple +from typing import Dict, List, Optional, Sequence, Set, Tuple, Union from pydolphinscheduler.constants import ( - TaskPriority, ProcessDefinitionDefault, TaskFlag, + TaskPriority, TaskTimeoutFlag, ) from pydolphinscheduler.core.base import Base -from pydolphinscheduler.core.process_definition import ProcessDefinition -from pydolphinscheduler.core.process_definition import ProcessDefinitionContext +from pydolphinscheduler.core.process_definition import ( + ProcessDefinition, + ProcessDefinitionContext, +) from pydolphinscheduler.java_gateway import launch_gateway -from pydolphinscheduler.utils.string import snake2camel, class_name2camel +from pydolphinscheduler.utils.string import class_name2camel, snake2camel class ObjectJsonBase: diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/java_gateway.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/java_gateway.py index 027ca94bc4..f40bb363a6 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/java_gateway.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/java_gateway.py @@ -20,7 +20,7 @@ from typing import Any, Optional from py4j.java_collections import JavaMap -from py4j.java_gateway import JavaGateway, GatewayParameters +from py4j.java_gateway import GatewayParameters, JavaGateway from pydolphinscheduler.constants import JavaGatewayDefault diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/side/project.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/side/project.py index 96051a2941..02382cadc5 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/side/project.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/side/project.py @@ -19,8 +19,8 @@ from typing import Optional -from pydolphinscheduler.core.base_side import BaseSide from pydolphinscheduler.constants import ProcessDefinitionDefault +from pydolphinscheduler.core.base_side import BaseSide from pydolphinscheduler.java_gateway import launch_gateway diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/side/queue.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/side/queue.py index 720135186c..9d6664e14b 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/side/queue.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/side/queue.py @@ -21,7 +21,7 @@ from typing import Optional from pydolphinscheduler.constants import ProcessDefinitionDefault from pydolphinscheduler.core.base_side import BaseSide -from pydolphinscheduler.java_gateway import launch_gateway, gateway_result_checker +from pydolphinscheduler.java_gateway import gateway_result_checker, launch_gateway class Queue(BaseSide): diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/utils/date.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/utils/date.py index e2a4cd1cd4..18cf93e318 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/utils/date.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/utils/date.py @@ -18,6 +18,7 @@ """Date util function collections.""" from datetime import datetime + from pydolphinscheduler.constants import Delimiter, Time LEN_SUPPORT_DATETIME = ( diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_process_definition.py b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_process_definition.py index 930909a4fc..a678649041 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_process_definition.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_process_definition.py @@ -20,9 +20,8 @@ from datetime import datetime from typing import Any -from pydolphinscheduler.utils.date import conv_to_schedule - import pytest +from freezegun import freeze_time from pydolphinscheduler.constants import ( ProcessDefinitionDefault, @@ -30,9 +29,9 @@ from pydolphinscheduler.constants import ( ) from pydolphinscheduler.core.process_definition import ProcessDefinition from pydolphinscheduler.core.task import TaskParams -from pydolphinscheduler.side import Tenant, Project, User +from pydolphinscheduler.side import Project, Tenant, User +from pydolphinscheduler.utils.date import conv_to_schedule from tests.testing.task import Task -from freezegun import freeze_time TEST_PROCESS_DEFINITION_NAME = "simple-test-process-definition" diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py index 46103375e7..63d6496672 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py @@ -18,9 +18,10 @@ """Test Task class function.""" from unittest.mock import patch + import pytest -from pydolphinscheduler.core.task import TaskParams, TaskRelation, Task +from pydolphinscheduler.core.task import Task, TaskParams, TaskRelation from tests.testing.task import Task as testTask diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/test_java_gateway.py b/dolphinscheduler-python/pydolphinscheduler/tests/test_java_gateway.py index d0456a6af2..3c8831e16d 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/test_java_gateway.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/test_java_gateway.py @@ -18,7 +18,7 @@ """Test pydolphinscheduler java gateway.""" -from py4j.java_gateway import java_import, JavaGateway +from py4j.java_gateway import JavaGateway, java_import def test_gateway_connect(): diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/utils/test_date.py b/dolphinscheduler-python/pydolphinscheduler/tests/utils/test_date.py index 648f2c40a8..bb204c9294 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/utils/test_date.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/utils/test_date.py @@ -17,13 +17,11 @@ """Test utils.date module.""" -import pytest from datetime import datetime -from pydolphinscheduler.utils.date import ( - conv_from_str, - conv_to_schedule, - FMT_STD, -) + +import pytest + +from pydolphinscheduler.utils.date import FMT_STD, conv_from_str, conv_to_schedule curr_date = datetime.now() diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/utils/test_string.py b/dolphinscheduler-python/pydolphinscheduler/tests/utils/test_string.py index 6942d7e75e..2ccd206df1 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/utils/test_string.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/utils/test_string.py @@ -17,9 +17,10 @@ """Test utils.string module.""" -from pydolphinscheduler.utils.string import attr2camel, snake2camel, class_name2camel import pytest +from pydolphinscheduler.utils.string import attr2camel, class_name2camel, snake2camel + @pytest.mark.parametrize( "snake, expect",