Browse Source

[ci][python] Add isort to sort out import (#6871)

* Add isort config file to fix conflict with black
* Add some doc about isort
3.0.0/version-upgrade
Jiajie Zhong 3 years ago committed by GitHub
parent
commit
7c5c7ec907
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 2
      .github/workflows/py-ci.yml
  2. 19
      dolphinscheduler-python/pydolphinscheduler/.isort.cfg
  3. 18
      dolphinscheduler-python/pydolphinscheduler/README.md
  4. 1
      dolphinscheduler-python/pydolphinscheduler/examples/tutorial.py
  5. 1
      dolphinscheduler-python/pydolphinscheduler/requirements_dev.txt
  6. 2
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/base.py
  7. 8
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/process_definition.py
  8. 12
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py
  9. 2
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/java_gateway.py
  10. 2
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/side/project.py
  11. 2
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/side/queue.py
  12. 1
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/utils/date.py
  13. 7
      dolphinscheduler-python/pydolphinscheduler/tests/core/test_process_definition.py
  14. 3
      dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py
  15. 2
      dolphinscheduler-python/pydolphinscheduler/tests/test_java_gateway.py
  16. 10
      dolphinscheduler-python/pydolphinscheduler/tests/utils/test_date.py
  17. 3
      dolphinscheduler-python/pydolphinscheduler/tests/utils/test_string.py

2
.github/workflows/py-ci.yml

@ -50,6 +50,8 @@ jobs:
python-version: 3.7
- name: Install Development Dependences
run: pip install -r requirements_dev.txt
- name: Run Isort Checking
run: isort --check .
- name: Run Black Checking
run: black --check .
- name: Run Flake8 Checking

19
dolphinscheduler-python/pydolphinscheduler/.isort.cfg

@ -0,0 +1,19 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
[settings]
profile=black

18
dolphinscheduler-python/pydolphinscheduler/README.md

@ -21,6 +21,7 @@
[![GitHub Build][ga-py-test]][ga]
[![Code style: black][black-shield]][black-gh]
[![Imports: isort][isort-shield]][isort-gh]
pydolphinscheduler is python API for Apache DolphinScheduler, which allow you definition
your workflow by python code, aka workflow-as-codes.
@ -104,16 +105,20 @@ and would be implemented in the further.
### Code Style
We use [Black][black] for code formatter and [Flake8][flake8] for pep8 checker. If you use [pycharm][pycharm]
or [IntelliJ IDEA][idea], maybe you could follow [Black-integration][black-editor] to configure them in your environment.
We use [isort][isort] to automatically keep Python imports alphabetically, and use [Black][black] for code
formatter and [Flake8][flake8] for pep8 checker. If you use [pycharm][pycharm]or [IntelliJ IDEA][idea],
maybe you could follow [Black-integration][black-editor] to configure them in your environment.
Our Python API CI would automatically run unittest when you submit pull request in GitHub, you could also run
static check locally.
Our Python API CI would automatically run code style checker and unittest when you submit pull request in
GitHub, you could also run static check locally.
```shell
# We recommend you run Black before Flake8, because Black could auto fix some code style issue
# We recommend you run isort and Black before Flake8, because Black could auto fix some code style issue
# but Flake8 just hint when code style not match pep8
# Run Isort
isort .
# Run Black
black .
@ -158,8 +163,11 @@ this command output.
[flake8]: https://flake8.pycqa.org/en/latest/index.html
[black-editor]: https://black.readthedocs.io/en/stable/integrations/editors.html#pycharm-intellij-idea
[coverage]: https://coverage.readthedocs.io/en/stable/
[isort]: https://pycqa.github.io/isort/index.html
<!-- badge -->
[ga-py-test]: https://github.com/apache/dolphinscheduler/actions/workflows/py-ci.yml/badge.svg?branch=dev
[ga]: https://github.com/apache/dolphinscheduler/actions
[black-shield]: https://img.shields.io/badge/code%20style-black-000000.svg
[black-gh]: https://github.com/psf/black
[isort-shield]: https://img.shields.io/badge/%20imports-isort-%231674b1?style=flat&labelColor=ef8336
[isort-gh]: https://pycqa.github.io/isort/

1
dolphinscheduler-python/pydolphinscheduler/examples/tutorial.py

@ -33,7 +33,6 @@ it will instantiate and run all the task it have.
from pydolphinscheduler.core.process_definition import ProcessDefinition
from pydolphinscheduler.tasks.shell import Shell
with ProcessDefinition(
name="tutorial",
schedule="0 0 0 * * ? *",

1
dolphinscheduler-python/pydolphinscheduler/requirements_dev.txt

@ -24,3 +24,4 @@ coverage
flake8
flake8-docstrings
flake8-black
isort

2
dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/base.py

@ -17,7 +17,7 @@
"""DolphinScheduler Base object."""
from typing import Optional, Dict
from typing import Dict, Optional
# from pydolphinscheduler.side.user import User
from pydolphinscheduler.utils.string import attr2camel

8
dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/process_definition.py

@ -19,16 +19,16 @@
import json
from datetime import datetime
from typing import Optional, List, Dict, Set, Any
from typing import Any, Dict, List, Optional, Set
from pydolphinscheduler.constants import (
ProcessDefinitionReleaseState,
ProcessDefinitionDefault,
ProcessDefinitionReleaseState,
)
from pydolphinscheduler.core.base import Base
from pydolphinscheduler.java_gateway import launch_gateway
from pydolphinscheduler.side import Tenant, Project, User
from pydolphinscheduler.utils.date import conv_from_str, conv_to_schedule, MAX_DATETIME
from pydolphinscheduler.side import Project, Tenant, User
from pydolphinscheduler.utils.date import MAX_DATETIME, conv_from_str, conv_to_schedule
class ProcessDefinitionContext:

12
dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py

@ -17,19 +17,21 @@
"""DolphinScheduler ObjectJsonBase, TaskParams and Task object."""
from typing import Optional, List, Dict, Set, Union, Sequence, Tuple
from typing import Dict, List, Optional, Sequence, Set, Tuple, Union
from pydolphinscheduler.constants import (
TaskPriority,
ProcessDefinitionDefault,
TaskFlag,
TaskPriority,
TaskTimeoutFlag,
)
from pydolphinscheduler.core.base import Base
from pydolphinscheduler.core.process_definition import ProcessDefinition
from pydolphinscheduler.core.process_definition import ProcessDefinitionContext
from pydolphinscheduler.core.process_definition import (
ProcessDefinition,
ProcessDefinitionContext,
)
from pydolphinscheduler.java_gateway import launch_gateway
from pydolphinscheduler.utils.string import snake2camel, class_name2camel
from pydolphinscheduler.utils.string import class_name2camel, snake2camel
class ObjectJsonBase:

2
dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/java_gateway.py

@ -20,7 +20,7 @@
from typing import Any, Optional
from py4j.java_collections import JavaMap
from py4j.java_gateway import JavaGateway, GatewayParameters
from py4j.java_gateway import GatewayParameters, JavaGateway
from pydolphinscheduler.constants import JavaGatewayDefault

2
dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/side/project.py

@ -19,8 +19,8 @@
from typing import Optional
from pydolphinscheduler.core.base_side import BaseSide
from pydolphinscheduler.constants import ProcessDefinitionDefault
from pydolphinscheduler.core.base_side import BaseSide
from pydolphinscheduler.java_gateway import launch_gateway

2
dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/side/queue.py

@ -21,7 +21,7 @@ from typing import Optional
from pydolphinscheduler.constants import ProcessDefinitionDefault
from pydolphinscheduler.core.base_side import BaseSide
from pydolphinscheduler.java_gateway import launch_gateway, gateway_result_checker
from pydolphinscheduler.java_gateway import gateway_result_checker, launch_gateway
class Queue(BaseSide):

1
dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/utils/date.py

@ -18,6 +18,7 @@
"""Date util function collections."""
from datetime import datetime
from pydolphinscheduler.constants import Delimiter, Time
LEN_SUPPORT_DATETIME = (

7
dolphinscheduler-python/pydolphinscheduler/tests/core/test_process_definition.py

@ -20,9 +20,8 @@
from datetime import datetime
from typing import Any
from pydolphinscheduler.utils.date import conv_to_schedule
import pytest
from freezegun import freeze_time
from pydolphinscheduler.constants import (
ProcessDefinitionDefault,
@ -30,9 +29,9 @@ from pydolphinscheduler.constants import (
)
from pydolphinscheduler.core.process_definition import ProcessDefinition
from pydolphinscheduler.core.task import TaskParams
from pydolphinscheduler.side import Tenant, Project, User
from pydolphinscheduler.side import Project, Tenant, User
from pydolphinscheduler.utils.date import conv_to_schedule
from tests.testing.task import Task
from freezegun import freeze_time
TEST_PROCESS_DEFINITION_NAME = "simple-test-process-definition"

3
dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py

@ -18,9 +18,10 @@
"""Test Task class function."""
from unittest.mock import patch
import pytest
from pydolphinscheduler.core.task import TaskParams, TaskRelation, Task
from pydolphinscheduler.core.task import Task, TaskParams, TaskRelation
from tests.testing.task import Task as testTask

2
dolphinscheduler-python/pydolphinscheduler/tests/test_java_gateway.py

@ -18,7 +18,7 @@
"""Test pydolphinscheduler java gateway."""
from py4j.java_gateway import java_import, JavaGateway
from py4j.java_gateway import JavaGateway, java_import
def test_gateway_connect():

10
dolphinscheduler-python/pydolphinscheduler/tests/utils/test_date.py

@ -17,13 +17,11 @@
"""Test utils.date module."""
import pytest
from datetime import datetime
from pydolphinscheduler.utils.date import (
conv_from_str,
conv_to_schedule,
FMT_STD,
)
import pytest
from pydolphinscheduler.utils.date import FMT_STD, conv_from_str, conv_to_schedule
curr_date = datetime.now()

3
dolphinscheduler-python/pydolphinscheduler/tests/utils/test_string.py

@ -17,9 +17,10 @@
"""Test utils.string module."""
from pydolphinscheduler.utils.string import attr2camel, snake2camel, class_name2camel
import pytest
from pydolphinscheduler.utils.string import attr2camel, class_name2camel, snake2camel
@pytest.mark.parametrize(
"snake, expect",

Loading…
Cancel
Save