Browse Source

[python] Refactor structure to avoid cycle import (#11167)

Currently, our core and side module dependent on each others. and will cause
cycle import in our codebase, especially in issue #10905, we try to refactor
to solve this problem.

This patch do the following change:

* Rename module `side` to `models`
* Move `core/base` and `core/sidebase` to dir `modules`
* Move `configuration` and `default_config.yaml` to the root of pydolphinscheduler
3.1.0-release
Jiajie Zhong 2 years ago committed by GitHub
parent
commit
7a766cbcf2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 2
      dolphinscheduler-python/pydolphinscheduler/DEVELOP.md
  2. 2
      dolphinscheduler-python/pydolphinscheduler/docs/source/api.rst
  3. 2
      dolphinscheduler-python/pydolphinscheduler/docs/source/config.rst
  4. 2
      dolphinscheduler-python/pydolphinscheduler/setup.py
  5. 2
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/cli/commands.py
  6. 0
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/configuration.py
  7. 2
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/__init__.py
  8. 13
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/process_definition.py
  9. 2
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/resource.py
  10. 4
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py
  11. 2
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/default_config.yaml
  12. 2
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_dependent_example.py
  13. 2
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/java_gateway.py
  14. 14
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/__init__.py
  15. 2
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/base.py
  16. 8
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/base_side.py
  17. 4
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/project.py
  18. 4
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/queue.py
  19. 4
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/tenant.py
  20. 5
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/user.py
  21. 2
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/worker_group.py
  22. 2
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/tasks/condition.py
  23. 6
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/tasks/dependent.py
  24. 2
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/tasks/switch.py
  25. 2
      dolphinscheduler-python/pydolphinscheduler/tests/cli/test_config.py
  26. 4
      dolphinscheduler-python/pydolphinscheduler/tests/core/test_configuration.py
  27. 4
      dolphinscheduler-python/pydolphinscheduler/tests/core/test_process_definition.py
  28. 2
      dolphinscheduler-python/pydolphinscheduler/tests/testing/path.py

2
dolphinscheduler-python/pydolphinscheduler/DEVELOP.md

@ -34,7 +34,7 @@ Now, we should install all dependence to make sure we could run test or check co
```shell
cd dolphinscheduler/dolphinscheduler-python/pydolphinscheduler
python -m pip install .[dev]
python -m pip install -e '.[dev]'
```
Next, we have to open pydolphinscheduler project in you editor. We recommend you use [pycharm][pycharm]

2
dolphinscheduler-python/pydolphinscheduler/docs/source/api.rst

@ -27,7 +27,7 @@ Core
Sides
-----
.. automodule:: pydolphinscheduler.side
.. automodule:: pydolphinscheduler.models
:inherited-members:
Tasks

2
dolphinscheduler-python/pydolphinscheduler/docs/source/config.rst

@ -200,7 +200,7 @@ All Configurations in File
Here are all our configurations for pydolphinscheduler.
.. literalinclude:: ../../src/pydolphinscheduler/core/default_config.yaml
.. literalinclude:: ../../src/pydolphinscheduler/default_config.yaml
:language: yaml
:lines: 18-

2
dolphinscheduler-python/pydolphinscheduler/setup.py

@ -147,7 +147,7 @@ setup(
package_dir={"": "src"},
include_package_data=True,
package_data={
"pydolphinscheduler": ["core/default_config.yaml"],
"pydolphinscheduler": ["default_config.yaml"],
},
platforms=["any"],
classifiers=[

2
dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/cli/commands.py

@ -21,7 +21,7 @@ import click
from click import echo
from pydolphinscheduler import __version__
from pydolphinscheduler.core.configuration import (
from pydolphinscheduler.configuration import (
get_single_config,
init_config_file,
set_single_config,

0
dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/configuration.py → dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/configuration.py

2
dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/__init__.py

@ -23,8 +23,8 @@ from pydolphinscheduler.core.process_definition import ProcessDefinition
from pydolphinscheduler.core.task import Task
__all__ = [
"Database",
"Engine",
"ProcessDefinition",
"Task",
"Database",
]

13
dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/process_definition.py

@ -21,12 +21,11 @@ import json
from datetime import datetime
from typing import Any, Dict, List, Optional, Set
from pydolphinscheduler import configuration
from pydolphinscheduler.constants import TaskType
from pydolphinscheduler.core import configuration
from pydolphinscheduler.core.base import Base
from pydolphinscheduler.exceptions import PyDSParamException, PyDSTaskNoFoundException
from pydolphinscheduler.java_gateway import launch_gateway
from pydolphinscheduler.side import Project, Tenant, User
from pydolphinscheduler.models import Base, Project, Tenant, User
from pydolphinscheduler.utils.date import MAX_DATETIME, conv_from_str, conv_to_schedule
@ -170,7 +169,7 @@ class ProcessDefinition(Base):
def user(self) -> User:
"""Get user object.
For now we just get from python side but not from java gateway side, so it may not correct.
For now we just get from python models but not from java gateway models, so it may not correct.
"""
return User(name=self._user, tenant=self._tenant)
@ -358,10 +357,10 @@ class ProcessDefinition(Base):
self.start()
def _ensure_side_model_exists(self):
"""Ensure process definition side model exists.
"""Ensure process definition models model exists.
For now, side object including :class:`pydolphinscheduler.side.project.Project`,
:class:`pydolphinscheduler.side.tenant.Tenant`, :class:`pydolphinscheduler.side.user.User`.
For now, models object including :class:`pydolphinscheduler.models.project.Project`,
:class:`pydolphinscheduler.models.tenant.Tenant`, :class:`pydolphinscheduler.models.user.User`.
If these model not exists, would create default value in
:class:`pydolphinscheduler.constants.ProcessDefinitionDefault`.
"""

2
dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/resource.py

@ -19,7 +19,7 @@
from typing import Optional
from pydolphinscheduler.core.base import Base
from pydolphinscheduler.models import Base
class Resource(Base):

4
dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py

@ -20,6 +20,7 @@
from logging import getLogger
from typing import Dict, List, Optional, Sequence, Set, Tuple, Union
from pydolphinscheduler import configuration
from pydolphinscheduler.constants import (
Delimiter,
ResourceKey,
@ -27,13 +28,12 @@ from pydolphinscheduler.constants import (
TaskPriority,
TaskTimeoutFlag,
)
from pydolphinscheduler.core import configuration
from pydolphinscheduler.core.base import Base
from pydolphinscheduler.core.process_definition import (
ProcessDefinition,
ProcessDefinitionContext,
)
from pydolphinscheduler.java_gateway import launch_gateway
from pydolphinscheduler.models import Base
logger = getLogger(__name__)

2
dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/default_config.yaml → dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/default_config.yaml

@ -22,7 +22,7 @@ java_gateway:
address: 127.0.0.1
# The port of Python gateway server start. Define which port you could connect to Python gateway server from
# Python API side.
# Python API models.
port: 25333
# Whether automatically convert Python objects to Java Objects. Default value is ``True``. There is some

2
dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_dependent_example.py

@ -35,7 +35,7 @@ task_dependent:
task_dependent(this task dependent on task_dependent_external.task_1 and task_dependent_external.task_2).
"""
from pydolphinscheduler.core import configuration
from pydolphinscheduler import configuration
from pydolphinscheduler.core.process_definition import ProcessDefinition
from pydolphinscheduler.tasks.dependent import And, Dependent, DependentItem, Or
from pydolphinscheduler.tasks.shell import Shell

2
dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/java_gateway.py

@ -22,8 +22,8 @@ from typing import Any, Optional
from py4j.java_collections import JavaMap
from py4j.java_gateway import GatewayParameters, JavaGateway
from pydolphinscheduler import configuration
from pydolphinscheduler.constants import JavaGatewayDefault
from pydolphinscheduler.core import configuration
from pydolphinscheduler.exceptions import PyDSJavaGatewayException

14
dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/side/__init__.py → dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/__init__.py

@ -17,13 +17,17 @@
"""Init Side package, Side package keep object related to DolphinScheduler but not in the Core part."""
from pydolphinscheduler.side.project import Project
from pydolphinscheduler.side.queue import Queue
from pydolphinscheduler.side.tenant import Tenant
from pydolphinscheduler.side.user import User
from pydolphinscheduler.side.worker_group import WorkerGroup
from pydolphinscheduler.models.base import Base
from pydolphinscheduler.models.base_side import BaseSide
from pydolphinscheduler.models.project import Project
from pydolphinscheduler.models.queue import Queue
from pydolphinscheduler.models.tenant import Tenant
from pydolphinscheduler.models.user import User
from pydolphinscheduler.models.worker_group import WorkerGroup
__all__ = [
"Base",
"BaseSide",
"Project",
"Tenant",
"User",

2
dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/base.py → dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/base.py

@ -19,7 +19,7 @@
from typing import Dict, Optional
# from pydolphinscheduler.side.user import User
# from pydolphinscheduler.models.user import User
from pydolphinscheduler.utils.string import attr2camel

8
dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/base_side.py → dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/base_side.py

@ -15,16 +15,16 @@
# specific language governing permissions and limitations
# under the License.
"""Module for side object."""
"""Module for models object."""
from typing import Optional
from pydolphinscheduler.core import configuration
from pydolphinscheduler.core.base import Base
from pydolphinscheduler import configuration
from pydolphinscheduler.models import Base
class BaseSide(Base):
"""Base class for side object, it declare base behavior for them."""
"""Base class for models object, it declare base behavior for them."""
def __init__(self, name: str, description: Optional[str] = None):
super().__init__(name, description)

4
dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/side/project.py → dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/project.py

@ -19,9 +19,9 @@
from typing import Optional
from pydolphinscheduler.core import configuration
from pydolphinscheduler.core.base_side import BaseSide
from pydolphinscheduler import configuration
from pydolphinscheduler.java_gateway import launch_gateway
from pydolphinscheduler.models import BaseSide
class Project(BaseSide):

4
dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/side/queue.py → dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/queue.py

@ -19,9 +19,9 @@
from typing import Optional
from pydolphinscheduler.core import configuration
from pydolphinscheduler.core.base_side import BaseSide
from pydolphinscheduler import configuration
from pydolphinscheduler.java_gateway import gateway_result_checker, launch_gateway
from pydolphinscheduler.models import BaseSide
class Queue(BaseSide):

4
dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/side/tenant.py → dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/tenant.py

@ -19,9 +19,9 @@
from typing import Optional
from pydolphinscheduler.core import configuration
from pydolphinscheduler.core.base_side import BaseSide
from pydolphinscheduler import configuration
from pydolphinscheduler.java_gateway import launch_gateway
from pydolphinscheduler.models import BaseSide
class Tenant(BaseSide):

5
dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/side/user.py → dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/user.py

@ -19,10 +19,9 @@
from typing import Optional
from pydolphinscheduler.core import configuration
from pydolphinscheduler.core.base_side import BaseSide
from pydolphinscheduler import configuration
from pydolphinscheduler.java_gateway import launch_gateway
from pydolphinscheduler.side.tenant import Tenant
from pydolphinscheduler.models import BaseSide, Tenant
class User(BaseSide):

2
dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/side/worker_group.py → dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/worker_group.py

@ -19,7 +19,7 @@
from typing import Optional
from pydolphinscheduler.core.base_side import BaseSide
from pydolphinscheduler.models import BaseSide
class WorkerGroup(BaseSide):

2
dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/tasks/condition.py

@ -20,9 +20,9 @@
from typing import Dict, List
from pydolphinscheduler.constants import TaskType
from pydolphinscheduler.core.base import Base
from pydolphinscheduler.core.task import Task
from pydolphinscheduler.exceptions import PyDSParamException
from pydolphinscheduler.models.base import Base
class Status(Base):

6
dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/tasks/dependent.py

@ -20,10 +20,10 @@
from typing import Dict, Optional, Tuple
from pydolphinscheduler.constants import TaskType
from pydolphinscheduler.core.base import Base
from pydolphinscheduler.core.task import Task
from pydolphinscheduler.exceptions import PyDSJavaGatewayException, PyDSParamException
from pydolphinscheduler.java_gateway import launch_gateway
from pydolphinscheduler.models.base import Base
DEPENDENT_ALL_TASK_IN_WORKFLOW = "0"
@ -31,8 +31,8 @@ DEPENDENT_ALL_TASK_IN_WORKFLOW = "0"
class DependentDate(str):
"""Constant of Dependent date value.
These values set according to Java server side, if you want to add and change it,
please change Java server side first.
These values set according to Java server models, if you want to add and change it,
please change Java server models first.
"""
# TODO Maybe we should add parent level to DependentDate for easy to use, such as

2
dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/tasks/switch.py

@ -20,9 +20,9 @@
from typing import Dict, Optional
from pydolphinscheduler.constants import TaskType
from pydolphinscheduler.core.base import Base
from pydolphinscheduler.core.task import Task
from pydolphinscheduler.exceptions import PyDSParamException
from pydolphinscheduler.models.base import Base
class SwitchBranch(Base):

2
dolphinscheduler-python/pydolphinscheduler/tests/cli/test_config.py

@ -23,7 +23,7 @@ from pathlib import Path
import pytest
from pydolphinscheduler.cli.commands import cli
from pydolphinscheduler.core.configuration import BUILD_IN_CONFIG_PATH, config_path
from pydolphinscheduler.configuration import BUILD_IN_CONFIG_PATH, config_path
from tests.testing.cli import CliTestWrapper
from tests.testing.constants import DEV_MODE, ENV_PYDS_HOME
from tests.testing.file import get_file_content

4
dolphinscheduler-python/pydolphinscheduler/tests/core/test_configuration.py

@ -24,8 +24,8 @@ from typing import Any
import pytest
from pydolphinscheduler.core import configuration
from pydolphinscheduler.core.configuration import (
from pydolphinscheduler import configuration
from pydolphinscheduler.configuration import (
BUILD_IN_CONFIG_PATH,
config_path,
get_single_config,

4
dolphinscheduler-python/pydolphinscheduler/tests/core/test_process_definition.py

@ -24,11 +24,11 @@ from unittest.mock import patch
import pytest
from freezegun import freeze_time
from pydolphinscheduler.core import configuration
from pydolphinscheduler import configuration
from pydolphinscheduler.core.process_definition import ProcessDefinition
from pydolphinscheduler.core.resource import Resource
from pydolphinscheduler.exceptions import PyDSParamException
from pydolphinscheduler.side import Project, Tenant, User
from pydolphinscheduler.models import Project, Tenant, User
from pydolphinscheduler.tasks.switch import Branch, Default, Switch, SwitchCondition
from pydolphinscheduler.utils.date import conv_to_schedule
from tests.testing.task import Task

2
dolphinscheduler-python/pydolphinscheduler/tests/testing/path.py

@ -26,7 +26,7 @@ path_code_tasks = project_root.joinpath("src", "pydolphinscheduler", "tasks")
path_example = project_root.joinpath("src", "pydolphinscheduler", "examples")
path_doc_tasks = project_root.joinpath("docs", "source", "tasks")
path_default_config_yaml = project_root.joinpath(
"src", "pydolphinscheduler", "core", "default_config.yaml"
"src", "pydolphinscheduler", "default_config.yaml"
)

Loading…
Cancel
Save