Browse Source

[python] Fix task relation wrong hashable function (#7628)

* [python] Fix task relation wrong hashable function

Currently our :class:`TaskRelation` have wrong __hash__
function, would mistake a object as same object. It failed
our dependence and process definition. this patch fix it
by adding `_KEY_ATTR` to overwrite TaskRelation's __eq__
and correct __hash__ func

close: #7627

* Fix code format
3.0.0/version-upgrade
Jiajie Zhong 3 years ago committed by GitHub
parent
commit
b1afd99c9e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 1
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/constants.py
  2. 12
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py
  3. 46
      dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py

1
dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/constants.py

@ -104,6 +104,7 @@ class Delimiter(str):
DASH = "/" DASH = "/"
COLON = ":" COLON = ":"
UNDERSCORE = "_" UNDERSCORE = "_"
DIRECTION = "->"
class Time(str): class Time(str):

12
dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py

@ -21,6 +21,7 @@ import logging
from typing import Dict, List, Optional, Sequence, Set, Tuple, Union from typing import Dict, List, Optional, Sequence, Set, Tuple, Union
from pydolphinscheduler.constants import ( from pydolphinscheduler.constants import (
Delimiter,
ProcessDefinitionDefault, ProcessDefinitionDefault,
TaskFlag, TaskFlag,
TaskPriority, TaskPriority,
@ -37,6 +38,13 @@ from pydolphinscheduler.java_gateway import launch_gateway
class TaskRelation(Base): class TaskRelation(Base):
"""TaskRelation object, describe the relation of exactly two tasks.""" """TaskRelation object, describe the relation of exactly two tasks."""
# Add attr `_KEY_ATTR` to overwrite :func:`__eq__`, it is make set
# `Task.process_definition._task_relations` work correctly.
_KEY_ATTR = {
"pre_task_code",
"post_task_code",
}
_DEFINE_ATTR = { _DEFINE_ATTR = {
"pre_task_code", "pre_task_code",
"post_task_code", "post_task_code",
@ -61,7 +69,7 @@ class TaskRelation(Base):
self.post_task_code = post_task_code self.post_task_code = post_task_code
def __hash__(self): def __hash__(self):
return hash(f"{self.post_task_code}, {self.post_task_code}") return hash(f"{self.pre_task_code} {Delimiter.DIRECTION} {self.post_task_code}")
class Task(Base): class Task(Base):
@ -219,6 +227,7 @@ class Task(Base):
task_relation = TaskRelation( task_relation = TaskRelation(
pre_task_code=task.code, pre_task_code=task.code,
post_task_code=self.code, post_task_code=self.code,
name=f"{task.name} {Delimiter.DIRECTION} {self.name}",
) )
self.process_definition._task_relations.add(task_relation) self.process_definition._task_relations.add(task_relation)
else: else:
@ -229,6 +238,7 @@ class Task(Base):
task_relation = TaskRelation( task_relation = TaskRelation(
pre_task_code=self.code, pre_task_code=self.code,
post_task_code=task.code, post_task_code=task.code,
name=f"{self.name} {Delimiter.DIRECTION} {task.name}",
) )
self.process_definition._task_relations.add(task_relation) self.process_definition._task_relations.add(task_relation)

46
dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py

@ -24,6 +24,9 @@ import pytest
from pydolphinscheduler.core.task import Task, TaskRelation from pydolphinscheduler.core.task import Task, TaskRelation
from tests.testing.task import Task as testTask from tests.testing.task import Task as testTask
TEST_TASK_RELATION_SET = set()
TEST_TASK_RELATION_SIZE = 0
@pytest.mark.parametrize( @pytest.mark.parametrize(
"attr, expect", "attr, expect",
@ -66,6 +69,45 @@ def test_property_task_params(attr, expect):
assert expect == task.task_params assert expect == task.task_params
@pytest.mark.parametrize(
"pre_code, post_code, expect",
[
(123, 456, hash("123 -> 456")),
(12345678, 987654321, hash("12345678 -> 987654321")),
],
)
def test_task_relation_hash_func(pre_code, post_code, expect):
"""Test TaskRelation magic function :func:`__hash__`."""
task_param = TaskRelation(pre_task_code=pre_code, post_task_code=post_code)
assert hash(task_param) == expect
@pytest.mark.parametrize(
"pre_code, post_code, size_add",
[
(123, 456, 1),
(123, 456, 0),
(456, 456, 1),
(123, 123, 1),
(456, 123, 1),
(0, 456, 1),
(123, 0, 1),
],
)
def test_task_relation_add_to_set(pre_code, post_code, size_add):
"""Test TaskRelation with different pre_code and post_code add to set behavior.
Here we use global variable to keep set of :class:`TaskRelation` instance and the number we expect
of the size when we add a new task relation to exists set.
"""
task_relation = TaskRelation(pre_task_code=pre_code, post_task_code=post_code)
TEST_TASK_RELATION_SET.add(task_relation)
# hint python interpreter use global variable instead of local's
global TEST_TASK_RELATION_SIZE
TEST_TASK_RELATION_SIZE += size_add
assert len(TEST_TASK_RELATION_SET) == TEST_TASK_RELATION_SIZE
def test_task_relation_to_dict(): def test_task_relation_to_dict():
"""Test TaskRelation object function to_dict.""" """Test TaskRelation object function to_dict."""
pre_task_code = 123 pre_task_code = 123
@ -79,10 +121,10 @@ def test_task_relation_to_dict():
"conditionType": 0, "conditionType": 0,
"conditionParams": {}, "conditionParams": {},
} }
task_param = TaskRelation( task_relation = TaskRelation(
pre_task_code=pre_task_code, post_task_code=post_task_code pre_task_code=pre_task_code, post_task_code=post_task_code
) )
assert task_param.get_define() == expect assert task_relation.get_define() == expect
def test_task_get_define(): def test_task_get_define():

Loading…
Cancel
Save