@ -14,36 +14,46 @@
# KIND, either express or implied. See the License for the # KIND, either express or implied. See the License for the
# specific language governing permissions and limitations # specific language governing permissions and limitations
# under the License. # under the License.
""" Module process definition, core class for workflow define. """
import json import json
from typing import Optional , List , Dict , Set from typing import Optional , List , Dict , Set
from pydolphinscheduler . constants import ProcessDefinitionReleaseState , ProcessDefinitionDefault from pydolphinscheduler . constants import (
ProcessDefinitionReleaseState ,
ProcessDefinitionDefault ,
)
from pydolphinscheduler . core . base import Base from pydolphinscheduler . core . base import Base
from pydolphinscheduler . java_gateway import launch_gateway from pydolphinscheduler . java_gateway import launch_gateway
from pydolphinscheduler . side import Tenant , Project , User from pydolphinscheduler . side import Tenant , Project , User
class ProcessDefinitionContext : class ProcessDefinitionContext :
""" Class process definition context, use when task get process definition from context expression. """
_context_managed_process_definition : Optional [ " ProcessDefinition " ] = None _context_managed_process_definition : Optional [ " ProcessDefinition " ] = None
@classmethod @classmethod
def set ( cls , pd : " ProcessDefinition " ) - > None : def set ( cls , pd : " ProcessDefinition " ) - > None :
""" Set attribute self._context_managed_process_definition. """
cls . _context_managed_process_definition = pd cls . _context_managed_process_definition = pd
@classmethod @classmethod
def get ( cls ) - > Optional [ " ProcessDefinition " ] : def get ( cls ) - > Optional [ " ProcessDefinition " ] :
""" Get attribute self._context_managed_process_definition. """
return cls . _context_managed_process_definition return cls . _context_managed_process_definition
@classmethod @classmethod
def delete ( cls ) - > None : def delete ( cls ) - > None :
""" Delete attribute self._context_managed_process_definition. """
cls . _context_managed_process_definition = None cls . _context_managed_process_definition = None
class ProcessDefinition ( Base ) : class ProcessDefinition ( Base ) :
""" """ process definition object, will define process definition attribute, task, relation.
ProcessDefinition
TODO : ref : comment may not correct ref TODO : maybe we should rename this class , currently use DS object name .
TODO : maybe we should rename this class , currently use DS object name
""" """
# key attribute for identify ProcessDefinition object # key attribute for identify ProcessDefinition object
@ -70,17 +80,17 @@ class ProcessDefinition(Base):
} }
def __init__ ( def __init__ (
self , self ,
name : str , name : str ,
description : Optional [ str ] = None , description : Optional [ str ] = None ,
user : Optional [ str ] = ProcessDefinitionDefault . USER , user : Optional [ str ] = ProcessDefinitionDefault . USER ,
project : Optional [ str ] = ProcessDefinitionDefault . PROJECT , project : Optional [ str ] = ProcessDefinitionDefault . PROJECT ,
tenant : Optional [ str ] = ProcessDefinitionDefault . TENANT , tenant : Optional [ str ] = ProcessDefinitionDefault . TENANT ,
queue : Optional [ str ] = ProcessDefinitionDefault . QUEUE , queue : Optional [ str ] = ProcessDefinitionDefault . QUEUE ,
worker_group : Optional [ str ] = ProcessDefinitionDefault . WORKER_GROUP , worker_group : Optional [ str ] = ProcessDefinitionDefault . WORKER_GROUP ,
timeout : Optional [ int ] = 0 , timeout : Optional [ int ] = 0 ,
release_state : Optional [ str ] = ProcessDefinitionReleaseState . ONLINE , release_state : Optional [ str ] = ProcessDefinitionReleaseState . ONLINE ,
param : Optional [ List ] = None param : Optional [ List ] = None ,
) : ) :
super ( ) . __init__ ( name , description ) super ( ) . __init__ ( name , description )
self . _user = user self . _user = user
@ -93,7 +103,7 @@ class ProcessDefinition(Base):
self . param = param self . param = param
self . tasks : dict = { } self . tasks : dict = { }
# TODO how to fix circle import # TODO how to fix circle import
self . _task_relations : set [ " TaskRelation " ] = set ( ) self . _task_relations : set [ " TaskRelation " ] = set ( ) # noqa: F821
self . _process_definition_code = None self . _process_definition_code = None
def __enter__ ( self ) - > " ProcessDefinition " : def __enter__ ( self ) - > " ProcessDefinition " :
@ -105,32 +115,43 @@ class ProcessDefinition(Base):
@property @property
def tenant ( self ) - > Tenant : def tenant ( self ) - > Tenant :
""" Get attribute tenant. """
return Tenant ( self . _tenant ) return Tenant ( self . _tenant )
@tenant . setter @tenant . setter
def tenant ( self , tenant : Tenant ) - > None : def tenant ( self , tenant : Tenant ) - > None :
""" Set attribute tenant. """
self . _tenant = tenant . name self . _tenant = tenant . name
@property @property
def project ( self ) - > Project : def project ( self ) - > Project :
""" Get attribute project. """
return Project ( self . _project ) return Project ( self . _project )
@project . setter @project . setter
def project ( self , project : Project ) - > None : def project ( self , project : Project ) - > None :
""" Set attribute project. """
self . _project = project . name self . _project = project . name
@property @property
def user ( self ) - > User : def user ( self ) - > User :
return User ( self . _user , """ Get user object.
ProcessDefinitionDefault . USER_PWD ,
ProcessDefinitionDefault . USER_EMAIL , For now we just get from python side but not from java gateway side , so it may not correct .
ProcessDefinitionDefault . USER_PHONE , """
ProcessDefinitionDefault . TENANT , return User (
ProcessDefinitionDefault . QUEUE , self . _user ,
ProcessDefinitionDefault . USER_STATE ) ProcessDefinitionDefault . USER_PWD ,
ProcessDefinitionDefault . USER_EMAIL ,
ProcessDefinitionDefault . USER_PHONE ,
ProcessDefinitionDefault . TENANT ,
ProcessDefinitionDefault . QUEUE ,
ProcessDefinitionDefault . USER_STATE ,
)
@property @property
def task_definition_json ( self ) - > List [ Dict ] : def task_definition_json ( self ) - > List [ Dict ] :
""" Return all tasks definition in list of dict. """
if not self . tasks : if not self . tasks :
return [ self . tasks ] return [ self . tasks ]
else : else :
@ -138,26 +159,39 @@ class ProcessDefinition(Base):
@property @property
def task_relation_json ( self ) - > List [ Dict ] : def task_relation_json ( self ) - > List [ Dict ] :
""" Return all relation between tasks pair in list of dict. """
if not self . tasks : if not self . tasks :
return [ self . tasks ] return [ self . tasks ]
else : else :
self . _handle_root_relation ( ) self . _handle_root_relation ( )
return [ tr . to_dict ( ) for tr in self . _task_relations ] return [ tr . to_dict ( ) for tr in self . _task_relations ]
# TODO inti DAG's tasks are in the same place # TODO inti DAG's tasks are in the same location with default {x: 0, y: 0}
@property @property
def task_location ( self ) - > List [ Dict ] : def task_location ( self ) - > List [ Dict ] :
""" Return all tasks location for all process definition.
For now , we only set all location with same x and y valued equal to 0. Because we do not
find a good way to set task locations . This is requests from java gateway interface .
"""
if not self . tasks : if not self . tasks :
return [ self . tasks ] return [ self . tasks ]
else : else :
return [ { " taskCode " : task_code , " x " : 0 , " y " : 0 } for task_code in self . tasks ] return [ { " taskCode " : task_code , " x " : 0 , " y " : 0 } for task_code in self . tasks ]
@property @property
def task_list ( self ) - > List [ " Task " ] : def task_list ( self ) - > List [ " Task " ] : # noqa: F821
""" Return list of tasks objects. """
return list ( self . tasks . values ( ) ) return list ( self . tasks . values ( ) )
def _handle_root_relation ( self ) : def _handle_root_relation ( self ) :
""" Handle root task property :class:`pydolphinscheduler.core.task.TaskRelation`.
Root task in DAG do not have dominant upstream node , but we have to add an exactly default
upstream task with task_code equal to ` 0 ` . This is requests from java gateway interface .
"""
from pydolphinscheduler . core . task import TaskRelation from pydolphinscheduler . core . task import TaskRelation
post_relation_code = set ( ) post_relation_code = set ( )
for relation in self . _task_relations : for relation in self . _task_relations :
post_relation_code . add ( relation . post_task_code ) post_relation_code . add ( relation . post_task_code )
@ -166,46 +200,62 @@ class ProcessDefinition(Base):
root_relation = TaskRelation ( pre_task_code = 0 , post_task_code = task . code ) root_relation = TaskRelation ( pre_task_code = 0 , post_task_code = task . code )
self . _task_relations . add ( root_relation ) self . _task_relations . add ( root_relation )
def add_task ( self , task : " Task " ) - > None : def add_task ( self , task : " Task " ) - > None : # noqa: F821
""" Add a single task to process definition. """
self . tasks [ task . code ] = task self . tasks [ task . code ] = task
task . _process_definition = self task . _process_definition = self
def add_tasks ( self , tasks : List [ " Task " ] ) - > None : def add_tasks ( self , tasks : List [ " Task " ] ) - > None : # noqa: F821
""" Add task sequence to process definition, it a wrapper of :func:`add_task`. """
for task in tasks : for task in tasks :
self . add_task ( task ) self . add_task ( task )
def get_task ( self , code : str ) - > " Task " : def get_task ( self , code : str ) - > " Task " : # noqa: F821
""" Get task object from process definition by given code. """
if code not in self . tasks : if code not in self . tasks :
raise ValueError ( " Task with code %s can not found in process definition % " , ( code , self . name ) ) raise ValueError (
" Task with code %s can not found in process definition % " ,
( code , self . name ) ,
)
return self . tasks [ code ] return self . tasks [ code ]
# TODO which tying should return in this case # TODO which tying should return in this case
def get_tasks_by_name ( self , name : str ) - > Set [ " Task " ] : def get_tasks_by_name ( self , name : str ) - > Set [ " Task " ] : # noqa: F821
""" Get tasks object by given name, if will return all tasks with this name. """
find = set ( ) find = set ( )
for task in self . tasks . values ( ) : for task in self . tasks . values ( ) :
if task . name == name : if task . name == name :
find . add ( task ) find . add ( task )
return find return find
def get_one_task_by_name ( self , name : str ) - > " Task " : def get_one_task_by_name ( self , name : str ) - > " Task " : # noqa: F821
""" Get exact one task from process definition by given name.
Function always return one task even though this process definition have more than one task with
this name .
"""
tasks = self . get_tasks_by_name ( name ) tasks = self . get_tasks_by_name ( name )
if not tasks : if not tasks :
raise ValueError ( f " Can not find task with name { name } . " ) raise ValueError ( f " Can not find task with name { name } . " )
return tasks . pop ( ) return tasks . pop ( )
def run ( self ) : def run ( self ) :
""" """ Submit and Start ProcessDefinition instance.
Run ProcessDefinition instance , a shortcut for : ref : submit and : ref : start
Only support manual for now , schedule run will coming soon Shortcut for function : func : ` submit ` and function : func : ` start ` . Only support manual start workflow
for now , and schedule run will coming soon .
: return : : return :
""" """
self . submit ( ) self . submit ( )
self . start ( ) self . start ( )
def _ensure_side_model_exists ( self ) : def _ensure_side_model_exists ( self ) :
""" """ Ensure process definition side model exists.
Ensure side model exists which including : ref : Project , Tenant , User .
If those model not exists , would create default value in : ref : ProcessDefinitionDefault For now , side object including : class : ` pydolphinscheduler . side . project . Project ` ,
: class : ` pydolphinscheduler . side . tenant . Tenant ` , : class : ` pydolphinscheduler . side . user . User ` .
If these model not exists , would create default value in
: class : ` pydolphinscheduler . constants . ProcessDefinitionDefault ` .
""" """
# TODO used metaclass for more pythonic # TODO used metaclass for more pythonic
self . tenant . create_if_not_exists ( self . _queue ) self . tenant . create_if_not_exists ( self . _queue )
@ -215,10 +265,7 @@ class ProcessDefinition(Base):
self . project . create_if_not_exists ( self . _user ) self . project . create_if_not_exists ( self . _user )
def submit ( self ) - > int : def submit ( self ) - > int :
""" """ Submit ProcessDefinition instance to java gateway. """
Submit ProcessDefinition instance to java gateway
: return :
"""
self . _ensure_side_model_exists ( ) self . _ensure_side_model_exists ( )
gateway = launch_gateway ( ) gateway = launch_gateway ( )
self . _process_definition_code = gateway . entry_point . createOrUpdateProcessDefinition ( self . _process_definition_code = gateway . entry_point . createOrUpdateProcessDefinition (
@ -238,9 +285,9 @@ class ProcessDefinition(Base):
return self . _process_definition_code return self . _process_definition_code
def start ( self ) - > None : def start ( self ) - > None :
""" """ Create and start ProcessDefinition instance.
Start ProcessDefinition instance which post to ` start - process - instance ` to java gateway
: return : which post to ` start - process - instance ` to java gateway
""" """
gateway = launch_gateway ( ) gateway = launch_gateway ( )
gateway . entry_point . execProcessInstance ( gateway . entry_point . execProcessInstance (
Jiajie Zhong
3 years ago
committed by
GitHub