JieguangZhou
2 years ago
committed by
GitHub
39 changed files with 1606 additions and 1 deletions
@ -0,0 +1,43 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
# Define the workflow |
||||
workflow: |
||||
name: "Condition" |
||||
|
||||
# Define the tasks under the workflow |
||||
tasks: |
||||
- { "task_type": "Shell", "name": "pre_task_1", "command": "echo pre_task_1" } |
||||
- { "task_type": "Shell", "name": "pre_task_2", "command": "echo pre_task_2" } |
||||
- { "task_type": "Shell", "name": "pre_task_3", "command": "echo pre_task_3" } |
||||
- { "task_type": "Shell", "name": "success_branch", "command": "echo success_branch" } |
||||
- { "task_type": "Shell", "name": "fail_branch", "command": "echo fail_branch" } |
||||
|
||||
- name: condition |
||||
task_type: Condition |
||||
success_task: success_branch |
||||
failed_task: fail_branch |
||||
op: AND |
||||
groups: |
||||
- op: AND |
||||
groups: |
||||
- task: pre_task_1 |
||||
flag: true |
||||
- task: pre_task_2 |
||||
flag: true |
||||
- task: pre_task_3 |
||||
flag: false |
||||
@ -0,0 +1,33 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
# Define the workflow |
||||
workflow: |
||||
name: "DataX" |
||||
|
||||
# Define the tasks under the workflow |
||||
tasks: |
||||
- name: task |
||||
task_type: DataX |
||||
datasource_name: db |
||||
datatarget_name: db |
||||
sql: show tables; |
||||
target_table: table_test |
||||
|
||||
- name: task_custon_config |
||||
task_type: CustomDataX |
||||
json: $FILE{"example_datax.json"} |
||||
@ -0,0 +1,76 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
workflow: |
||||
name: "Dependent" |
||||
|
||||
# Define the tasks under the workflow |
||||
tasks: |
||||
- name: dependent |
||||
task_type: Dependent |
||||
denpendence: |
||||
op: and |
||||
groups: |
||||
- op: or |
||||
groups: |
||||
- project_name: pydolphin |
||||
process_definition_name: task_dependent_external |
||||
dependent_task_name: task_1 |
||||
|
||||
- project_name: pydolphin |
||||
process_definition_name: task_dependent_external |
||||
dependent_task_name: task_2 |
||||
|
||||
- op: and |
||||
groups: |
||||
- project_name: pydolphin |
||||
process_definition_name: task_dependent_external |
||||
dependent_task_name: task_1 |
||||
dependent_date: LAST_WEDNESDAY |
||||
|
||||
- project_name: pydolphin |
||||
process_definition_name: task_dependent_external |
||||
dependent_task_name: task_2 |
||||
dependent_date: last24Hours |
||||
|
||||
- name: dependent_var |
||||
task_type: Dependent |
||||
denpendence: |
||||
op: and |
||||
groups: |
||||
- op: or |
||||
# we can use ${CONFIG.WORKFLOW_PROJECT} to set the value to configuration.WORKFLOW_PROJECT |
||||
# we can use $WORKFLOW{"Dependent_External.yaml"} to create or update a workflow from dependent_external.yaml and set the value to that workflow name |
||||
groups: |
||||
- project_name: ${CONFIG.WORKFLOW_PROJECT} |
||||
process_definition_name: $WORKFLOW{"Dependent_External.yaml"} |
||||
dependent_task_name: task_1 |
||||
|
||||
- project_name: ${CONFIG.WORKFLOW_PROJECT} |
||||
process_definition_name: $WORKFLOW{"Dependent_External.yaml"} |
||||
dependent_task_name: task_2 |
||||
- op: and |
||||
groups: |
||||
- project_name: ${CONFIG.WORKFLOW_PROJECT} |
||||
process_definition_name: $WORKFLOW{"Dependent_External.yaml"} |
||||
dependent_task_name: task_1 |
||||
dependent_date: LAST_WEDNESDAY |
||||
|
||||
- project_name: ${CONFIG.WORKFLOW_PROJECT} |
||||
process_definition_name: $WORKFLOW{"Dependent_External.yaml"} |
||||
dependent_task_name: task_2 |
||||
dependent_date: last24Hours |
||||
@ -0,0 +1,26 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
# Define the workflow |
||||
workflow: |
||||
name: "task_dependent_external" |
||||
|
||||
# Define the tasks under the workflow |
||||
tasks: |
||||
- { "task_type": "Shell", "name": "task_1", "command": "echo task 1" } |
||||
- { "task_type": "Shell", "name": "task_2", "command": "echo task 2" } |
||||
- { "task_type": "Shell", "name": "task_3", "command": "echo task 3" } |
||||
@ -0,0 +1,29 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
# Define the workflow |
||||
workflow: |
||||
name: "Flink" |
||||
|
||||
# Define the tasks under the workflow |
||||
tasks: |
||||
- name: task |
||||
task_type: Flink |
||||
main_class: org.apache.flink.streaming.examples.wordcount.WordCount |
||||
main_package: test_java.jar |
||||
program_type: JAVA |
||||
deploy_mode: local |
||||
@ -0,0 +1,37 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
# Define the workflow |
||||
workflow: |
||||
name: "Http" |
||||
|
||||
# Define the tasks under the workflow |
||||
tasks: |
||||
- name: task |
||||
task_type: Http |
||||
url: "https://httpbin.org/get" |
||||
http_method: "GET" |
||||
http_params: |
||||
- { "prop": "a", "httpParametersType": "PARAMETER", "value": "1" } |
||||
- { "prop": "b", "httpParametersType": "PARAMETER", "value": "2" } |
||||
- { |
||||
"prop": "Content-Type", |
||||
"httpParametersType": "header", |
||||
"value": "test", |
||||
} |
||||
http_check_condition: "STATUS_CODE_CUSTOM" |
||||
condition: "404" |
||||
@ -0,0 +1,29 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
# Define the workflow |
||||
workflow: |
||||
name: "MapReduce" |
||||
|
||||
# Define the tasks under the workflow |
||||
tasks: |
||||
- name: task |
||||
task_type: MR |
||||
main_class: wordcount |
||||
main_package: test_java.jar |
||||
program_type: SCALA |
||||
main_args: /dolphinscheduler/tenant_exists/resources/file.txt /output/ds |
||||
@ -0,0 +1,40 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
# Define the workflow |
||||
workflow: |
||||
name: "MoreConfiguration" |
||||
param: |
||||
n: 1 |
||||
|
||||
# Define the tasks under the workflow |
||||
tasks: |
||||
- name: shell_0 |
||||
task_type: Shell |
||||
description: "yaml define task" |
||||
flag: "YES" |
||||
command: | |
||||
echo "$ENV{HOME}" |
||||
echo "${n}" |
||||
task_priority: "HIGH" |
||||
delay_time: 20 |
||||
fail_retry_times: 30 |
||||
fail_retry_interval: 5 |
||||
timeout_flag: "CLOSE" |
||||
timeout: 60 |
||||
local_params: |
||||
- { "prop": "n", "direct": "IN", "type": "VARCHAR", "value": "${n}" } |
||||
@ -0,0 +1,27 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
# Define the workflow |
||||
workflow: |
||||
name: "Procedure" |
||||
|
||||
# Define the tasks under the workflow |
||||
tasks: |
||||
- name: task |
||||
task_type: Procedure |
||||
datasource_name: db |
||||
method: show tables; |
||||
@ -0,0 +1,30 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
# Define the workflow |
||||
workflow: |
||||
name: "Python" |
||||
|
||||
# Define the tasks under the workflow |
||||
tasks: |
||||
- name: python |
||||
task_type: Python |
||||
definition: | |
||||
import os |
||||
print(os) |
||||
print("1") |
||||
print("2") |
||||
@ -0,0 +1,40 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
# Define the workflow |
||||
workflow: |
||||
name: "Shell" |
||||
release_state: "offline" |
||||
run: true |
||||
|
||||
# Define the tasks under the process |
||||
tasks: |
||||
- name: task_parent |
||||
task_type: Shell |
||||
command: | |
||||
echo hello pydolphinscheduler |
||||
echo run task parent |
||||
|
||||
- name: task_child_one |
||||
task_type: Shell |
||||
deps: [task_parent] |
||||
command: echo "child one" |
||||
|
||||
- name: task_child_two |
||||
task_type: Shell |
||||
deps: [task_parent] |
||||
command: echo "child two" |
||||
@ -0,0 +1,30 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
# Define the workflow |
||||
workflow: |
||||
name: "Spark" |
||||
|
||||
# Define the tasks under the workflow |
||||
tasks: |
||||
- name: task |
||||
task_type: Spark |
||||
main_class: org.apache.spark.examples.SparkPi |
||||
main_package: test_java.jar |
||||
program_type: SCALA |
||||
deploy_mode: local |
||||
spark_version: SPARK1 |
||||
@ -0,0 +1,45 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
# Define the workflow |
||||
workflow: |
||||
name: "Sql" |
||||
|
||||
# Define the tasks under the workflow |
||||
tasks: |
||||
- name: task_base |
||||
task_type: Sql |
||||
datasource_name: "db" |
||||
sql: show tables; |
||||
|
||||
- name: task_multi_line |
||||
task_type: Sql |
||||
datasource_name: "db" |
||||
sql: | |
||||
show tables; |
||||
select id from version where id=1; |
||||
|
||||
- name: task_file |
||||
task_type: Sql |
||||
datasource_name: "db" |
||||
sql: $FILE{"example_sql.sql"} |
||||
|
||||
# Or you can define task "task_union" it with one line |
||||
- { "task_type": "Sql", "name": "task_base_one_line", "datasource_name": "db", "sql": "select id from version where id=1;"} |
||||
|
||||
# Or you can define task "task_union" it with one line |
||||
- { "task_type": "Sql", "name": "task_file_one_line", "datasource_name": "db", "sql": '$FILE{"example_sql.sql"}'} |
||||
@ -0,0 +1,27 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
# Define the workflow |
||||
workflow: |
||||
name: "SubWorkflow" |
||||
|
||||
tasks: |
||||
- name: example_workflow |
||||
task_type: SubProcess |
||||
process_definition_name: $WORKFLOW{"example_sub_workflow.yaml"} |
||||
|
||||
- { "task_type": "Shell", "deps": [example_workflow], "name": "task_3", "command": "echo task 3" } |
||||
@ -0,0 +1,39 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
# Define the workflow |
||||
workflow: |
||||
name: "Switch" |
||||
param: |
||||
var: 1 |
||||
|
||||
# Define the tasks under the workflow |
||||
tasks: |
||||
- name: switch_child_1 |
||||
task_type: Shell |
||||
command: echo switch_child_1 |
||||
|
||||
- name: switch_child_2 |
||||
task_type: Shell |
||||
command: echo switch_child_2 |
||||
|
||||
- name: switch |
||||
task_type: Switch |
||||
condition: |
||||
- task: switch_child_1 |
||||
condition: "${var} > 1" |
||||
- task: switch_child_2 |
||||
@ -0,0 +1,62 @@
|
||||
{ |
||||
"job": { |
||||
"content": [ |
||||
{ |
||||
"reader": { |
||||
"name": "mysqlreader", |
||||
"parameter": { |
||||
"username": "usr", |
||||
"password": "pwd", |
||||
"column": [ |
||||
"id", |
||||
"name", |
||||
"code", |
||||
"description" |
||||
], |
||||
"splitPk": "id", |
||||
"connection": [ |
||||
{ |
||||
"table": [ |
||||
"source_table" |
||||
], |
||||
"jdbcUrl": [ |
||||
"jdbc:mysql://127.0.0.1:3306/source_db" |
||||
] |
||||
} |
||||
] |
||||
} |
||||
}, |
||||
"writer": { |
||||
"name": "mysqlwriter", |
||||
"parameter": { |
||||
"writeMode": "insert", |
||||
"username": "usr", |
||||
"password": "pwd", |
||||
"column": [ |
||||
"id", |
||||
"name" |
||||
], |
||||
"connection": [ |
||||
{ |
||||
"jdbcUrl": "jdbc:mysql://127.0.0.1:3306/target_db", |
||||
"table": [ |
||||
"target_table" |
||||
] |
||||
} |
||||
] |
||||
} |
||||
} |
||||
} |
||||
], |
||||
"setting": { |
||||
"errorLimit": { |
||||
"percentage": 0, |
||||
"record": 0 |
||||
}, |
||||
"speed": { |
||||
"channel": 1, |
||||
"record": 1000 |
||||
} |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,22 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0 |
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
select id from version where id=1; |
||||
select id from version where id=2; |
||||
select id from version where id=3; |
||||
select id from version where id=4; |
||||
select id from version where id=5; |
||||
@ -0,0 +1,26 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
# Define the workflow |
||||
workflow: |
||||
name: "example_workflow_for_sub_workflow" |
||||
|
||||
# Define the tasks under the workflow |
||||
tasks: |
||||
- { "task_type": "Shell", "name": "task_1", "command": "echo task 1" } |
||||
- { "task_type": "Shell", "deps": [task_1], "name": "task_2", "command": "echo task 2" } |
||||
- { "task_type": "Shell", "deps": [task_2], "name": "task_3", "command": "echo task 3" } |
||||
@ -0,0 +1,46 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
# Define the workflow |
||||
workflow: |
||||
name: "tutorial" |
||||
schedule: "0 0 0 * * ? *" |
||||
start_time: "2021-01-01" |
||||
tenant: "tenant_exists" |
||||
release_state: "offline" |
||||
run: true |
||||
|
||||
# Define the tasks under the workflow |
||||
tasks: |
||||
- name: task_parent |
||||
task_type: Shell |
||||
command: echo hello pydolphinscheduler |
||||
|
||||
- name: task_child_one |
||||
task_type: Shell |
||||
deps: [task_parent] |
||||
command: echo "child one" |
||||
|
||||
- name: task_child_two |
||||
task_type: Shell |
||||
deps: [task_parent] |
||||
command: echo "child two" |
||||
|
||||
- name: task_union |
||||
task_type: Shell |
||||
deps: [task_child_one, task_child_two] |
||||
command: echo "union" |
||||
@ -0,0 +1,466 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
"""Parse YAML file to create process.""" |
||||
|
||||
import logging |
||||
import os |
||||
import re |
||||
from pathlib import Path |
||||
from typing import Any, Dict |
||||
|
||||
from pydolphinscheduler import configuration, tasks |
||||
from pydolphinscheduler.core.process_definition import ProcessDefinition |
||||
from pydolphinscheduler.core.task import Task |
||||
from pydolphinscheduler.exceptions import PyDSTaskNoFoundException |
||||
from pydolphinscheduler.utils.yaml_parser import YamlParser |
||||
|
||||
logger = logging.getLogger(__file__) |
||||
|
||||
KEY_PROCESS = "workflow" |
||||
KEY_TASK = "tasks" |
||||
KEY_TASK_TYPE = "task_type" |
||||
KEY_DEPS = "deps" |
||||
KEY_OP = "op" |
||||
|
||||
TASK_SPECIAL_KEYS = [KEY_TASK_TYPE, KEY_DEPS] |
||||
|
||||
|
||||
class ParseTool: |
||||
"""Enhanced parsing tools.""" |
||||
|
||||
@staticmethod |
||||
def parse_string_param_if_file(string_param: str, **kwargs): |
||||
"""Use $FILE{"data_path"} to load file from "data_path".""" |
||||
if string_param.startswith("$FILE"): |
||||
path = re.findall(r"\$FILE\{\"(.*?)\"\}", string_param)[0] |
||||
base_folder = kwargs.get("base_folder", ".") |
||||
path = ParseTool.get_possible_path(path, base_folder) |
||||
with open(path, "r") as read_file: |
||||
string_param = "".join(read_file) |
||||
return string_param |
||||
|
||||
@staticmethod |
||||
def parse_string_param_if_env(string_param: str, **kwargs): |
||||
"""Use $ENV{env_name} to load environment variable "env_name".""" |
||||
if "$ENV" in string_param: |
||||
key = re.findall(r"\$ENV\{(.*?)\}", string_param)[0] |
||||
env_value = os.environ.get(key, "$%s" % key) |
||||
string_param = string_param.replace("$ENV{%s}" % key, env_value) |
||||
return string_param |
||||
|
||||
@staticmethod |
||||
def parse_string_param_if_config(string_param: str, **kwargs): |
||||
"""Use ${CONFIG.var_name} to load variable "var_name" from configuration.""" |
||||
if "${CONFIG" in string_param: |
||||
key = re.findall(r"\$\{CONFIG\.(.*?)\}", string_param)[0] |
||||
if hasattr(configuration, key): |
||||
string_param = getattr(configuration, key) |
||||
else: |
||||
string_param = configuration.get_single_config(key) |
||||
|
||||
return string_param |
||||
|
||||
@staticmethod |
||||
def get_possible_path(file_path, base_folder): |
||||
"""Get file possible path. |
||||
|
||||
Return new path if file_path is not exists, but base_folder + file_path exists |
||||
""" |
||||
possible_path = file_path |
||||
if not Path(file_path).exists(): |
||||
new_path = Path(base_folder).joinpath(file_path) |
||||
if new_path.exists(): |
||||
possible_path = new_path |
||||
logger.info(f"{file_path} not exists, convert to {possible_path}") |
||||
|
||||
return possible_path |
||||
|
||||
|
||||
def get_task_cls(task_type) -> Task: |
||||
"""Get the task class object by task_type (case compatible).""" |
||||
# only get task class from tasks.__all__ |
||||
all_task_types = {type_.capitalize(): type_ for type_ in tasks.__all__} |
||||
task_type_cap = task_type.capitalize() |
||||
if task_type_cap not in all_task_types: |
||||
raise PyDSTaskNoFoundException("cant not find task %s" % task_type) |
||||
|
||||
standard_name = all_task_types[task_type_cap] |
||||
return getattr(tasks, standard_name) |
||||
|
||||
|
||||
class YamlProcess(YamlParser): |
||||
"""Yaml parser for create process. |
||||
|
||||
:param yaml_file: yaml file path. |
||||
|
||||
examples1 :: |
||||
|
||||
parser = YamlParser(yaml_file=...) |
||||
parser.create_process_definition() |
||||
|
||||
examples2 :: |
||||
|
||||
YamlParser(yaml_file=...).create_process_definition() |
||||
|
||||
""" |
||||
|
||||
_parse_rules = [ |
||||
ParseTool.parse_string_param_if_file, |
||||
ParseTool.parse_string_param_if_env, |
||||
ParseTool.parse_string_param_if_config, |
||||
] |
||||
|
||||
def __init__(self, yaml_file: str): |
||||
with open(yaml_file, "r") as f: |
||||
content = f.read() |
||||
|
||||
self._base_folder = Path(yaml_file).parent |
||||
content = self.prepare_refer_process(content) |
||||
super().__init__(content) |
||||
|
||||
def create_process_definition(self): |
||||
"""Create process main function.""" |
||||
# get process parameters with key "workflow" |
||||
process_params = self[KEY_PROCESS] |
||||
|
||||
# pop "run" parameter, used at the end |
||||
is_run = process_params.pop("run", False) |
||||
|
||||
# use YamlProcess._parse_rules to parse special value of yaml file |
||||
process_params = self.parse_params(process_params) |
||||
|
||||
process_name = process_params["name"] |
||||
logger.info(f"Create Process: {process_name}") |
||||
with ProcessDefinition(**process_params) as pd: |
||||
|
||||
# save dependencies between tasks |
||||
dependencies = {} |
||||
|
||||
# save name and task mapping |
||||
name2task = {} |
||||
|
||||
# get task datas with key "tasks" |
||||
for task_data in self[KEY_TASK]: |
||||
task = self.parse_task(task_data, name2task) |
||||
|
||||
deps = task_data.get(KEY_DEPS, []) |
||||
if deps: |
||||
dependencies[task.name] = deps |
||||
name2task[task.name] = task |
||||
|
||||
# build dependencies between task |
||||
for downstream_task_name, deps in dependencies.items(): |
||||
downstream_task = name2task[downstream_task_name] |
||||
for upstream_task_name in deps: |
||||
upstream_task = name2task[upstream_task_name] |
||||
upstream_task >> downstream_task |
||||
|
||||
pd.submit() |
||||
# if set is_run, run the process after submit |
||||
if is_run: |
||||
logger.info(f"run workflow: {pd}") |
||||
pd.run() |
||||
|
||||
return process_name |
||||
|
||||
def parse_params(self, params: Any): |
||||
"""Recursively resolves the parameter values. |
||||
|
||||
The function operates params only when it encounters a string; other types continue recursively. |
||||
""" |
||||
if isinstance(params, str): |
||||
for parse_rule in self._parse_rules: |
||||
params_ = params |
||||
params = parse_rule(params, base_folder=self._base_folder) |
||||
if params_ != params: |
||||
logger.info(f"parse {params_} -> {params}") |
||||
|
||||
elif isinstance(params, list): |
||||
for index in range(len(params)): |
||||
params[index] = self.parse_params(params[index]) |
||||
|
||||
elif isinstance(params, dict): |
||||
for key, value in params.items(): |
||||
params[key] = self.parse_params(value) |
||||
|
||||
return params |
||||
|
||||
@classmethod |
||||
def parse(cls, yaml_file: str): |
||||
"""Recursively resolves the parameter values. |
||||
|
||||
The function operates params only when it encounters a string; other types continue recursively. |
||||
""" |
||||
process_name = cls(yaml_file).create_process_definition() |
||||
return process_name |
||||
|
||||
def prepare_refer_process(self, content): |
||||
"""Allow YAML files to reference process derived from other YAML files.""" |
||||
process_paths = re.findall(r"\$WORKFLOW\{\"(.*?)\"\}", content) |
||||
for process_path in process_paths: |
||||
logger.info( |
||||
f"find special token {process_path}, load process form {process_path}" |
||||
) |
||||
possible_path = ParseTool.get_possible_path(process_path, self._base_folder) |
||||
process_name = YamlProcess.parse(possible_path) |
||||
content = content.replace('$WORKFLOW{"%s"}' % process_path, process_name) |
||||
|
||||
return content |
||||
|
||||
def parse_task(self, task_data: dict, name2task: Dict[str, Task]): |
||||
"""Parse various types of tasks. |
||||
|
||||
:param task_data: dict. |
||||
{ |
||||
"task_type": "Shell", |
||||
"params": {"name": "shell_task", "command":"ehco hellp"} |
||||
} |
||||
|
||||
:param name2task: Dict[str, Task]), mapping of task_name and task |
||||
|
||||
|
||||
Some task type have special parse func: |
||||
if task type is Switch, use parse_switch; |
||||
if task type is Condition, use parse_condition; |
||||
if task type is Dependent, use parse_dependent; |
||||
other, we pass all task_params as input to task class, like "task_cls(**task_params)". |
||||
""" |
||||
task_type = task_data["task_type"] |
||||
# get params without special key |
||||
task_params = {k: v for k, v in task_data.items() if k not in TASK_SPECIAL_KEYS} |
||||
|
||||
task_cls = get_task_cls(task_type) |
||||
|
||||
# use YamlProcess._parse_rules to parse special value of yaml file |
||||
task_params = self.parse_params(task_params) |
||||
|
||||
if task_cls == tasks.Switch: |
||||
task = self.parse_switch(task_params, name2task) |
||||
|
||||
elif task_cls == tasks.Condition: |
||||
task = self.parse_condition(task_params, name2task) |
||||
|
||||
elif task_cls == tasks.Dependent: |
||||
task = self.parse_dependent(task_params, name2task) |
||||
|
||||
else: |
||||
task = task_cls(**task_params) |
||||
logger.info(task_type, task) |
||||
return task |
||||
|
||||
def parse_switch(self, task_params, name2task): |
||||
"""Parse Switch Task. |
||||
|
||||
This is an example Yaml fragment of task_params |
||||
|
||||
name: switch |
||||
condition: |
||||
- ["${var} > 1", switch_child_1] |
||||
- switch_child_2 |
||||
""" |
||||
from pydolphinscheduler.tasks.switch import ( |
||||
Branch, |
||||
Default, |
||||
Switch, |
||||
SwitchCondition, |
||||
) |
||||
|
||||
condition_datas = task_params["condition"] |
||||
conditions = [] |
||||
for condition_data in condition_datas: |
||||
assert "task" in condition_data, "task must be in %s" % condition_data |
||||
task_name = condition_data["task"] |
||||
condition_string = condition_data.get("condition", None) |
||||
|
||||
# if condition_string is None, for example: {"task": "switch_child_2"}, set it to Default branch |
||||
if condition_string is None: |
||||
conditions.append(Default(task=name2task.get(task_name))) |
||||
|
||||
# if condition_string is not None, for example: |
||||
# {"task": "switch_child_2", "condition": "${var} > 1"} set it to Branch |
||||
else: |
||||
conditions.append( |
||||
Branch(condition_string, task=name2task.get(task_name)) |
||||
) |
||||
|
||||
switch = Switch( |
||||
name=task_params["name"], condition=SwitchCondition(*conditions) |
||||
) |
||||
return switch |
||||
|
||||
def parse_condition(self, task_params, name2task): |
||||
"""Parse Condition Task. |
||||
|
||||
This is an example Yaml fragment of task_params |
||||
|
||||
name: condition |
||||
success_task: success_branch |
||||
failed_task: fail_branch |
||||
OP: AND |
||||
groups: |
||||
- |
||||
OP: AND |
||||
groups: |
||||
- [pre_task_1, true] |
||||
- [pre_task_2, true] |
||||
- [pre_task_3, false] |
||||
- |
||||
OP: AND |
||||
groups: |
||||
- [pre_task_1, false] |
||||
- [pre_task_2, true] |
||||
- [pre_task_3, true] |
||||
|
||||
""" |
||||
from pydolphinscheduler.tasks.condition import ( |
||||
FAILURE, |
||||
SUCCESS, |
||||
And, |
||||
Condition, |
||||
Or, |
||||
) |
||||
|
||||
def get_op_cls(op): |
||||
cls = None |
||||
if op.lower() == "and": |
||||
cls = And |
||||
elif op.lower() == "or": |
||||
cls = Or |
||||
else: |
||||
raise Exception("OP must be in And or Or, but get: %s" % op) |
||||
return cls |
||||
|
||||
second_cond_ops = [] |
||||
for first_group in task_params["groups"]: |
||||
second_op = first_group["op"] |
||||
task_ops = [] |
||||
for condition_data in first_group["groups"]: |
||||
assert "task" in condition_data, "task must be in %s" % condition_data |
||||
assert "flag" in condition_data, "flag must be in %s" % condition_data |
||||
task_name = condition_data["task"] |
||||
flag = condition_data["flag"] |
||||
task = name2task[task_name] |
||||
|
||||
# for example: task = pre_task_1, flag = true |
||||
if flag: |
||||
task_ops.append(SUCCESS(task)) |
||||
else: |
||||
task_ops.append(FAILURE(task)) |
||||
|
||||
second_cond_ops.append(get_op_cls(second_op)(*task_ops)) |
||||
|
||||
first_op = task_params["op"] |
||||
cond_operator = get_op_cls(first_op)(*second_cond_ops) |
||||
|
||||
condition = Condition( |
||||
name=task_params["name"], |
||||
condition=cond_operator, |
||||
success_task=name2task[task_params["success_task"]], |
||||
failed_task=name2task[task_params["failed_task"]], |
||||
) |
||||
return condition |
||||
|
||||
def parse_dependent(self, task_params, name2task): |
||||
"""Parse Dependent Task. |
||||
|
||||
This is an example Yaml fragment of task_params |
||||
|
||||
name: dependent |
||||
denpendence: |
||||
OP: AND |
||||
groups: |
||||
- |
||||
OP: Or |
||||
groups: |
||||
- [pydolphin, task_dependent_external, task_1] |
||||
- [pydolphin, task_dependent_external, task_2] |
||||
- |
||||
OP: And |
||||
groups: |
||||
- [pydolphin, task_dependent_external, task_1, LAST_WEDNESDAY] |
||||
- [pydolphin, task_dependent_external, task_2, last24Hours] |
||||
|
||||
""" |
||||
from pydolphinscheduler.tasks.dependent import ( |
||||
And, |
||||
Dependent, |
||||
DependentDate, |
||||
DependentItem, |
||||
Or, |
||||
) |
||||
|
||||
def process_dependent_date(dependent_date): |
||||
"""Parse dependent date (Compatible with key and value of DependentDate).""" |
||||
dependent_date_upper = dependent_date.upper() |
||||
if hasattr(DependentDate, dependent_date_upper): |
||||
dependent_date = getattr(DependentDate, dependent_date_upper) |
||||
return dependent_date |
||||
|
||||
def get_op_cls(op): |
||||
cls = None |
||||
if op.lower() == "and": |
||||
cls = And |
||||
elif op.lower() == "or": |
||||
cls = Or |
||||
else: |
||||
raise Exception("OP must be in And or Or, but get: %s" % op) |
||||
return cls |
||||
|
||||
def create_dependent_item(source_items): |
||||
"""Parse dependent item. |
||||
|
||||
project_name: pydolphin |
||||
process_definition_name: task_dependent_external |
||||
dependent_task_name: task_1 |
||||
dependent_date: LAST_WEDNESDAY |
||||
""" |
||||
project_name = source_items["project_name"] |
||||
process_definition_name = source_items["process_definition_name"] |
||||
dependent_task_name = source_items["dependent_task_name"] |
||||
dependent_date = source_items.get("dependent_date", DependentDate.TODAY) |
||||
dependent_item = DependentItem( |
||||
project_name=project_name, |
||||
process_definition_name=process_definition_name, |
||||
dependent_task_name=dependent_task_name, |
||||
dependent_date=process_dependent_date(dependent_date), |
||||
) |
||||
|
||||
return dependent_item |
||||
|
||||
second_dependences = [] |
||||
for first_group in task_params["groups"]: |
||||
second_op = first_group[KEY_OP] |
||||
dependence_items = [] |
||||
for source_items in first_group["groups"]: |
||||
dependence_items.append(create_dependent_item(source_items)) |
||||
|
||||
second_dependences.append(get_op_cls(second_op)(*dependence_items)) |
||||
|
||||
first_op = task_params[KEY_OP] |
||||
dependence = get_op_cls(first_op)(*second_dependences) |
||||
|
||||
task = Dependent( |
||||
name=task_params["name"], |
||||
dependence=dependence, |
||||
) |
||||
return task |
||||
|
||||
|
||||
def create_process_definition(yaml_file): |
||||
"""CLI.""" |
||||
YamlProcess.parse(yaml_file) |
||||
@ -0,0 +1,191 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
"""Test YAML process.""" |
||||
|
||||
import os |
||||
from pathlib import Path |
||||
from unittest.mock import patch |
||||
|
||||
import pytest |
||||
|
||||
from pydolphinscheduler import configuration, tasks |
||||
from pydolphinscheduler.core.process_definition import ProcessDefinition |
||||
from pydolphinscheduler.core.yaml_process_define import ( |
||||
ParseTool, |
||||
create_process_definition, |
||||
get_task_cls, |
||||
) |
||||
from pydolphinscheduler.exceptions import PyDSTaskNoFoundException |
||||
from tests.testing.path import path_yaml_example |
||||
from tests.testing.task import Task |
||||
|
||||
|
||||
@pytest.mark.parametrize( |
||||
"string_param, expect", |
||||
[ |
||||
("$ENV{PROJECT_NAME}", "~/pydolphinscheduler"), |
||||
], |
||||
) |
||||
def test_parse_tool_env_exist(string_param, expect): |
||||
"""Test parsing the environment variable.""" |
||||
os.environ["PROJECT_NAME"] = expect |
||||
assert expect == ParseTool.parse_string_param_if_env(string_param) |
||||
|
||||
|
||||
def test_parse_tool_env_not_exist(): |
||||
"""Test parsing the not exist environment variable.""" |
||||
key = "THIS_ENV_NOT_EXIST_0000000" |
||||
string_param = "$ENV{%s}" % key |
||||
expect = "$" + key |
||||
assert expect == ParseTool.parse_string_param_if_env(string_param) |
||||
|
||||
|
||||
@pytest.mark.parametrize( |
||||
"string_param, expect_key", |
||||
[ |
||||
("${CONFIG.java_gateway.address}", "java_gateway.address"), |
||||
("${CONFIG.WORKFLOW_PROJECT}", "default.workflow.project"), |
||||
], |
||||
) |
||||
def test_parse_tool_config(string_param, expect_key): |
||||
"""Test parsing configuration.""" |
||||
expect = configuration.get_single_config(expect_key) |
||||
assert expect == ParseTool.parse_string_param_if_config(string_param) |
||||
|
||||
|
||||
def test_parse_possible_yaml_file(): |
||||
"""Test parsing possible path.""" |
||||
folder = Path(path_yaml_example) |
||||
file_name = "Shell.yaml" |
||||
path = folder.joinpath(file_name) |
||||
|
||||
with open(path, "r") as f: |
||||
expect = "".join(f) |
||||
|
||||
string_param = '$FILE{"%s"}' % file_name |
||||
content_ = ParseTool.parse_string_param_if_file(string_param, base_folder=folder) |
||||
|
||||
assert expect == content_ |
||||
|
||||
|
||||
def test_parse_tool_parse_possible_path_file(): |
||||
"""Test parsing possible path.""" |
||||
folder = Path(path_yaml_example) |
||||
file_name = "Shell.yaml" |
||||
path = folder.joinpath(file_name) |
||||
|
||||
possible_path = ParseTool.get_possible_path(path, base_folder=folder) |
||||
assert path == possible_path |
||||
|
||||
possible_path = ParseTool.get_possible_path(file_name, base_folder=folder) |
||||
assert path == possible_path |
||||
|
||||
possible_path = ParseTool.get_possible_path(file_name, base_folder=".") |
||||
assert path != possible_path |
||||
|
||||
|
||||
@pytest.mark.parametrize( |
||||
"task_type, expect", |
||||
[ |
||||
("shell", tasks.Shell), |
||||
("Shell", tasks.Shell), |
||||
("ShEll", tasks.Shell), |
||||
("Condition", tasks.Condition), |
||||
("DataX", tasks.DataX), |
||||
("CustomDataX", tasks.CustomDataX), |
||||
("Dependent", tasks.Dependent), |
||||
("Flink", tasks.Flink), |
||||
("Http", tasks.Http), |
||||
("MR", tasks.MR), |
||||
("Procedure", tasks.Procedure), |
||||
("Python", tasks.Python), |
||||
("Shell", tasks.Shell), |
||||
("Spark", tasks.Spark), |
||||
("Sql", tasks.Sql), |
||||
("SubProcess", tasks.SubProcess), |
||||
("Switch", tasks.Switch), |
||||
("SageMaker", tasks.SageMaker), |
||||
], |
||||
) |
||||
def test_get_task(task_type, expect): |
||||
"""Test get task function.""" |
||||
assert expect == get_task_cls(task_type) |
||||
|
||||
|
||||
@pytest.mark.parametrize( |
||||
"task_type", |
||||
[ |
||||
("MYSQL"), |
||||
], |
||||
) |
||||
def test_get_error(task_type): |
||||
"""Test get task cls error.""" |
||||
with pytest.raises( |
||||
PyDSTaskNoFoundException, |
||||
match=f"not find task {task_type}", |
||||
): |
||||
get_task_cls(task_type) |
||||
|
||||
|
||||
@pytest.mark.parametrize( |
||||
"yaml_file", |
||||
[ |
||||
("Condition.yaml"), |
||||
("DataX.yaml"), |
||||
("Dependent.yaml"), |
||||
("Flink.yaml"), |
||||
("Procedure.yaml"), |
||||
("Http.yaml"), |
||||
("MapReduce.yaml"), |
||||
("Python.yaml"), |
||||
("Shell.yaml"), |
||||
("Spark.yaml"), |
||||
("Sql.yaml"), |
||||
("SubProcess.yaml"), |
||||
# ("Switch.yaml"), |
||||
("MoreConfiguration.yaml"), |
||||
], |
||||
) |
||||
@patch( |
||||
"pydolphinscheduler.core.engine.Engine.get_resource_info", |
||||
return_value=({"id": 1, "name": "test"}), |
||||
) |
||||
@patch( |
||||
"pydolphinscheduler.core.database.Database.get_database_info", |
||||
return_value=({"id": 1, "type": "mock_type"}), |
||||
) |
||||
@patch( |
||||
"pydolphinscheduler.tasks.dependent.DependentItem.get_code_from_gateway", |
||||
return_value={ |
||||
"projectCode": 0, |
||||
"processDefinitionCode": 0, |
||||
"taskDefinitionCode": 0, |
||||
}, |
||||
) |
||||
@patch.object(ProcessDefinition, "run") |
||||
@patch.object(ProcessDefinition, "submit") |
||||
def test_get_create_process_definition( |
||||
prun, psubmit, dep_item, db_info, resource_info, yaml_file |
||||
): |
||||
"""Test create_process_definition function to parse example YAML file.""" |
||||
yaml_file_path = Path(path_yaml_example).joinpath(yaml_file) |
||||
with patch( |
||||
"pydolphinscheduler.core.task.Task.gen_code_and_version", |
||||
side_effect=Task("test_func_wrap", "func_wrap").gen_code_and_version, |
||||
): |
||||
create_process_definition(yaml_file_path) |
||||
Loading…
Reference in new issue