#3435: [DSIP-11][python] create workflows from YAML configuration (#11611)

2 years ago · b5d6bd4797
2 changed files with 158 additions and 104 deletions
--- a/commit.ref
+++ b/commit.ref
@ -1 +1 @@
-Subproject commit 8988492c4329c93b4872ba3a3059a92582e99bda
+Subproject commit 38ee91fb1edf9fc5a86384cbcade4da7d003687a
--- a/commit.yml
+++ b/commit.yml
@ -2,115 +2,169 @@
 branches: {
  }
 fileChange:
-  /dolphinscheduler-api:
-    newFile: !uuid '3025a5a8c8dc750047f5cc1746c643d2c380282d'
-    oldFile: !uuid '70df813ddc9b0fdca6e67fd0db97a93c7706d874'
-  /dolphinscheduler-api/src:
-    newFile: !uuid '8377194a77c2cba6d9e3a42a9a2a3c4b4cc3e367'
-    oldFile: !uuid '568d9d565bbddca08d5840e7773eef7e44712eab'
-  /dolphinscheduler-api/src/main:
-    newFile: !uuid '694ae0c2d0ab28bfb6430e155dd239f63818a5a4'
-    oldFile: !uuid 'a91babb02b6160a913daa8ca1e084c3d3fdcc8b3'
-  /dolphinscheduler-api/src/main/java:
-    newFile: !uuid '58f0d59ab34c6a425d70afec20c40a2345d5835c'
-    oldFile: !uuid 'e227fa68bb50776ccfe4c7fc2826a31565483045'
-  /dolphinscheduler-api/src/main/java/org:
-    newFile: !uuid '3b6be7471212dd9a9f1f222b5fbe706e7908d9fe'
-    oldFile: !uuid '302af3e63d828c00a5a5c4573b301117ecda7008'
-  /dolphinscheduler-api/src/main/java/org/apache:
-    newFile: !uuid '66c255d2e28ddf6e16d38f7d6da015b9fd8a0cb8'
-    oldFile: !uuid '6f0d8bff31a1c7f7c1cc8635ebfd07fc596d4115'
-  /dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler:
-    newFile: !uuid '445a2847f35b2cf4ff8b4d9cda9bb6bbbe4bb113'
-    oldFile: !uuid '65f28a51079b4e70a77f909140e10ee308617cd2'
-  /dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api:
-    newFile: !uuid '08f3b286324ac46c42221b35cd8cfd4bb268b4a1'
-    oldFile: !uuid '2b9eee16138ad3320f153790bdd00e9422f76c5f'
-  /dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/python:
-    newFile: !uuid '9eef6e71935134b597ef726a173edb6697fafd1f'
-    oldFile: !uuid 'a2f7fe9f058beac14c6d4ae73214d680badbb6cc'
-  /dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/python/PythonGateway.java:
-    newFile: !uuid '5d0873e6c9bdb3ba4d399d0320db87995d56622a'
-    oldFile: !uuid 'bcb62f7333599d8aed20d461d1770d1605813ba5'
  /dolphinscheduler-python:
-    newFile: !uuid '6e5b9e1368336848c916b31f985b4b326440bac6'
-    oldFile: !uuid 'ef8c9413ba9d8a1e95fa0b2b03265d12713d4a23'
+    newFile: !uuid '443d468937022077e3521dd9575d2bcb3f76460b'
+    oldFile: !uuid '6e5b9e1368336848c916b31f985b4b326440bac6'
  /dolphinscheduler-python/pydolphinscheduler:
-    newFile: !uuid '88bee9f5befc9a39939f32472339bfd1d3d19f55'
-    oldFile: !uuid '555156f226d0b54aa079a08e95c31f946c197821'
+    newFile: !uuid '1b3d26d3e3d65faa9c7f8ea205c96af7c610f57f'
+    oldFile: !uuid '88bee9f5befc9a39939f32472339bfd1d3d19f55'
+  /dolphinscheduler-python/pydolphinscheduler/docs:
+    newFile: !uuid 'f4f34f0e29da64b6e52279bbd91244413c17a6d5'
+    oldFile: !uuid '078c94c0b2e4c090403980d061c4181411160ba9'
+  /dolphinscheduler-python/pydolphinscheduler/docs/source:
+    newFile: !uuid 'ac8dcf695a55d82bae403836b69a978a6dc86366'
+    oldFile: !uuid '14adf6010baecb3e6974fdaef6037b2ed744b08b'
+  /dolphinscheduler-python/pydolphinscheduler/docs/source/tasks:
+    newFile: !uuid 'a5d4f9bd4ebc13ac90f5ec59c3bde5321d957940'
+    oldFile: !uuid '9c7c38125c6a7fb6f0e16bb350ca4736308ba9b0'
+  /dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/condition.rst:
+    newFile: !uuid 'f6d7e6ad8f35e8e9953bfed0bce51bbcd7b40008'
+    oldFile: !uuid '20b03500781dbef33d1ef744f2e1106da3fb51a2'
+  /dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/datax.rst:
+    newFile: !uuid 'cb67a2fa9ee6ad6bf7140a65c40eafa4be38943c'
+    oldFile: !uuid 'c07726941e32b72ad476eaea0030cd7905eb6b69'
+  /dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/dependent.rst:
+    newFile: !uuid 'd8e1599b2d0cd3c6dca76ee7510006673b970691'
+    oldFile: !uuid 'fe26d0f30a392420b7d7599349c033a50bebf58c'
+  /dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/flink.rst:
+    newFile: !uuid '76eb484718958a0555369622d20e35a779420e56'
+    oldFile: !uuid '8db9ac266dbe8ca3746c7305d2c5a683eedc3672'
+  /dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/func_wrap.rst:
+    newFile: !uuid 'a4a29729331a0f002774a649670dd69d4addf8ba'
+    oldFile: !uuid '5f41b80cfdabf17c1f8bcf5445d73654d93429ac'
+  /dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/http.rst:
+    newFile: !uuid '4e138c99893930e2ed7b62a94e99a2838bf216a5'
+    oldFile: !uuid '4c6d8f8e402d3ab3b8a560d14d3d5f2660345d35'
+  /dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/map_reduce.rst:
+    newFile: !uuid '7356880b26cbc6ea5a343805985e312c42995ff9'
+    oldFile: !uuid '068b8d8b4165292b53e4d8a57b2e8a5a368235ee'
+  /dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/procedure.rst:
+    newFile: !uuid '2f28efc526c6f96ce7cba2f73f7ff74bf9bec932'
+    oldFile: !uuid 'cd79eff140b403352b374a6d577ac4bcdb0e9a9a'
+  /dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/python.rst:
+    newFile: !uuid '1bf6210018f2ca85217983548a49849cfe8f20f1'
+    oldFile: !uuid '660e46a6e5f8f6fc2a7d73ae9a5efbcd36c4d116'
+  /dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/shell.rst:
+    newFile: !uuid '2dd106a3b815154e552b9341e0f34265aab3f292'
+    oldFile: !uuid '5ce16c3c9f49aa83ac0ee910506db4caf93bbd55'
+  /dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/spark.rst:
+    newFile: !uuid 'd5a51db91a625e4511b052e8cb445e6b4c158c2e'
+    oldFile: !uuid 'cdb5902c370187f3f7591df480dca5bf1d4aa355'
+  /dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/sql.rst:
+    newFile: !uuid '52df042b74646ffae8da491a876f64634318a703'
+    oldFile: !uuid '21eaec7ae97e461b5cd1cc23271d4b56a3fd6ca6'
+  /dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/sub_process.rst:
+    newFile: !uuid '894dd0fbadd49fa593a6a75f4736ba9833e6f82b'
+    oldFile: !uuid '8a9f56220054f921ce0b97be7a3e28023c31ebd9'
+  /dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/switch.rst:
+    newFile: !uuid '2fef589efbb40cf96ac25a1ee606824d31552543'
+    oldFile: !uuid 'd8b34a4ac9301c08ca8fdb739cf57dd10e028a30'
+  /dolphinscheduler-python/pydolphinscheduler/docs/source/tutorial.rst:
+    newFile: !uuid '4efc09ae7b6e1606eb5b6f670c539e46a7383260'
+    oldFile: !uuid '6366c803bb5cc0536c55a13419d9a93ae4c31a19'
+  /dolphinscheduler-python/pydolphinscheduler/examples:
+    newFile: !uuid '35f2ad7ec1da42dd7a212a464abe8d833d685b2f'
+    oldFile: null
+  /dolphinscheduler-python/pydolphinscheduler/examples/yaml_define:
+    newFile: !uuid '1cad896d21475c950cefdec39bbf0ef0577ee873'
+    oldFile: null
+  /dolphinscheduler-python/pydolphinscheduler/examples/yaml_define/Condition.yaml:
+    newFile: !uuid 'c65b8c7aeb9f8afdb4c3de45833b390800247c01'
+    oldFile: null
+  /dolphinscheduler-python/pydolphinscheduler/examples/yaml_define/DataX.yaml:
+    newFile: !uuid '00ecd54685b830b30de49bf6b4d811f9fbab4171'
+    oldFile: null
+  /dolphinscheduler-python/pydolphinscheduler/examples/yaml_define/Dependent.yaml:
+    newFile: !uuid 'd69fac05da0f19efb1f9023d96d3bf8c7018c6ce'
+    oldFile: null
+  /dolphinscheduler-python/pydolphinscheduler/examples/yaml_define/Dependent_External.yaml:
+    newFile: !uuid '577ff6a807cd76a3e5a3e37782793e6344bd3b2d'
+    oldFile: null
+  /dolphinscheduler-python/pydolphinscheduler/examples/yaml_define/Flink.yaml:
+    newFile: !uuid '2449d435a374c6a093f92e7f1349d0de26d3cd5d'
+    oldFile: null
+  /dolphinscheduler-python/pydolphinscheduler/examples/yaml_define/Http.yaml:
+    newFile: !uuid '1483aeb3d842cb5528f27497a4c20ea6628ae1ff'
+    oldFile: null
+  /dolphinscheduler-python/pydolphinscheduler/examples/yaml_define/MapReduce.yaml:
+    newFile: !uuid 'e1a2b5709c755b9bb51f074b9ada17747f45d92e'
+    oldFile: null
+  /dolphinscheduler-python/pydolphinscheduler/examples/yaml_define/MoreConfiguration.yaml:
+    newFile: !uuid '258aa33433e91f4617168bb225ad6a8637f96ff4'
+    oldFile: null
+  /dolphinscheduler-python/pydolphinscheduler/examples/yaml_define/Procedure.yaml:
+    newFile: !uuid '829a961c1ab58c9c9ecffd94351d747f91962219'
+    oldFile: null
+  /dolphinscheduler-python/pydolphinscheduler/examples/yaml_define/Python.yaml:
+    newFile: !uuid '728b5c928e68a57838592d5a8d476c70c4eb5cb5'
+    oldFile: null
+  /dolphinscheduler-python/pydolphinscheduler/examples/yaml_define/Shell.yaml:
+    newFile: !uuid 'fdbe1263278288802936ddb0b5b0dd72ef1e974c'
+    oldFile: null
+  /dolphinscheduler-python/pydolphinscheduler/examples/yaml_define/Spark.yaml:
+    newFile: !uuid '6132b8d7498880b4126527c93ef3cbe9adc26a77'
+    oldFile: null
+  /dolphinscheduler-python/pydolphinscheduler/examples/yaml_define/Sql.yaml:
+    newFile: !uuid 'c3c7e88ee1e8514ff34f65448a97cea2d12126ab'
+    oldFile: null
+  /dolphinscheduler-python/pydolphinscheduler/examples/yaml_define/SubProcess.yaml:
+    newFile: !uuid '0ea7549db4fd142555902a3a8bb2d048a50f5f80'
+    oldFile: null
+  /dolphinscheduler-python/pydolphinscheduler/examples/yaml_define/Switch.yaml:
+    newFile: !uuid '33ed68813ee34f4fbf44c13abc20eb79cf24b697'
+    oldFile: null
+  /dolphinscheduler-python/pydolphinscheduler/examples/yaml_define/example_datax.json:
+    newFile: !uuid '3db8092cb6edc90f1781ae4cf858ef1b57d1ce49'
+    oldFile: null
+  /dolphinscheduler-python/pydolphinscheduler/examples/yaml_define/example_sql.sql:
+    newFile: !uuid '06b5c4c16cce03fcd708fb763619090c5039ac49'
+    oldFile: null
+  /dolphinscheduler-python/pydolphinscheduler/examples/yaml_define/example_sub_workflow.yaml:
+    newFile: !uuid 'af3a863da9af75df667f848dc151143550f59e86'
+    oldFile: null
+  /dolphinscheduler-python/pydolphinscheduler/examples/yaml_define/tutorial.yaml:
+    newFile: !uuid '104a8c367b034b5c41ee738219a95c70046b6551'
+    oldFile: null
  /dolphinscheduler-python/pydolphinscheduler/src:
-    newFile: !uuid '2024d662075eec972eb04c039e240b8affb23e9b'
-    oldFile: !uuid 'e5a0cd8b49209f190606e4179bc82ca74db4e04a'
+    newFile: !uuid '901653cb9beef42b37e3322721a027f76087fbeb'
+    oldFile: !uuid '2024d662075eec972eb04c039e240b8affb23e9b'
  /dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler:
-    newFile: !uuid '6738420aa701ccc3941cf7346d6d670566c62074'
-    oldFile: !uuid '9f568287d758d84cc8bb8801961c1b1ac31c1817'
+    newFile: !uuid 'ab34f00e3e8757faddd4bdfee19bfd6ffd2e8c53'
+    oldFile: !uuid '6738420aa701ccc3941cf7346d6d670566c62074'
+  /dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/cli:
+    newFile: !uuid 'e0be0f304257e0a54855b0abb325fc8f64c91309'
+    oldFile: !uuid '366a86efc756bdec499c91e4985e4bdae07635cb'
+  /dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/cli/commands.py:
+    newFile: !uuid '8d923f140616e09c607146f8010a0f074385462b'
+    oldFile: !uuid 'd78e503dbbbd7e7b504b2bf0240afbf49ba2157e'
  /dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core:
-    newFile: !uuid '3f3f943caa405a1501464c76a121e6f0e7c20e81'
-    oldFile: !uuid '51b17b53518109dd41a737116b0c361481fca8db'
-  /dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py:
-    newFile: !uuid 'd1a2eae9dd5d9d44be0ca206397356ac75bca0db'
-    oldFile: !uuid '93c5f283422541523d4898228aedb2d26d1d2702'
-  /dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/java_gateway.py:
-    newFile: !uuid '0ff74ba65529900e4a763bbef2d5c1d799bccda3'
-    oldFile: !uuid '59d411b62ca38e1056f9b7b1c3cc836e69e0f7ee'
+    newFile: !uuid '83bb34fdec81d8a64cb53c721c699c63753eb0b8'
+    oldFile: !uuid '3f3f943caa405a1501464c76a121e6f0e7c20e81'
+  /dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/yaml_process_define.py:
+    newFile: !uuid '0944925a48f41d7967410b8119252d4e5899ea43'
+    oldFile: null
+  /dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/tasks:
+    newFile: !uuid '312de987b3c1e51f2fb89d0eb2893e8880f6d72a'
+    oldFile: !uuid '373ca1a6f2a3861a8d9b077da1af9202c7337a0f'
+  /dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/tasks/__init__.py:
+    newFile: !uuid '53b462ca902b6b280849564fac9ed0af6e2d4fc3'
+    oldFile: !uuid 'd49a1d394c386ac41c616ca3fa159873f6b5a7ad'
  /dolphinscheduler-python/pydolphinscheduler/tests:
-    newFile: !uuid '977d32fa832c73fe82f46f8fef46f8ef3991ed21'
-    oldFile: !uuid '88983acacd9163cca172475da951f34bf05dd307'
+    newFile: !uuid '8145885d63d6b94168b6437fd100f76a8bbff6e8'
+    oldFile: !uuid '977d32fa832c73fe82f46f8fef46f8ef3991ed21'
  /dolphinscheduler-python/pydolphinscheduler/tests/core:
-    newFile: !uuid '09d5e2b628d7df0938ba34cdfe4b4d68f8756ed6'
-    oldFile: !uuid 'd8f7d2eadeb08a9a47d90693141b18c54d603e80'
-  /dolphinscheduler-python/pydolphinscheduler/tests/core/test_engine.py:
-    newFile: !uuid 'ba44fad669d87460c9477df327b7634d75d95647'
-    oldFile: !uuid 'e36c47ba1b6451398f8563c8c75ecc33688ec8f0'
-  /dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py:
-    newFile: !uuid '3909077b987223508f608b3e2bca8c4404b4174a'
-    oldFile: !uuid '201bdb30cde592fdd93afc72f185ce6c620297bd'
-  /dolphinscheduler-python/pydolphinscheduler/tests/tasks:
-    newFile: !uuid '5eab6305883ef11c16e47a66f93178d5ac063df0'
-    oldFile: !uuid 'be2b7f87a1f0777fa70cbf155f2f7cfdfae56884'
-  /dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_condition.py:
-    newFile: !uuid '72eec28ed725c0e5a166a61be23492e68dee5f0a'
-    oldFile: !uuid '523264034a857a3d3fb8e854315d29fd4550c4f8'
-  /dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_datax.py:
-    newFile: !uuid '5d1890e83d074655a8f0f938538bc1e5b7a6deab'
-    oldFile: !uuid '9473f5732121df3a46d1aa4277287131b9e1ef09'
-  /dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_dependent.py:
-    newFile: !uuid 'f55700e04b6d76e9eb1db7d6eccb61480f75c538'
-    oldFile: !uuid 'f16e291c82313ca3673f8b781b539bd569c332a7'
-  /dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_flink.py:
-    newFile: !uuid '2f30a494b94ecbf039d0b5c3026f1463c03f5ce3'
-    oldFile: !uuid '92ae3ba91fac5fd9b73bb3ead522e06b2ee007cb'
-  /dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_http.py:
-    newFile: !uuid '399829b68c6d658314679b39464e4f0d0a4f1c71'
-    oldFile: !uuid '060cdec0b0d85735a954623039759f79df3a584d'
-  /dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_map_reduce.py:
-    newFile: !uuid '5d38e93aa40ec643d9e2a9bdf949da504d82fc76'
-    oldFile: !uuid 'dbe9e513f5c34f379dd1fcf428746d3b99b24809'
-  /dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_procedure.py:
-    newFile: !uuid '80afe7b8793204869f2aa3698965813869d6abdd'
-    oldFile: !uuid '17825939555efeb6f26d937e8266a0813824fa61'
-  /dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_python.py:
-    newFile: !uuid 'e8f7f10d773c440e7c57dc1404c4f1b0725a92c5'
-    oldFile: !uuid '1cdd85d2cbfab4cdcc09ea4dee8e354929d57d05'
-  /dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_sagemaker.py:
-    newFile: !uuid '20edc22805816037ffa709a564a11acc8a5c4c4a'
-    oldFile: !uuid '8838eaf49766cef53f897d708b3c2a8275918c7f'
-  /dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_shell.py:
-    newFile: !uuid 'e2c87d8e7d999d4b9a2c975c79245dc9da57bb8f'
-    oldFile: !uuid 'e42f6dc0fb9da1ae49781a1ed344b7f28c6ddd0c'
-  /dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_spark.py:
-    newFile: !uuid 'ed83f9f953f42af1f1671c0f8a7dfb5e1fa6585b'
-    oldFile: !uuid '3b0672f9634513ff04400719895e2fe246a3cb30'
-  /dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_sql.py:
-    newFile: !uuid 'ba9daa9b2d36fd77b44a43eec473760ad263d2f9'
-    oldFile: !uuid '50ccd946a9dad316919e443ea7ee767341e69095'
-  /dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_sub_process.py:
-    newFile: !uuid '126ab1015ebcb037e6eee253cab10bd9ef872d98'
-    oldFile: !uuid '7f471a1b8b258103899f80737eb5d0262738cbd4'
-  /dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_switch.py:
-    newFile: !uuid '3206b12f7ec331ec7399ebc7be5e2a92cff5d344'
-    oldFile: !uuid '1f6ff5bfa2dd05a20966db7ab77d40a67a4b4e5a'
-gitCommitId: !uuid '8988492c4329c93b4872ba3a3059a92582e99bda'
+    newFile: !uuid '3b49e8567946cf92ae37371b88d95eff204d5801'
+    oldFile: !uuid '09d5e2b628d7df0938ba34cdfe4b4d68f8756ed6'
+  /dolphinscheduler-python/pydolphinscheduler/tests/core/test_yaml_process_define.py:
+    newFile: !uuid '99ad179a5fecfda7d53310bba800c5647047c7b2'
+    oldFile: null
+  /dolphinscheduler-python/pydolphinscheduler/tests/testing:
+    newFile: !uuid '23d40b2b84d337a8a9ab882ff2307a39b3cc3b54'
+    oldFile: !uuid 'ee5357d281e94cd76ab9300857f44ce2fa43243b'
+  /dolphinscheduler-python/pydolphinscheduler/tests/testing/path.py:
+    newFile: !uuid '974ab3d47cf9ba25f833d5f94c0a87aaea64e8d9'
+    oldFile: !uuid '68d93c404b51accd0c65fbaca60206f299b91518'
+gitCommitId: !uuid '38ee91fb1edf9fc5a86384cbcade4da7d003687a'
 renames: {
  }
-revisionId: 3434
+revisionId: 3435