[deploy] Make Python API as independent directory in release's dist (#8470)

This patch add python api package as independent
directory in final dist directory when user run
command `mvn -U install package -Prelease`.

There have `tar.gz` and `whl` in the python dist directory

```
<the-old-dist-package>
python
  |--- python-independent-pkg.tar.gz
  |--- python-independent-pkg.whl
```

close: #8343

* Give dedicated control to docker build/push goals

Co-authored-by: kezhenxu94 <kezhenxu94@apache.org>

dolphinscheduler-dist/pom.xml

@@ -104,6 +104,22 @@
                                 </configuration>
                             </execution>
 
+                            <execution>
+                                <id>python</id>
+                                <phase>package</phase>
+                                <goals>
+                                    <goal>single</goal>
+                                </goals>
+                                <configuration>
+                                    <!-- Make final directory with simple name `python`, and without any addtion information -->
+                                    <finalName>python</finalName>
+                                    <appendAssemblyId>false</appendAssemblyId>
+                                    <descriptors>
+                                        <descriptor>src/main/assembly/dolphinscheduler-python-api.xml</descriptor>
+                                    </descriptors>
+                                </configuration>
+                            </execution>
+
                         </executions>
                     </plugin>
                 </plugins>

dolphinscheduler-dist/src/main/assembly/dolphinscheduler-python-api.xml

@@ -0,0 +1,34 @@
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<assembly
+        xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+    <id>python-api</id>
+    <formats>
+        <format>dir</format>
+    </formats>
+    <includeBaseDirectory>false</includeBaseDirectory>
+
+    <fileSets>
+        <fileSet>
+            <directory>${basedir}/../dolphinscheduler-python/pydolphinscheduler/dist</directory>
+            <outputDirectory>.</outputDirectory>
+        </fileSet>
+    </fileSets>
+</assembly>

dolphinscheduler-dist/src/main/assembly/dolphinscheduler-src.xml

@@ -57,6 +57,13 @@
                 <exclude>**/dolphinscheduler-ui/node/**</exclude>
                 <exclude>**/dolphinscheduler-ui/node_modules/**</exclude>
 
+                <!-- python ignore -->
+                <exclude>**/dolphinscheduler-python/pydolphinscheduler/.pytest_cache/**</exclude>
+                <exclude>**/dolphinscheduler-python/pydolphinscheduler/build/**</exclude>
+                <exclude>**/dolphinscheduler-python/pydolphinscheduler/dist/**</exclude>
+                <exclude>**/dolphinscheduler-python/pydolphinscheduler/dist/**</exclude>
+                <exclude>**/dolphinscheduler-python/pydolphinscheduler/htmlcov/**</exclude>
+
                 <!-- eclipse ignore -->
                 <exclude>**/.settings/**</exclude>
                 <exclude>**/.project</exclude>

dolphinscheduler-python/pom.xml

@@ -104,5 +104,67 @@
                 </plugins>
             </build>
         </profile>
+        <profile>
+            <id>release</id>
+            <build>
+                <plugins>
+                    <plugin>
+                        <groupId>org.codehaus.mojo</groupId>
+                        <artifactId>exec-maven-plugin</artifactId>
+                        <executions>
+                            <execution>
+                                <id>python-api-prepare</id>
+                                <phase>prepare-package</phase>
+                                <goals>
+                                    <goal>exec</goal>
+                                </goals>
+                                <configuration>
+                                    <executable>python3</executable>
+                                    <workingDirectory>${project.basedir}/pydolphinscheduler</workingDirectory>
+                                    <arguments>
+                                        <argument>-m</argument>
+                                        <argument>pip</argument>
+                                        <argument>install</argument>
+                                        <argument>--upgrade</argument>
+                                        <argument>pip</argument>
+                                        <argument>.[build]</argument>
+                                    </arguments>
+                                </configuration>
+                            </execution>
+                            <execution>
+                                <id>python-api-clean</id>
+                                <phase>prepare-package</phase>
+                                <goals>
+                                    <goal>exec</goal>
+                                </goals>
+                                <configuration>
+                                    <executable>python3</executable>
+                                    <workingDirectory>${project.basedir}/pydolphinscheduler</workingDirectory>
+                                    <arguments>
+                                        <argument>setup.py</argument>
+                                        <argument>pre_clean</argument>
+                                    </arguments>
+                                </configuration>
+                            </execution>
+                            <execution>
+                                <id>python-api-build</id>
+                                <phase>prepare-package</phase>
+                                <goals>
+                                    <goal>exec</goal>
+                                </goals>
+                                <configuration>
+                                    <executable>python3</executable>
+                                    <workingDirectory>${project.basedir}/pydolphinscheduler</workingDirectory>
+                                    <arguments>
+                                        <argument>-m</argument>
+                                        <argument>build</argument>
+                                    </arguments>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
     </profiles>
 </project>

dolphinscheduler-python/pydolphinscheduler/setup.py

@@ -16,18 +16,23 @@
 # under the License.
 
 """The script for setting up pydolphinscheduler."""
-
+import logging
+import os
 import sys
+from distutils.dir_util import remove_tree
 from os.path import dirname, join
+from typing import List
 
-from setuptools import find_packages, setup
+from setuptools import Command, find_packages, setup
 
 if sys.version_info[0] < 3:
     raise Exception(
         "pydolphinscheduler does not support Python 2. Please upgrade to Python 3."
     )
 
-version = "0.1.0"
+logger = logging.getLogger(__name__)
+
+version = "2.0.4"
 
 # Start package required
 prod = [
@@ -35,6 +40,12 @@ prod = [
     "py4j~=0.10",
 ]
 
+build = [
+    "build",
+    "setuptools>=42",
+    "wheel",
+]
+
 doc = [
     "sphinx>=4.3",
     "sphinx_rtd_theme>=1.0",
@@ -54,7 +65,7 @@ style = [
     "isort>=5.10",
 ]
 
-dev = style + test + doc
+dev = style + test + doc + build
 
 all_dep = prod + dev
 # End package required
@@ -67,6 +78,39 @@ def read(*names, **kwargs):
     ).read()
 
 
+class CleanCommand(Command):
+    """Command to clean up python api before setup by running `python setup.py pre_clean`."""
+
+    description = "Clean up project root"
+    user_options: List[str] = []
+    clean_list = [
+        "build",
+        "htmlcov",
+        "dist",
+        ".pytest_cache",
+        ".coverage",
+    ]
+
+    def initialize_options(self) -> None:
+        """Set default values for options."""
+        pass
+
+    def finalize_options(self) -> None:
+        """Set final values for options."""
+        pass
+
+    def run(self) -> None:
+        """Run and remove temporary files."""
+        for cl in self.clean_list:
+            if not os.path.exists(cl):
+                logger.info("Path %s do not exists.", cl)
+            elif os.path.isdir(cl):
+                remove_tree(cl)
+            else:
+                os.remove(cl)
+        logger.info("Finish pre_clean process.")
+
+
 setup(
     name="apache-dolphinscheduler",
     version=version,
@@ -126,6 +170,10 @@ setup(
         "style": style,
         "test": test,
         "doc": doc,
+        "build": build,
+    },
+    cmdclass={
+        "pre_clean": CleanCommand,
     },
     entry_points={
         "console_scripts": [

pom.xml

@@ -133,6 +133,8 @@
         <docker.hub>apache</docker.hub>
         <docker.repo>${project.name}</docker.repo>
         <docker.tag>${project.version}</docker.tag>
+        <docker.build.skip>true</docker.build.skip>
+        <docker.push.skip>true</docker.push.skip>
     </properties>
 
     <dependencyManagement>
@@ -1002,6 +1004,7 @@
                                 <goal>exec</goal>
                             </goals>
                             <configuration>
+                                <skip>${docker.build.skip}</skip>
                                 <environmentVariables>
                                     <DOCKER_BUILDKIT>1</DOCKER_BUILDKIT>
                                 </environmentVariables>
@@ -1026,6 +1029,7 @@
                                 <goal>exec</goal>
                             </goals>
                             <configuration>
+                                <skip>${docker.push.skip}</skip>
                                 <environmentVariables>
                                     <DOCKER_BUILDKIT>1</DOCKER_BUILDKIT>
                                 </environmentVariables>
@@ -1233,6 +1237,16 @@
         </plugins>
     </build>
 
+    <profiles>
+        <profile>
+            <id>docker</id>
+            <properties>
+                <docker.build.skip>false</docker.build.skip>
+                <docker.push.skip>false</docker.push.skip>
+            </properties>
+        </profile>
+    </profiles>
+
     <dependencies>
         <!--
           NOTE: only development / test phase dependencies (scope = test / provided)