[Feature] Add CURD to the project/tenant/user section of the python-DS (#11162)

- Add CURD in project
- Add CURD in tenant
- Add CURD in user
- Add test in user

Co-authored-by: Jiajie Zhong <zhongjiajie955@gmail.com>

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/python/PythonGateway.java

@@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.api.python;
 import java.io.IOException;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
+import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
@@ -44,6 +45,7 @@ import org.apache.dolphinscheduler.api.service.TenantService;
 import org.apache.dolphinscheduler.api.service.UsersService;
 import org.apache.dolphinscheduler.api.utils.Result;
 import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.common.enums.AuthorizationType;
 import org.apache.dolphinscheduler.common.enums.ComplementDependentMode;
 import org.apache.dolphinscheduler.common.enums.FailureStrategy;
 import org.apache.dolphinscheduler.common.enums.Priority;
@@ -55,6 +57,7 @@ import org.apache.dolphinscheduler.common.enums.TaskDependType;
 import org.apache.dolphinscheduler.common.enums.UserType;
 import org.apache.dolphinscheduler.common.enums.WarningType;
 import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
+import org.apache.dolphinscheduler.common.utils.PropertyUtils;
 import org.apache.dolphinscheduler.dao.entity.DataSource;
 import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
 import org.apache.dolphinscheduler.dao.entity.Project;
@@ -400,7 +403,7 @@ public class PythonGateway {
 
     public Project queryProjectByName(String userName, String projectName) {
         User user = usersService.queryUser(userName);
-        return (Project) projectService.queryByName(user, projectName);
+        return (Project) projectService.queryByName(user, projectName).get(Constants.DATA_LIST);
     }
 
     public void updateProject(String userName, Long projectCode, String projectName, String desc) {
@@ -417,9 +420,8 @@ public class PythonGateway {
         return tenantService.createTenantIfNotExists(tenantCode, desc, queueName, queueName);
     }
 
-    public Result queryTenantList(String userName, String searchVal, Integer pageNo, Integer pageSize) {
+    public Tenant queryTenantByCode(String tenantCode) {
-        User user = usersService.queryUser(userName);
+        return (Tenant) tenantService.queryByTenantCode(tenantCode).get(Constants.DATA_LIST);
-        return tenantService.queryTenantList(user, searchVal, pageNo, pageSize);
     }
 
     public void updateTenant(String userName, int id, String tenantCode, int queueId, String desc) throws Exception {
@@ -432,27 +434,32 @@ public class PythonGateway {
         tenantService.deleteTenantById(user, tenantId);
     }
 
-    public void createUser(String userName,
+    public User createUser(String userName,
                            String userPassword,
                            String email,
                            String phone,
                            String tenantCode,
                            String queue,
                            int state) throws IOException {
-        usersService.createUserIfNotExists(userName, userPassword, email, phone, tenantCode, queue, state);
+        return usersService.createUserIfNotExists(userName, userPassword, email, phone, tenantCode, queue, state);
     }
 
     public User queryUser(int id) {
-        return usersService.queryUser(id);
+        User user = usersService.queryUser(id);
+        if (user == null) {
+            throw new RuntimeException("User not found");
+        }
+        return user;
     }
 
-    public void updateUser(String userName, String userPassword, String email, String phone, String tenantCode, String queue, int state) throws Exception {
+    public User updateUser(String userName, String userPassword, String email, String phone, String tenantCode, String queue, int state) throws Exception {
-        usersService.createUserIfNotExists(userName, userPassword, email, phone, tenantCode, queue, state);
+        return usersService.createUserIfNotExists(userName, userPassword, email, phone, tenantCode, queue, state);
     }
 
-    public void deleteUser(String userName, int id) throws Exception {
+    public User deleteUser(String userName, int id) throws Exception {
         User user = usersService.queryUser(userName);
         usersService.deleteUserById(user, id);
+        return usersService.queryUser(userName);
     }
 
     /**

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java

@@ -1363,6 +1363,7 @@ public class UsersServiceImpl extends BaseServiceImpl implements UsersService {
         }
 
         updateUser(user, user.getId(), userName, userPassword, email, user.getTenantId(), phone, queue, state, null);
+        user = userMapper.queryDetailsById(user.getId());
         return user;
     }
 }

dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java

@@ -741,12 +741,14 @@ public class UsersServiceTest {
         String userName = "userTest0001";
         String userPassword = "userTest";
         String email = "abc@x.com";
-        String phone = "123456789";
+        String phone = "17366666666";
         String tenantCode = "tenantCode";
         int stat = 1;
 
         // User exists
         Mockito.when(userMapper.existUser(userName)).thenReturn(true);
+        Mockito.when(userMapper.selectById(getUser().getId())).thenReturn(getUser());
+        Mockito.when(userMapper.queryDetailsById(getUser().getId())).thenReturn(getUser());
         Mockito.when(userMapper.queryByUserNameAccurately(userName)).thenReturn(getUser());
         Mockito.when(tenantMapper.queryByTenantCode(tenantCode)).thenReturn(getTenant());
         user = usersService.createUserIfNotExists(userName, userPassword, email, phone, tenantCode, queueName, stat);

dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/java_gateway.py

@@ -118,6 +118,22 @@ class JavaGate:
             user, name, description
         )
 
+    def query_project_by_name(self, user: str, name: str):
+        """Query project through java gateway."""
+        return self.java_gateway.entry_point.queryProjectByName(user, name)
+
+    def update_project(
+        self, user: str, project_code: int, project_name: str, description: str
+    ):
+        """Update project through java gateway."""
+        return self.java_gateway.entry_point.updateProject(
+            user, project_code, project_name, description
+        )
+
+    def delete_project(self, user: str, code: int):
+        """Delete project through java gateway."""
+        return self.java_gateway.entry_point.deleteProject(user, code)
+
     def create_tenant(
         self, tenant_name: str, queue_name: str, description: Optional[str] = None
     ):
@@ -126,6 +142,31 @@ class JavaGate:
             tenant_name, description, queue_name
         )
 
+    def query_tenant(self, tenant_code: str):
+        """Query tenant through java gateway."""
+        return self.java_gateway.entry_point.queryTenantByCode(tenant_code)
+
+    def grant_tenant_to_user(self, user_name: str, tenant_code: str):
+        """Grant tenant to user through java gateway."""
+        return self.java_gateway.entry_point.grantTenantToUser(user_name, tenant_code)
+
+    def update_tenant(
+        self,
+        user: str,
+        tenant_id: int,
+        code: str,
+        queue_id: int,
+        description: Optional[str] = None,
+    ):
+        """Update tenant through java gateway."""
+        return self.java_gateway.entry_point.updateTenant(
+            user, tenant_id, code, queue_id, description
+        )
+
+    def delete_tenant(self, user: str, tenant_id: int):
+        """Delete tenant through java gateway."""
+        return self.java_gateway.entry_point.deleteTenantById(user, tenant_id)
+
     def create_user(
         self,
         name: str,
@@ -141,6 +182,29 @@ class JavaGate:
             name, password, email, phone, tenant, queue, status
         )
 
+    def query_user(self, user_id: int):
+        """Query user through java gateway."""
+        return self.java_gateway.queryUser(user_id)
+
+    def update_user(
+        self,
+        name: str,
+        password: str,
+        email: str,
+        phone: str,
+        tenant: str,
+        queue: str,
+        status: int,
+    ):
+        """Update user through java gateway."""
+        return self.java_gateway.entry_point.updateUser(
+            name, password, email, phone, tenant, queue, status
+        )
+
+    def delete_user(self, name: str, user_id: int):
+        """Delete user through java gateway."""
+        return self.java_gateway.entry_point.deleteUser(name, user_id)
+
     def get_dependent_info(
         self,
         project_name: str,

dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/base_side.py

@@ -38,3 +38,11 @@ class BaseSide(Base):
     ):
         """Create Base if not exists."""
         raise NotImplementedError
+
+    def delete_all(self):
+        """Delete all method."""
+        if not self:
+            return
+        list_pro = [key for key in self.__dict__.keys()]
+        for key in list_pro:
+            self.__delattr__(key)

dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/project.py

@@ -31,11 +31,42 @@ class Project(BaseSide):
         self,
         name: str = configuration.WORKFLOW_PROJECT,
         description: Optional[str] = None,
+        code: Optional[int] = None,
     ):
         super().__init__(name, description)
+        self.code = code
 
     def create_if_not_exists(self, user=configuration.USER_NAME) -> None:
         """Create Project if not exists."""
         JavaGate().create_or_grant_project(user, self.name, self.description)
         # TODO recover result checker
         # gateway_result_checker(result, None)
+
+    @classmethod
+    def get_project_by_name(cls, user=configuration.USER_NAME, name=None) -> "Project":
+        """Get Project by name."""
+        project = JavaGate().query_project_by_name(user, name)
+        if project is None:
+            return cls()
+        return cls(
+            name=project.getName(),
+            description=project.getDescription(),
+            code=project.getCode(),
+        )
+
+    def update(
+        self,
+        user=configuration.USER_NAME,
+        project_code=None,
+        project_name=None,
+        description=None,
+    ) -> None:
+        """Update Project."""
+        JavaGate().update_project(user, project_code, project_name, description)
+        self.name = project_name
+        self.description = description
+
+    def delete(self, user=configuration.USER_NAME) -> None:
+        """Delete Project."""
+        JavaGate().delete_project(user, self.code)
+        self.delete_all()

dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/tenant.py

@@ -32,13 +32,49 @@ class Tenant(BaseSide):
         name: str = configuration.WORKFLOW_TENANT,
         queue: str = configuration.WORKFLOW_QUEUE,
         description: Optional[str] = None,
+        tenant_id: Optional[int] = None,
+        code: Optional[str] = None,
+        user_name: Optional[str] = None,
     ):
         super().__init__(name, description)
+        self.tenant_id = tenant_id
         self.queue = queue
+        self.code = code
+        self.user_name = user_name
 
     def create_if_not_exists(
         self, queue_name: str, user=configuration.USER_NAME
     ) -> None:
         """Create Tenant if not exists."""
-        JavaGate().create_tenant(self.name, queue_name, self.description)
+        tenant = JavaGate().create_tenant(self.name, self.description, queue_name)
+        self.tenant_id = tenant.getId()
+        self.code = tenant.getTenantCode()
         # gateway_result_checker(result, None)
+
+    @classmethod
+    def get_tenant(cls, code: str) -> "Tenant":
+        """Get Tenant list."""
+        tenant = JavaGate().query_tenant(code)
+        if tenant is None:
+            return cls()
+        return cls(
+            description=tenant.getDescription(),
+            tenant_id=tenant.getId(),
+            code=tenant.getTenantCode(),
+            queue=tenant.getQueueId(),
+        )
+
+    def update(
+        self, user=configuration.USER_NAME, code=None, queue_id=None, description=None
+    ) -> None:
+        """Update Tenant."""
+        JavaGate().update_tenant(user, self.tenant_id, code, queue_id, description)
+        # TODO: check queue_id and queue_name
+        self.queue = str(queue_id)
+        self.code = code
+        self.description = description
+
+    def delete(self) -> None:
+        """Delete Tenant."""
+        JavaGate().delete_tenant(self.user_name, self.tenant_id)
+        self.delete_all()

dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/user.py

@@ -48,6 +48,7 @@ class User(BaseSide):
         status: Optional[int] = configuration.USER_STATE,
     ):
         super().__init__(name)
+        self.user_id: Optional[int] = None
         self.password = password
         self.email = email
         self.phone = phone
@@ -64,7 +65,7 @@ class User(BaseSide):
         """Create User if not exists."""
         # Should make sure queue already exists.
         self.create_tenant_if_not_exists()
-        JavaGate().create_user(
+        user = JavaGate().create_user(
             self.name,
             self.password,
             self.email,
@@ -73,5 +74,57 @@ class User(BaseSide):
             self.queue,
             self.status,
         )
+        self.user_id = user.getId()
         # TODO recover result checker
         # gateway_result_checker(result, None)
+
+    @classmethod
+    def get_user(cls, user_id) -> "User":
+        """Get User."""
+        user = JavaGate().query_user(user_id)
+        if user is None:
+            return cls("")
+        user_id = user.getId()
+        user = cls(
+            name=user.getUserName(),
+            password=user.getUserPassword(),
+            email=user.getEmail(),
+            phone=user.getPhone(),
+            tenant=user.getTenantCode(),
+            queue=user.getQueueName(),
+            status=user.getState(),
+        )
+        user.user_id = user_id
+        return user
+
+    def update(
+        self,
+        password=None,
+        email=None,
+        phone=None,
+        tenant=None,
+        queue=None,
+        status=None,
+    ) -> None:
+        """Update User."""
+        user = JavaGate().update_user(
+            self.name,
+            password,
+            email,
+            phone,
+            tenant,
+            queue,
+            status,
+        )
+        self.user_id = user.getId()
+        self.name = user.getUserName()
+        self.password = user.getUserPassword()
+        self.email = user.getEmail()
+        self.phone = user.getPhone()
+        self.queue = user.getQueueName()
+        self.status = user.getState()
+
+    def delete(self) -> None:
+        """Delete User."""
+        JavaGate().delete_user(self.name, self.user_id)
+        self.delete_all()

dolphinscheduler-python/pydolphinscheduler/tests/integration/conftest.py

@@ -42,7 +42,7 @@ def docker_setup_teardown():
             image="apache/dolphinscheduler-standalone-server:ci",
             container_name="ci-dolphinscheduler-standalone-server",
         )
-        ports = {"25333/tcp": 25333}
+        ports = {"25333/tcp": 25333, "12345/tcp": 12345}
         container = docker_wrapper.run_until_log(
             log="Started StandaloneServer in", tty=True, ports=ports
         )

dolphinscheduler-python/pydolphinscheduler/tests/integration/test_project.py

@@ -0,0 +1,78 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Test pydolphinscheduler project."""
+import pytest
+
+from pydolphinscheduler.models import Project, User
+
+
+def get_user(
+    name="test-name",
+    password="test-password",
+    email="test-email@abc.com",
+    phone="17366637777",
+    tenant="test-tenant",
+    queue="test-queue",
+    status=1,
+):
+    """Get a test user."""
+    user = User(name, password, email, phone, tenant, queue, status)
+    user.create_if_not_exists()
+    return user
+
+
+def get_project(name="test-name-1", description="test-description", code=1):
+    """Get a test project."""
+    project = Project(name, description, code=code)
+    user = get_user()
+    project.create_if_not_exists(user=user.name)
+    return project
+
+
+def test_create_and_get_project():
+    """Test create and get project from java gateway."""
+    project = get_project()
+    project_ = Project.get_project_by_name(user="test-name", name=project.name)
+    assert project_.name == project.name
+    assert project_.description == project.description
+
+
+def test_update_project():
+    """Test update project from java gateway."""
+    project = get_project()
+    project = project.get_project_by_name(user="test-name", name=project.name)
+    project.update(
+        user="test-name",
+        project_code=project.code,
+        project_name="test-name-updated",
+        description="test-description-updated",
+    )
+    project_ = Project.get_project_by_name(user="test-name", name="test-name-updated")
+    assert project_.description == "test-description-updated"
+
+
+def test_delete_project():
+    """Test delete project from java gateway."""
+    project = get_project()
+    project.get_project_by_name(user="test-name", name=project.name)
+    project.delete(user="test-name")
+
+    with pytest.raises(AttributeError) as excinfo:
+        _ = project.name
+
+    assert excinfo.type == AttributeError

dolphinscheduler-python/pydolphinscheduler/tests/integration/test_tenant.py

@@ -0,0 +1,86 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Test pydolphinscheduler tenant."""
+import pytest
+
+from pydolphinscheduler.models import Tenant, User
+
+
+def get_user(
+    name="test-name",
+    password="test-password",
+    email="test-email@abc.com",
+    phone="17366637777",
+    tenant="test-tenant",
+    queue="test-queue",
+    status=1,
+):
+    """Get a test user."""
+    user = User(name, password, email, phone, tenant, queue, status)
+    user.create_if_not_exists()
+    return user
+
+
+def get_tenant(
+    name="test-name-1",
+    queue="test-queue-1",
+    description="test-description",
+    tenant_code="test-tenant-code",
+    user_name=None,
+):
+    """Get a test tenant."""
+    tenant = Tenant(name, queue, description, code=tenant_code, user_name=user_name)
+    tenant.create_if_not_exists(name)
+    return tenant
+
+
+def test_create_tenant():
+    """Test create tenant from java gateway."""
+    tenant = get_tenant()
+    assert tenant.tenant_id is not None
+
+
+def test_get_tenant():
+    """Test get tenant from java gateway."""
+    tenant = get_tenant()
+    tenant_ = Tenant.get_tenant(tenant.code)
+    assert tenant_.tenant_id == tenant.tenant_id
+
+
+def test_update_tenant():
+    """Test update tenant from java gateway."""
+    tenant = get_tenant(user_name="admin")
+    tenant.update(
+        user="admin",
+        code="test-code-updated",
+        queue_id=1,
+        description="test-description-updated",
+    )
+    tenant_ = Tenant.get_tenant(code=tenant.code)
+    assert tenant_.code == "test-code-updated"
+    assert tenant_.queue == 1
+
+
+def test_delete_tenant():
+    """Test delete tenant from java gateway."""
+    tenant = get_tenant(user_name="admin")
+    tenant.delete()
+    with pytest.raises(AttributeError) as excinfo:
+        _ = tenant.tenant_id
+
+    assert excinfo.type == AttributeError

dolphinscheduler-python/pydolphinscheduler/tests/integration/test_user.py

@@ -0,0 +1,107 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Test pydolphinscheduler user."""
+
+import hashlib
+
+import pytest
+
+from pydolphinscheduler.models import User
+
+
+def md5(str):
+    """MD5 a string."""
+    hl = hashlib.md5()
+    hl.update(str.encode(encoding="utf-8"))
+    return hl.hexdigest()
+
+
+def get_user(
+    name="test-name",
+    password="test-password",
+    email="test-email@abc.com",
+    phone="17366637777",
+    tenant="test-tenant",
+    queue="test-queue",
+    status=1,
+):
+    """Get a test user."""
+    user = User(
+        name=name,
+        password=password,
+        email=email,
+        phone=phone,
+        tenant=tenant,
+        queue=queue,
+        status=status,
+    )
+    user.create_if_not_exists()
+    return user
+
+
+def test_create_user():
+    """Test weather client could connect java gate way or not."""
+    user = User(
+        name="test-name",
+        password="test-password",
+        email="test-email@abc.com",
+        phone="17366637777",
+        tenant="test-tenant",
+        queue="test-queue",
+        status=1,
+    )
+    user.create_if_not_exists()
+    assert user.user_id is not None
+
+
+def test_get_user():
+    """Test get user from java gateway."""
+    user = get_user()
+    user_ = User.get_user(user.user_id)
+    assert user_.password == md5(user.password)
+    assert user_.email == user.email
+    assert user_.phone == user.phone
+    assert user_.status == user.status
+
+
+def test_update_user():
+    """Test update user from java gateway."""
+    user = get_user()
+    user.update(
+        password="test-password-",
+        email="test-email-updated@abc.com",
+        phone="17366637766",
+        tenant="test-tenant-updated",
+        queue="test-queue-updated",
+        status=2,
+    )
+    user_ = User.get_user(user.user_id)
+    assert user_.password == md5("test-password-")
+    assert user_.email == "test-email-updated@abc.com"
+    assert user_.phone == "17366637766"
+    assert user_.status == 2
+
+
+def test_delete_user():
+    """Test delete user from java gateway."""
+    user = get_user()
+    user.delete()
+    with pytest.raises(AttributeError) as excinfo:
+        _ = user.user_id
+
+    assert excinfo.type == AttributeError