Browse Source

[Feature] Add CURD to the project/tenant/user section of the python-DS (#11162)

- Add CURD in project
- Add CURD in tenant
- Add CURD in user
- Add test in user

Co-authored-by: Jiajie Zhong <zhongjiajie955@gmail.com>
3.2.0-release
Lyle Shaw 2 years ago committed by GitHub
parent
commit
cc492c3e13
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 27
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/python/PythonGateway.java
  2. 1
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java
  3. 4
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java
  4. 64
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/java_gateway.py
  5. 8
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/base_side.py
  6. 31
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/project.py
  7. 38
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/tenant.py
  8. 55
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/user.py
  9. 2
      dolphinscheduler-python/pydolphinscheduler/tests/integration/conftest.py
  10. 78
      dolphinscheduler-python/pydolphinscheduler/tests/integration/test_project.py
  11. 86
      dolphinscheduler-python/pydolphinscheduler/tests/integration/test_tenant.py
  12. 107
      dolphinscheduler-python/pydolphinscheduler/tests/integration/test_user.py

27
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/python/PythonGateway.java

@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.api.python;
import java.io.IOException;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
@ -44,6 +45,7 @@ import org.apache.dolphinscheduler.api.service.TenantService;
import org.apache.dolphinscheduler.api.service.UsersService;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.AuthorizationType;
import org.apache.dolphinscheduler.common.enums.ComplementDependentMode;
import org.apache.dolphinscheduler.common.enums.FailureStrategy;
import org.apache.dolphinscheduler.common.enums.Priority;
@ -55,6 +57,7 @@ import org.apache.dolphinscheduler.common.enums.TaskDependType;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.enums.WarningType;
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.dao.entity.DataSource;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.Project;
@ -400,7 +403,7 @@ public class PythonGateway {
public Project queryProjectByName(String userName, String projectName) {
User user = usersService.queryUser(userName);
return (Project) projectService.queryByName(user, projectName);
return (Project) projectService.queryByName(user, projectName).get(Constants.DATA_LIST);
}
public void updateProject(String userName, Long projectCode, String projectName, String desc) {
@ -417,9 +420,8 @@ public class PythonGateway {
return tenantService.createTenantIfNotExists(tenantCode, desc, queueName, queueName);
}
public Result queryTenantList(String userName, String searchVal, Integer pageNo, Integer pageSize) {
User user = usersService.queryUser(userName);
return tenantService.queryTenantList(user, searchVal, pageNo, pageSize);
public Tenant queryTenantByCode(String tenantCode) {
return (Tenant) tenantService.queryByTenantCode(tenantCode).get(Constants.DATA_LIST);
}
public void updateTenant(String userName, int id, String tenantCode, int queueId, String desc) throws Exception {
@ -432,27 +434,32 @@ public class PythonGateway {
tenantService.deleteTenantById(user, tenantId);
}
public void createUser(String userName,
public User createUser(String userName,
String userPassword,
String email,
String phone,
String tenantCode,
String queue,
int state) throws IOException {
usersService.createUserIfNotExists(userName, userPassword, email, phone, tenantCode, queue, state);
return usersService.createUserIfNotExists(userName, userPassword, email, phone, tenantCode, queue, state);
}
public User queryUser(int id) {
return usersService.queryUser(id);
User user = usersService.queryUser(id);
if (user == null) {
throw new RuntimeException("User not found");
}
return user;
}
public void updateUser(String userName, String userPassword, String email, String phone, String tenantCode, String queue, int state) throws Exception {
usersService.createUserIfNotExists(userName, userPassword, email, phone, tenantCode, queue, state);
public User updateUser(String userName, String userPassword, String email, String phone, String tenantCode, String queue, int state) throws Exception {
return usersService.createUserIfNotExists(userName, userPassword, email, phone, tenantCode, queue, state);
}
public void deleteUser(String userName, int id) throws Exception {
public User deleteUser(String userName, int id) throws Exception {
User user = usersService.queryUser(userName);
usersService.deleteUserById(user, id);
return usersService.queryUser(userName);
}
/**

1
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java

@ -1363,6 +1363,7 @@ public class UsersServiceImpl extends BaseServiceImpl implements UsersService {
}
updateUser(user, user.getId(), userName, userPassword, email, user.getTenantId(), phone, queue, state, null);
user = userMapper.queryDetailsById(user.getId());
return user;
}
}

4
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java

@ -741,12 +741,14 @@ public class UsersServiceTest {
String userName = "userTest0001";
String userPassword = "userTest";
String email = "abc@x.com";
String phone = "123456789";
String phone = "17366666666";
String tenantCode = "tenantCode";
int stat = 1;
// User exists
Mockito.when(userMapper.existUser(userName)).thenReturn(true);
Mockito.when(userMapper.selectById(getUser().getId())).thenReturn(getUser());
Mockito.when(userMapper.queryDetailsById(getUser().getId())).thenReturn(getUser());
Mockito.when(userMapper.queryByUserNameAccurately(userName)).thenReturn(getUser());
Mockito.when(tenantMapper.queryByTenantCode(tenantCode)).thenReturn(getTenant());
user = usersService.createUserIfNotExists(userName, userPassword, email, phone, tenantCode, queueName, stat);

64
dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/java_gateway.py

@ -118,6 +118,22 @@ class JavaGate:
user, name, description
)
def query_project_by_name(self, user: str, name: str):
"""Query project through java gateway."""
return self.java_gateway.entry_point.queryProjectByName(user, name)
def update_project(
self, user: str, project_code: int, project_name: str, description: str
):
"""Update project through java gateway."""
return self.java_gateway.entry_point.updateProject(
user, project_code, project_name, description
)
def delete_project(self, user: str, code: int):
"""Delete project through java gateway."""
return self.java_gateway.entry_point.deleteProject(user, code)
def create_tenant(
self, tenant_name: str, queue_name: str, description: Optional[str] = None
):
@ -126,6 +142,31 @@ class JavaGate:
tenant_name, description, queue_name
)
def query_tenant(self, tenant_code: str):
"""Query tenant through java gateway."""
return self.java_gateway.entry_point.queryTenantByCode(tenant_code)
def grant_tenant_to_user(self, user_name: str, tenant_code: str):
"""Grant tenant to user through java gateway."""
return self.java_gateway.entry_point.grantTenantToUser(user_name, tenant_code)
def update_tenant(
self,
user: str,
tenant_id: int,
code: str,
queue_id: int,
description: Optional[str] = None,
):
"""Update tenant through java gateway."""
return self.java_gateway.entry_point.updateTenant(
user, tenant_id, code, queue_id, description
)
def delete_tenant(self, user: str, tenant_id: int):
"""Delete tenant through java gateway."""
return self.java_gateway.entry_point.deleteTenantById(user, tenant_id)
def create_user(
self,
name: str,
@ -141,6 +182,29 @@ class JavaGate:
name, password, email, phone, tenant, queue, status
)
def query_user(self, user_id: int):
"""Query user through java gateway."""
return self.java_gateway.queryUser(user_id)
def update_user(
self,
name: str,
password: str,
email: str,
phone: str,
tenant: str,
queue: str,
status: int,
):
"""Update user through java gateway."""
return self.java_gateway.entry_point.updateUser(
name, password, email, phone, tenant, queue, status
)
def delete_user(self, name: str, user_id: int):
"""Delete user through java gateway."""
return self.java_gateway.entry_point.deleteUser(name, user_id)
def get_dependent_info(
self,
project_name: str,

8
dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/base_side.py

@ -38,3 +38,11 @@ class BaseSide(Base):
):
"""Create Base if not exists."""
raise NotImplementedError
def delete_all(self):
"""Delete all method."""
if not self:
return
list_pro = [key for key in self.__dict__.keys()]
for key in list_pro:
self.__delattr__(key)

31
dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/project.py

@ -31,11 +31,42 @@ class Project(BaseSide):
self,
name: str = configuration.WORKFLOW_PROJECT,
description: Optional[str] = None,
code: Optional[int] = None,
):
super().__init__(name, description)
self.code = code
def create_if_not_exists(self, user=configuration.USER_NAME) -> None:
"""Create Project if not exists."""
JavaGate().create_or_grant_project(user, self.name, self.description)
# TODO recover result checker
# gateway_result_checker(result, None)
@classmethod
def get_project_by_name(cls, user=configuration.USER_NAME, name=None) -> "Project":
"""Get Project by name."""
project = JavaGate().query_project_by_name(user, name)
if project is None:
return cls()
return cls(
name=project.getName(),
description=project.getDescription(),
code=project.getCode(),
)
def update(
self,
user=configuration.USER_NAME,
project_code=None,
project_name=None,
description=None,
) -> None:
"""Update Project."""
JavaGate().update_project(user, project_code, project_name, description)
self.name = project_name
self.description = description
def delete(self, user=configuration.USER_NAME) -> None:
"""Delete Project."""
JavaGate().delete_project(user, self.code)
self.delete_all()

38
dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/tenant.py

@ -32,13 +32,49 @@ class Tenant(BaseSide):
name: str = configuration.WORKFLOW_TENANT,
queue: str = configuration.WORKFLOW_QUEUE,
description: Optional[str] = None,
tenant_id: Optional[int] = None,
code: Optional[str] = None,
user_name: Optional[str] = None,
):
super().__init__(name, description)
self.tenant_id = tenant_id
self.queue = queue
self.code = code
self.user_name = user_name
def create_if_not_exists(
self, queue_name: str, user=configuration.USER_NAME
) -> None:
"""Create Tenant if not exists."""
JavaGate().create_tenant(self.name, queue_name, self.description)
tenant = JavaGate().create_tenant(self.name, self.description, queue_name)
self.tenant_id = tenant.getId()
self.code = tenant.getTenantCode()
# gateway_result_checker(result, None)
@classmethod
def get_tenant(cls, code: str) -> "Tenant":
"""Get Tenant list."""
tenant = JavaGate().query_tenant(code)
if tenant is None:
return cls()
return cls(
description=tenant.getDescription(),
tenant_id=tenant.getId(),
code=tenant.getTenantCode(),
queue=tenant.getQueueId(),
)
def update(
self, user=configuration.USER_NAME, code=None, queue_id=None, description=None
) -> None:
"""Update Tenant."""
JavaGate().update_tenant(user, self.tenant_id, code, queue_id, description)
# TODO: check queue_id and queue_name
self.queue = str(queue_id)
self.code = code
self.description = description
def delete(self) -> None:
"""Delete Tenant."""
JavaGate().delete_tenant(self.user_name, self.tenant_id)
self.delete_all()

55
dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/user.py

@ -48,6 +48,7 @@ class User(BaseSide):
status: Optional[int] = configuration.USER_STATE,
):
super().__init__(name)
self.user_id: Optional[int] = None
self.password = password
self.email = email
self.phone = phone
@ -64,7 +65,7 @@ class User(BaseSide):
"""Create User if not exists."""
# Should make sure queue already exists.
self.create_tenant_if_not_exists()
JavaGate().create_user(
user = JavaGate().create_user(
self.name,
self.password,
self.email,
@ -73,5 +74,57 @@ class User(BaseSide):
self.queue,
self.status,
)
self.user_id = user.getId()
# TODO recover result checker
# gateway_result_checker(result, None)
@classmethod
def get_user(cls, user_id) -> "User":
"""Get User."""
user = JavaGate().query_user(user_id)
if user is None:
return cls("")
user_id = user.getId()
user = cls(
name=user.getUserName(),
password=user.getUserPassword(),
email=user.getEmail(),
phone=user.getPhone(),
tenant=user.getTenantCode(),
queue=user.getQueueName(),
status=user.getState(),
)
user.user_id = user_id
return user
def update(
self,
password=None,
email=None,
phone=None,
tenant=None,
queue=None,
status=None,
) -> None:
"""Update User."""
user = JavaGate().update_user(
self.name,
password,
email,
phone,
tenant,
queue,
status,
)
self.user_id = user.getId()
self.name = user.getUserName()
self.password = user.getUserPassword()
self.email = user.getEmail()
self.phone = user.getPhone()
self.queue = user.getQueueName()
self.status = user.getState()
def delete(self) -> None:
"""Delete User."""
JavaGate().delete_user(self.name, self.user_id)
self.delete_all()

2
dolphinscheduler-python/pydolphinscheduler/tests/integration/conftest.py

@ -42,7 +42,7 @@ def docker_setup_teardown():
image="apache/dolphinscheduler-standalone-server:ci",
container_name="ci-dolphinscheduler-standalone-server",
)
ports = {"25333/tcp": 25333}
ports = {"25333/tcp": 25333, "12345/tcp": 12345}
container = docker_wrapper.run_until_log(
log="Started StandaloneServer in", tty=True, ports=ports
)

78
dolphinscheduler-python/pydolphinscheduler/tests/integration/test_project.py

@ -0,0 +1,78 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""Test pydolphinscheduler project."""
import pytest
from pydolphinscheduler.models import Project, User
def get_user(
name="test-name",
password="test-password",
email="test-email@abc.com",
phone="17366637777",
tenant="test-tenant",
queue="test-queue",
status=1,
):
"""Get a test user."""
user = User(name, password, email, phone, tenant, queue, status)
user.create_if_not_exists()
return user
def get_project(name="test-name-1", description="test-description", code=1):
"""Get a test project."""
project = Project(name, description, code=code)
user = get_user()
project.create_if_not_exists(user=user.name)
return project
def test_create_and_get_project():
"""Test create and get project from java gateway."""
project = get_project()
project_ = Project.get_project_by_name(user="test-name", name=project.name)
assert project_.name == project.name
assert project_.description == project.description
def test_update_project():
"""Test update project from java gateway."""
project = get_project()
project = project.get_project_by_name(user="test-name", name=project.name)
project.update(
user="test-name",
project_code=project.code,
project_name="test-name-updated",
description="test-description-updated",
)
project_ = Project.get_project_by_name(user="test-name", name="test-name-updated")
assert project_.description == "test-description-updated"
def test_delete_project():
"""Test delete project from java gateway."""
project = get_project()
project.get_project_by_name(user="test-name", name=project.name)
project.delete(user="test-name")
with pytest.raises(AttributeError) as excinfo:
_ = project.name
assert excinfo.type == AttributeError

86
dolphinscheduler-python/pydolphinscheduler/tests/integration/test_tenant.py

@ -0,0 +1,86 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""Test pydolphinscheduler tenant."""
import pytest
from pydolphinscheduler.models import Tenant, User
def get_user(
name="test-name",
password="test-password",
email="test-email@abc.com",
phone="17366637777",
tenant="test-tenant",
queue="test-queue",
status=1,
):
"""Get a test user."""
user = User(name, password, email, phone, tenant, queue, status)
user.create_if_not_exists()
return user
def get_tenant(
name="test-name-1",
queue="test-queue-1",
description="test-description",
tenant_code="test-tenant-code",
user_name=None,
):
"""Get a test tenant."""
tenant = Tenant(name, queue, description, code=tenant_code, user_name=user_name)
tenant.create_if_not_exists(name)
return tenant
def test_create_tenant():
"""Test create tenant from java gateway."""
tenant = get_tenant()
assert tenant.tenant_id is not None
def test_get_tenant():
"""Test get tenant from java gateway."""
tenant = get_tenant()
tenant_ = Tenant.get_tenant(tenant.code)
assert tenant_.tenant_id == tenant.tenant_id
def test_update_tenant():
"""Test update tenant from java gateway."""
tenant = get_tenant(user_name="admin")
tenant.update(
user="admin",
code="test-code-updated",
queue_id=1,
description="test-description-updated",
)
tenant_ = Tenant.get_tenant(code=tenant.code)
assert tenant_.code == "test-code-updated"
assert tenant_.queue == 1
def test_delete_tenant():
"""Test delete tenant from java gateway."""
tenant = get_tenant(user_name="admin")
tenant.delete()
with pytest.raises(AttributeError) as excinfo:
_ = tenant.tenant_id
assert excinfo.type == AttributeError

107
dolphinscheduler-python/pydolphinscheduler/tests/integration/test_user.py

@ -0,0 +1,107 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""Test pydolphinscheduler user."""
import hashlib
import pytest
from pydolphinscheduler.models import User
def md5(str):
"""MD5 a string."""
hl = hashlib.md5()
hl.update(str.encode(encoding="utf-8"))
return hl.hexdigest()
def get_user(
name="test-name",
password="test-password",
email="test-email@abc.com",
phone="17366637777",
tenant="test-tenant",
queue="test-queue",
status=1,
):
"""Get a test user."""
user = User(
name=name,
password=password,
email=email,
phone=phone,
tenant=tenant,
queue=queue,
status=status,
)
user.create_if_not_exists()
return user
def test_create_user():
"""Test weather client could connect java gate way or not."""
user = User(
name="test-name",
password="test-password",
email="test-email@abc.com",
phone="17366637777",
tenant="test-tenant",
queue="test-queue",
status=1,
)
user.create_if_not_exists()
assert user.user_id is not None
def test_get_user():
"""Test get user from java gateway."""
user = get_user()
user_ = User.get_user(user.user_id)
assert user_.password == md5(user.password)
assert user_.email == user.email
assert user_.phone == user.phone
assert user_.status == user.status
def test_update_user():
"""Test update user from java gateway."""
user = get_user()
user.update(
password="test-password-",
email="test-email-updated@abc.com",
phone="17366637766",
tenant="test-tenant-updated",
queue="test-queue-updated",
status=2,
)
user_ = User.get_user(user.user_id)
assert user_.password == md5("test-password-")
assert user_.email == "test-email-updated@abc.com"
assert user_.phone == "17366637766"
assert user_.status == 2
def test_delete_user():
"""Test delete user from java gateway."""
user = get_user()
user.delete()
with pytest.raises(AttributeError) as excinfo:
_ = user.user_id
assert excinfo.type == AttributeError
Loading…
Cancel
Save