chenrj
2 years ago
committed by
GitHub
19 changed files with 839 additions and 19 deletions
@ -0,0 +1,46 @@
|
||||
.. Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
.. http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
.. Unless required by applicable law or agreed to in writing, |
||||
software distributed under the License is distributed on an |
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
KIND, either express or implied. See the License for the |
||||
specific language governing permissions and limitations |
||||
under the License. |
||||
|
||||
GitLab |
||||
====== |
||||
|
||||
`GitLab` is a gitlab resource plugin for pydolphinscheduler. |
||||
|
||||
When using a gitlab resource plugin, you only need to add the `resource_plugin` parameter in the task subclass or workflow definition, |
||||
such as `resource_plugin=GitLab(prefix="xxx")`, if it is a public repository. |
||||
|
||||
If it is a private or Internal repository, you can use three ways to obtain authentication. |
||||
|
||||
The first is `Personal Access Tokens`, using `resource_plugin=GitLab(prefix="xxx", private_token="xxx")`. |
||||
|
||||
The second method is to obtain authentication through `username` and `password`: |
||||
|
||||
using `resource_plugin=GitLab(prefix="xxx", username="username", password="pwd")`. |
||||
|
||||
The third method is to obtain authentication through `OAuth Token`: |
||||
|
||||
using `resource_plugin=GitLab(prefix="xxx", oauth_token="xx")`. |
||||
|
||||
You can view this `document <https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html#create-a-personal-access-token>`_ |
||||
when creating a `Personal Access Tokens`. |
||||
|
||||
For the specific use of resource plugins, you can see `How to use` in :doc:`resource-plugin` |
||||
|
||||
Dive Into |
||||
--------- |
||||
|
||||
.. automodule:: pydolphinscheduler.resources_plugin.gitlab |
||||
@ -0,0 +1,44 @@
|
||||
.. Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
.. http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
.. Unless required by applicable law or agreed to in writing, |
||||
software distributed under the License is distributed on an |
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
KIND, either express or implied. See the License for the |
||||
specific language governing permissions and limitations |
||||
under the License. |
||||
|
||||
OSS |
||||
=== |
||||
|
||||
`OSS` is a Aliyun OSS resource plugin for pydolphinscheduler. |
||||
|
||||
When using a OSS resource plugin, you only need to add the `resource_plugin` parameter in the task subclass or workflow definition, |
||||
such as `resource_plugin=OSS(prefix="xxx")`, if the file is publicly readable. |
||||
|
||||
When the file is private, using `resource_plugin=OSS(prefix="xxx", access_key_id="xxx", access_key_secret="xxx")` |
||||
|
||||
Notice |
||||
The read permission of files in a bucket is inherited from the bucket by default. In other words, if the bucket is private, |
||||
the files in it are also private. |
||||
|
||||
But the read permission of the files in the bucket can be changed, in other words, the files in the private bucket can also be read publicly. |
||||
|
||||
So whether the `AccessKey` is needed depends on whether the file is private or not. |
||||
|
||||
You can view this `document <https://www.alibabacloud.com/help/en/tablestore/latest/how-can-i-obtain-an-accesskey-pair>`_ |
||||
when creating a pair `AccessKey`. |
||||
|
||||
For the specific use of resource plugins, you can see `How to use` in :doc:`resource-plugin` |
||||
|
||||
Dive Into |
||||
--------- |
||||
|
||||
.. automodule:: pydolphinscheduler.resources_plugin.OSS |
||||
@ -0,0 +1,36 @@
|
||||
.. Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
.. http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
.. Unless required by applicable law or agreed to in writing, |
||||
software distributed under the License is distributed on an |
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
KIND, either express or implied. See the License for the |
||||
specific language governing permissions and limitations |
||||
under the License. |
||||
|
||||
S3 |
||||
== |
||||
|
||||
`S3` is a Amazon S3 resource plugin for pydolphinscheduler. |
||||
|
||||
When using a Amazon S3 resource plugin, you only need to add the `resource_plugin` parameter in the task subclass or workflow definition, |
||||
such as `resource_plugin=S3(prefix="xxx")`, if the file is publicly readable. |
||||
|
||||
When the file is private, using `resource_plugin=S3(prefix="xxx", access_key_id="xxx", access_key_secret="xxx")` |
||||
|
||||
You can view this `document <https://docs.aws.amazon.com/general/latest/gr/aws-access-keys-best-practices.html>`_ |
||||
when creating a pair `AccessKey`. |
||||
|
||||
For the specific use of resource plugins, you can see `How to use` in :doc:`resource-plugin` |
||||
|
||||
Dive Into |
||||
--------- |
||||
|
||||
.. automodule:: pydolphinscheduler.resources_plugin.S3 |
||||
@ -0,0 +1,86 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
"""DolphinScheduler BucketFileInfo and Bucket object.""" |
||||
from abc import ABCMeta, abstractmethod |
||||
from typing import Optional |
||||
|
||||
|
||||
class BucketFileInfo: |
||||
"""A class that defines the details of BUCKET files. |
||||
|
||||
:param bucket: A string representing the bucket to which the bucket file belongs. |
||||
:param file_path: A string representing the bucket file path. |
||||
""" |
||||
|
||||
def __init__( |
||||
self, |
||||
bucket: Optional[str] = None, |
||||
file_path: Optional[str] = None, |
||||
*args, |
||||
**kwargs |
||||
): |
||||
self.bucket = bucket |
||||
self.file_path = file_path |
||||
|
||||
|
||||
class OSSFileInfo(BucketFileInfo): |
||||
"""A class that defines the details of OSS files. |
||||
|
||||
:param endpoint: A string representing the OSS file endpoint. |
||||
:param bucket: A string representing the bucket to which the OSS file belongs. |
||||
:param file_path: A string representing the OSS file path. |
||||
""" |
||||
|
||||
def __init__( |
||||
self, |
||||
endpoint: Optional[str] = None, |
||||
bucket: Optional[str] = None, |
||||
file_path: Optional[str] = None, |
||||
*args, |
||||
**kwargs |
||||
): |
||||
super().__init__(bucket=bucket, file_path=file_path, *args, **kwargs) |
||||
self.endpoint = endpoint |
||||
|
||||
|
||||
class S3FileInfo(BucketFileInfo): |
||||
"""A class that defines the details of S3 files. |
||||
|
||||
:param bucket: A string representing the bucket to which the S3 file belongs. |
||||
:param file_path: A string representing the S3 file path. |
||||
""" |
||||
|
||||
def __init__( |
||||
self, |
||||
bucket: Optional[str] = None, |
||||
file_path: Optional[str] = None, |
||||
*args, |
||||
**kwargs |
||||
): |
||||
super().__init__(bucket=bucket, file_path=file_path, *args, **kwargs) |
||||
|
||||
|
||||
class Bucket(object, metaclass=ABCMeta): |
||||
"""An abstract class of online code repository based on git implementation.""" |
||||
|
||||
_bucket_file_info: Optional = None |
||||
|
||||
@abstractmethod |
||||
def get_bucket_file_info(self, path: str): |
||||
"""Get the detailed information of BUCKET file according to the file URL.""" |
||||
raise NotImplementedError |
||||
@ -0,0 +1,112 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
"""DolphinScheduler gitlab resource plugin.""" |
||||
from typing import Optional |
||||
from urllib.parse import urljoin, urlparse |
||||
|
||||
import gitlab |
||||
import requests |
||||
|
||||
from pydolphinscheduler.constants import Symbol |
||||
from pydolphinscheduler.core.resource_plugin import ResourcePlugin |
||||
from pydolphinscheduler.resources_plugin.base.git import Git, GitLabFileInfo |
||||
|
||||
|
||||
class GitLab(ResourcePlugin, Git): |
||||
"""GitLab object, declare GitLab resource plugin for task and workflow to dolphinscheduler. |
||||
|
||||
:param prefix: A string representing the prefix of GitLab. |
||||
:param private_token: A string used for identity authentication of GitLab private or Internal repository. |
||||
:param oauth_token: A string used for identity authentication of GitLab private or Internal repository. |
||||
:param username: A string representing the user of the repository. |
||||
:param password: A string representing the user password. |
||||
""" |
||||
|
||||
def __init__( |
||||
self, |
||||
prefix: str, |
||||
private_token: Optional[str] = None, |
||||
oauth_token: Optional[str] = None, |
||||
username: Optional[str] = None, |
||||
password: Optional[str] = None, |
||||
*args, |
||||
**kwargs, |
||||
): |
||||
super().__init__(prefix, *args, **kwargs) |
||||
self.private_token = private_token |
||||
self.oauth_token = oauth_token |
||||
self.username = username |
||||
self.password = password |
||||
|
||||
def get_git_file_info(self, path: str): |
||||
"""Get file information from the file url, like repository name, user, branch, and file path.""" |
||||
self.get_index(path, Symbol.SLASH, 8) |
||||
result = urlparse(path) |
||||
elements = result.path.split(Symbol.SLASH) |
||||
self._git_file_info = GitLabFileInfo( |
||||
host=f"{result.scheme}://{result.hostname}", |
||||
repo_name=elements[2], |
||||
branch=elements[5], |
||||
file_path=Symbol.SLASH.join( |
||||
str(elements[i]) for i in range(6, len(elements)) |
||||
), |
||||
user=elements[1], |
||||
) |
||||
|
||||
def authentication(self): |
||||
"""Gitlab authentication.""" |
||||
host = self._git_file_info.host |
||||
if self.private_token is not None: |
||||
return gitlab.Gitlab(host, private_token=self.private_token) |
||||
if self.oauth_token is not None: |
||||
return gitlab.Gitlab(host, oauth_token=self.oauth_token) |
||||
if self.username is not None and self.password is not None: |
||||
oauth_token = self.OAuth_token() |
||||
return gitlab.Gitlab(host, oauth_token=oauth_token) |
||||
return gitlab.Gitlab(host) |
||||
|
||||
def OAuth_token(self): |
||||
"""Obtain OAuth Token.""" |
||||
data = { |
||||
"grant_type": "password", |
||||
"username": self.username, |
||||
"password": self.password, |
||||
} |
||||
host = self._git_file_info.host |
||||
resp = requests.post("%s/oauth/token" % host, data=data) |
||||
oauth_token = resp.json()["access_token"] |
||||
return oauth_token |
||||
|
||||
def read_file(self, suf: str): |
||||
"""Get the content of the file. |
||||
|
||||
The address of the file is the prefix of the resource plugin plus the parameter suf. |
||||
""" |
||||
path = urljoin(self.prefix, suf) |
||||
self.get_git_file_info(path) |
||||
gl = self.authentication() |
||||
project = gl.projects.get( |
||||
"%s/%s" % (self._git_file_info.user, self._git_file_info.repo_name) |
||||
) |
||||
return ( |
||||
project.files.get( |
||||
file_path=self._git_file_info.file_path, ref=self._git_file_info.branch |
||||
) |
||||
.decode() |
||||
.decode() |
||||
) |
||||
@ -0,0 +1,76 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
"""DolphinScheduler oss resource plugin.""" |
||||
from typing import Optional |
||||
from urllib.parse import urljoin, urlparse |
||||
|
||||
import oss2 |
||||
|
||||
from pydolphinscheduler.constants import Symbol |
||||
from pydolphinscheduler.core.resource_plugin import ResourcePlugin |
||||
from pydolphinscheduler.resources_plugin.base.bucket import Bucket, OSSFileInfo |
||||
|
||||
|
||||
class OSS(ResourcePlugin, Bucket): |
||||
"""OSS object, declare OSS resource plugin for task and workflow to dolphinscheduler. |
||||
|
||||
:param prefix: A string representing the prefix of OSS. |
||||
:param access_key_id: A string representing the ID of AccessKey for AliCloud OSS. |
||||
:param access_key_secret: A string representing the secret of AccessKey for AliCloud OSS. |
||||
""" |
||||
|
||||
def __init__( |
||||
self, |
||||
prefix: str, |
||||
access_key_id: Optional[str] = None, |
||||
access_key_secret: Optional[str] = None, |
||||
*args, |
||||
**kwargs, |
||||
): |
||||
super().__init__(prefix, *args, **kwargs) |
||||
self.access_key_id = access_key_id |
||||
self.access_key_secret = access_key_secret |
||||
|
||||
_bucket_file_info: Optional[OSSFileInfo] = None |
||||
|
||||
def get_bucket_file_info(self, path: str): |
||||
"""Get file information from the file url, like repository name, user, branch, and file path.""" |
||||
self.get_index(path, Symbol.SLASH, 3) |
||||
result = urlparse(path) |
||||
hostname = result.hostname |
||||
elements = hostname.split(Symbol.POINT) |
||||
self._bucket_file_info = OSSFileInfo( |
||||
endpoint=f"{result.scheme}://" |
||||
f"{Symbol.POINT.join(str(elements[i]) for i in range(1, len(elements)))}", |
||||
bucket=hostname.split(Symbol.POINT)[0], |
||||
file_path=result.path[1:], |
||||
) |
||||
|
||||
def read_file(self, suf: str): |
||||
"""Get the content of the file. |
||||
|
||||
The address of the file is the prefix of the resource plugin plus the parameter suf. |
||||
""" |
||||
path = urljoin(self.prefix, suf) |
||||
self.get_bucket_file_info(path) |
||||
auth = oss2.Auth(self.access_key_id, self.access_key_secret) |
||||
bucket = oss2.Bucket( |
||||
auth, self._bucket_file_info.endpoint, self._bucket_file_info.bucket |
||||
) |
||||
result = bucket.get_object(self._bucket_file_info.file_path).read().decode() |
||||
return result.read().decode() |
||||
@ -0,0 +1,74 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
"""DolphinScheduler S3 resource plugin.""" |
||||
|
||||
from typing import Optional |
||||
from urllib.parse import urljoin |
||||
|
||||
import boto3 |
||||
|
||||
from pydolphinscheduler.constants import Symbol |
||||
from pydolphinscheduler.core.resource_plugin import ResourcePlugin |
||||
from pydolphinscheduler.resources_plugin.base.bucket import Bucket, S3FileInfo |
||||
|
||||
|
||||
class S3(ResourcePlugin, Bucket): |
||||
"""S3 object, declare S3 resource plugin for task and workflow to dolphinscheduler. |
||||
|
||||
:param prefix: A string representing the prefix of S3. |
||||
:param access_key_id: A string representing the ID of AccessKey for Amazon S3. |
||||
:param access_key_secret: A string representing the secret of AccessKey for Amazon S3. |
||||
""" |
||||
|
||||
def __init__( |
||||
self, |
||||
prefix: str, |
||||
access_key_id: Optional[str] = None, |
||||
access_key_secret: Optional[str] = None, |
||||
*args, |
||||
**kwargs |
||||
): |
||||
super().__init__(prefix, *args, **kwargs) |
||||
self.access_key_id = access_key_id |
||||
self.access_key_secret = access_key_secret |
||||
|
||||
_bucket_file_info: Optional[S3FileInfo] = None |
||||
|
||||
def get_bucket_file_info(self, path: str): |
||||
"""Get file information from the file url, like repository name, user, branch, and file path.""" |
||||
elements = path.split(Symbol.SLASH) |
||||
self.get_index(path, Symbol.SLASH, 3) |
||||
self._bucket_file_info = S3FileInfo( |
||||
bucket=elements[2].split(Symbol.POINT)[0], |
||||
file_path=Symbol.SLASH.join( |
||||
str(elements[i]) for i in range(3, len(elements)) |
||||
), |
||||
) |
||||
|
||||
def read_file(self, suf: str): |
||||
"""Get the content of the file. |
||||
|
||||
The address of the file is the prefix of the resource plugin plus the parameter suf. |
||||
""" |
||||
path = urljoin(self.prefix, suf) |
||||
self.get_bucket_file_info(path) |
||||
bucket = self._bucket_file_info.bucket |
||||
key = self._bucket_file_info.file_path |
||||
s3_resource = boto3.resource("s3") |
||||
s3_object = s3_resource.Object(bucket, key) |
||||
return s3_object.get()["Body"].read().decode("utf-8") |
||||
@ -0,0 +1,116 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
"""Test github resource plugin.""" |
||||
import pytest |
||||
|
||||
from pydolphinscheduler.resources_plugin.gitlab import GitLab |
||||
|
||||
|
||||
@pytest.mark.parametrize( |
||||
"attr, expected", |
||||
[ |
||||
( |
||||
"https://gitlab.com/pydolphinscheduler/ds-gitlab/-/blob/main/union.sh", |
||||
{ |
||||
"branch": "main", |
||||
"file_path": "union.sh", |
||||
"host": "https://gitlab.com", |
||||
"repo_name": "ds-gitlab", |
||||
"user": "pydolphinscheduler", |
||||
}, |
||||
), |
||||
( |
||||
"https://gitlab.com/pydolphinscheduler/ds/-/blob/dev/test/exc.sh", |
||||
{ |
||||
"branch": "dev", |
||||
"file_path": "test/exc.sh", |
||||
"host": "https://gitlab.com", |
||||
"repo_name": "ds", |
||||
"user": "pydolphinscheduler", |
||||
}, |
||||
), |
||||
], |
||||
) |
||||
def test_gitlab_get_git_file_info(attr, expected): |
||||
"""Test the get_file_info function of the gitlab resource plugin.""" |
||||
gitlab = GitLab(prefix="prefix") |
||||
gitlab.get_git_file_info(attr) |
||||
assert expected == gitlab._git_file_info.__dict__ |
||||
|
||||
|
||||
@pytest.mark.skip(reason="This test needs gitlab service") |
||||
@pytest.mark.parametrize( |
||||
"attr, expected", |
||||
[ |
||||
( |
||||
{ |
||||
"init": { |
||||
"prefix": "https://gitlab.com/pydolphinscheduler/ds-internal/-/blob/main", |
||||
"oauth_token": "24518bd4cf5bfe9xx", |
||||
}, |
||||
"file_path": "union.sh", |
||||
}, |
||||
"test gitlab resource plugin\n", |
||||
), |
||||
( |
||||
{ |
||||
"init": { |
||||
"prefix": "https://gitlab.com/pydolphinscheduler/ds/-/blob/main", |
||||
"private_token": "9TyTe2xx", |
||||
}, |
||||
"file_path": "union.sh", |
||||
}, |
||||
"test gitlab resource plugin\n", |
||||
), |
||||
( |
||||
{ |
||||
"init": { |
||||
"prefix": "https://gitlab.com/pydolphinscheduler/ds-gitlab/-/blob/main", |
||||
"username": "pydolphinscheduler", |
||||
"password": "4295xx", |
||||
}, |
||||
"file_path": "union.sh", |
||||
}, |
||||
"test gitlab resource plugin\n", |
||||
), |
||||
( |
||||
{ |
||||
"init": { |
||||
"prefix": "https://gitlab.com/pydolphinscheduler/ds-public/-/blob/main", |
||||
}, |
||||
"file_path": "union.sh", |
||||
}, |
||||
"test gitlab resource plugin\n", |
||||
), |
||||
( |
||||
{ |
||||
"init": { |
||||
"prefix": "https://gitlab.com/pydolphinscheduler/ds-internal/-/blob/main", |
||||
"username": "pydolphinscheduler", |
||||
"password": "429xxx", |
||||
}, |
||||
"file_path": "union.sh", |
||||
}, |
||||
"test gitlab resource plugin\n", |
||||
), |
||||
], |
||||
) |
||||
def test_gitlab_read_file(attr, expected): |
||||
"""Test the read_file function of the gitlab resource plug-in.""" |
||||
gitlab = GitLab(**attr.get("init")) |
||||
assert expected == gitlab.read_file(attr.get("file_path")) |
||||
@ -0,0 +1,112 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
"""Test oss resource plugin.""" |
||||
import pytest |
||||
|
||||
from pydolphinscheduler.resources_plugin.oss import OSS |
||||
|
||||
|
||||
@pytest.mark.parametrize( |
||||
"attr, expected", |
||||
[ |
||||
( |
||||
"https://ospp-ds-private.oss-cn-hangzhou.aliyuncs.com/a.sh", |
||||
{ |
||||
"endpoint": "https://oss-cn-hangzhou.aliyuncs.com", |
||||
"file_path": "a.sh", |
||||
"bucket": "ospp-ds-private", |
||||
}, |
||||
), |
||||
( |
||||
"https://ospp-ds-public.oss-cn-hangzhou.aliyuncs.com/dir/a.sh", |
||||
{ |
||||
"endpoint": "https://oss-cn-hangzhou.aliyuncs.com", |
||||
"file_path": "dir/a.sh", |
||||
"bucket": "ospp-ds-public", |
||||
}, |
||||
), |
||||
], |
||||
) |
||||
def test_oss_get_bucket_file_info(attr, expected): |
||||
"""Test the get_bucket_file_info function of the oss resource plugin.""" |
||||
oss = OSS(prefix="prefix") |
||||
oss.get_bucket_file_info(attr) |
||||
assert expected == oss._bucket_file_info.__dict__ |
||||
|
||||
|
||||
@pytest.mark.skip(reason="This test requires OSS services") |
||||
@pytest.mark.parametrize( |
||||
"attr, expected", |
||||
[ |
||||
( |
||||
{ |
||||
"init": { |
||||
"prefix": "https://ospp-ds-private.oss-cn-hangzhou.aliyuncs.com", |
||||
"access_key_id": "LTAI5tP25Mxx", |
||||
"access_key_secret": "cSur23Qbxx", |
||||
}, |
||||
"file_path": "a.sh", |
||||
}, |
||||
"test oss resource plugin\n", |
||||
), |
||||
( |
||||
{ |
||||
"init": { |
||||
"prefix": "https://ospp-ds-private.oss-cn-hangzhou.aliyuncs.com/dir/", |
||||
"access_key_id": "LTAxx", |
||||
"access_key_secret": "cSur23Qxx", |
||||
}, |
||||
"file_path": "b.sh", |
||||
}, |
||||
"test oss resource plugin\n", |
||||
), |
||||
( |
||||
{ |
||||
"init": { |
||||
"prefix": "https://ospp-ds-private.oss-cn-hangzhou.aliyuncs.com", |
||||
}, |
||||
"file_path": "b.sh", |
||||
}, |
||||
"test oss resource plugin\n", |
||||
), |
||||
( |
||||
{ |
||||
"init": { |
||||
"prefix": "https://ospp-ds-public.oss-cn-hangzhou.aliyuncs.com", |
||||
}, |
||||
"file_path": "b.sh", |
||||
}, |
||||
"test oss resource plugin\n", |
||||
), |
||||
( |
||||
{ |
||||
"init": { |
||||
"prefix": "https://ospp-ds-public.oss-cn-hangzhou.aliyuncs.com/dir/", |
||||
"access_key_id": "LTAIxx", |
||||
"access_key_secret": "cSurxx", |
||||
}, |
||||
"file_path": "a.sh", |
||||
}, |
||||
"test oss resource plugin\n", |
||||
), |
||||
], |
||||
) |
||||
def test_oss_read_file(attr, expected): |
||||
"""Test the read_file function of the oss resource plug-in.""" |
||||
oss = OSS(**attr.get("init")) |
||||
assert expected == oss.read_file(attr.get("file_path")) |
||||
@ -0,0 +1,79 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
"""Test oss resource plugin.""" |
||||
import pytest |
||||
|
||||
from pydolphinscheduler.resources_plugin import S3 |
||||
|
||||
|
||||
@pytest.mark.parametrize( |
||||
"attr, expected", |
||||
[ |
||||
( |
||||
"https://ds-resource-plugin-private.s3.amazonaws.com/a.sh", |
||||
{ |
||||
"file_path": "a.sh", |
||||
"bucket": "ds-resource-plugin-private", |
||||
}, |
||||
), |
||||
( |
||||
"https://ds-resource-plugin-public.s3.amazonaws.com/dir/a.sh", |
||||
{ |
||||
"file_path": "dir/a.sh", |
||||
"bucket": "ds-resource-plugin-public", |
||||
}, |
||||
), |
||||
], |
||||
) |
||||
def test_s3_get_bucket_file_info(attr, expected): |
||||
"""Test the get_bucket_file_info function of the s3 resource plugin.""" |
||||
s3 = S3(prefix="prefix") |
||||
s3.get_bucket_file_info(attr) |
||||
assert expected == s3._bucket_file_info.__dict__ |
||||
|
||||
|
||||
@pytest.mark.skip(reason="This test requires s3 services") |
||||
@pytest.mark.parametrize( |
||||
"attr, expected", |
||||
[ |
||||
( |
||||
{ |
||||
"init": { |
||||
"prefix": "https://ds-resource-plugin-private.s3.amazonaws.com/dir/", |
||||
"access_key_id": "LTAI5tP25Mxx", |
||||
"access_key_secret": "cSur23Qbxx", |
||||
}, |
||||
"file_path": "a.sh", |
||||
}, |
||||
"test s3 resource plugin\n", |
||||
), |
||||
( |
||||
{ |
||||
"init": { |
||||
"prefix": "https://ds-resource-plugin-public.s3.amazonaws.com/", |
||||
}, |
||||
"file_path": "a.sh", |
||||
}, |
||||
"test s3 resource plugin\n", |
||||
), |
||||
], |
||||
) |
||||
def test_s3_read_file(attr, expected): |
||||
"""Test the read_file function of the s3 resource plug-in.""" |
||||
s3 = S3(**attr.get("init")) |
||||
assert expected == s3.read_file(attr.get("file_path")) |
||||
Loading…
Reference in new issue