Browse Source

[Feature-16396][storage-plugin] Add Tencent Cloud COS Storage Plugin (#16565)

dev
Mighten Dai 2 months ago committed by GitHub
parent
commit
9024dcad3e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
  1. 31
      docs/docs/en/guide/resource/configuration.md
  2. 29
      docs/docs/zh/guide/resource/configuration.md
  3. 17
      dolphinscheduler-bom/pom.xml
  4. 5
      dolphinscheduler-common/pom.xml
  5. 12
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java
  6. 112
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/CosRemoteLogHandler.java
  7. 2
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/RemoteLogHandlerFactory.java
  8. 3
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java
  9. 70
      dolphinscheduler-common/src/main/resources/common.properties
  10. 34
      dolphinscheduler-common/src/main/resources/remote-logging.yaml
  11. 80
      dolphinscheduler-common/src/main/resources/resource-center.yaml
  12. 13
      dolphinscheduler-common/src/test/resources/common.properties
  13. 46
      dolphinscheduler-common/src/test/resources/remote-logging.yaml
  14. 26
      dolphinscheduler-dist/release-docs/LICENSE
  15. 22
      dolphinscheduler-dist/release-docs/licenses/LICENSE-cos_api.txt
  16. 22
      dolphinscheduler-dist/release-docs/licenses/LICENSE-java-jwt.txt
  17. 203
      dolphinscheduler-dist/release-docs/licenses/LICENSE-logging-interceptor.txt
  18. 57
      dolphinscheduler-dist/release-docs/licenses/LICENSE-mxparser.txt
  19. 203
      dolphinscheduler-dist/release-docs/licenses/LICENSE-tencentcloud-sdk-java-common.txt
  20. 203
      dolphinscheduler-dist/release-docs/licenses/LICENSE-tencentcloud-sdk-java-kms.txt
  21. 15
      dolphinscheduler-dist/release-docs/licenses/LICENSE-xmlpull.txt
  22. 29
      dolphinscheduler-dist/release-docs/licenses/LICENSE-xstream.txt
  23. 5
      dolphinscheduler-storage-plugin/dolphinscheduler-storage-all/pom.xml
  24. 6
      dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageType.java
  25. 70
      dolphinscheduler-storage-plugin/dolphinscheduler-storage-cos/pom.xml
  26. 30
      dolphinscheduler-storage-plugin/dolphinscheduler-storage-cos/src/main/java/org/apache/dolphinscheduler/plugin/storage/cos/CosStorageConstants.java
  27. 338
      dolphinscheduler-storage-plugin/dolphinscheduler-storage-cos/src/main/java/org/apache/dolphinscheduler/plugin/storage/cos/CosStorageOperator.java
  28. 59
      dolphinscheduler-storage-plugin/dolphinscheduler-storage-cos/src/main/java/org/apache/dolphinscheduler/plugin/storage/cos/CosStorageOperatorFactory.java
  29. 40
      dolphinscheduler-storage-plugin/dolphinscheduler-storage-cos/src/main/java/org/apache/dolphinscheduler/plugin/storage/cos/CosStorageProperties.java
  30. 21
      dolphinscheduler-storage-plugin/dolphinscheduler-storage-cos/src/test/resources/logback.xml
  31. 1
      dolphinscheduler-storage-plugin/pom.xml
  32. 9
      tools/dependencies/known-dependencies.txt

31
docs/docs/en/guide/resource/configuration.md

@ -1,7 +1,7 @@
# Resource Center Configuration
- You could use `Resource Center` to upload text files and other task-related files.
- You could configure `Resource Center` to use distributed file system like [Hadoop](https://hadoop.apache.org/docs/r2.7.0/) (2.6+), [MinIO](https://github.com/minio/minio) cluster or remote storage products like [AWS S3](https://aws.amazon.com/s3/), [Alibaba Cloud OSS](https://www.aliyun.com/product/oss), [Huawei Cloud OBS](https://support.huaweicloud.com/obs/index.html) etc.
- You could configure `Resource Center` to use distributed file system like [Hadoop](https://hadoop.apache.org/docs/r2.7.0/) (2.6+), [MinIO](https://github.com/minio/minio) cluster or remote storage products like [AWS S3](https://aws.amazon.com/s3/), [Alibaba Cloud OSS](https://www.aliyun.com/product/oss), [Huawei Cloud OBS](https://support.huaweicloud.com/obs/index.html), [Tencent Cloud COS](https://cloud.tencent.com/product/cos), etc.
- You could configure `Resource Center` to use local file system. If you deploy `DolphinScheduler` in `Standalone` mode, you could configure it to use local file system for `Resource Center` without the need of an external `HDFS` system or `S3`.
- Furthermore, if you deploy `DolphinScheduler` in `Cluster` mode, you could use [S3FS-FUSE](https://github.com/s3fs-fuse/s3fs-fuse) to mount `S3` or [JINDO-FUSE](https://help.aliyun.com/document_detail/187410.html) to mount `OSS` to your machines and use the local file system for `Resource Center`. In this way, you could operate remote files as if on your local machines.
@ -96,3 +96,32 @@ resource.huawei.cloud.obs.endpoint=obs.cn-southwest-2.huaweicloud.com
> * If you want to use the resource upload function, the deployment user in [installation and deployment](../installation/standalone.md) must have relevant operation authority.
> * If you using a Hadoop cluster with HA, you need to enable HDFS resource upload, and you need to copy the `core-site.xml` and `hdfs-site.xml` under the Hadoop cluster to `worker-server/conf` and `api-server/conf`, otherwise skip this copy step.
## connect COS
if you want to upload resources to `Resource Center` connected to `COS`, you need to configure `api-server/conf/resource-center.yaml` and `worker-server/conf/resource-center.yaml`. You can refer to the following:
config the following fields
```yaml
resource:
# Tencent Cloud Storage (COS) setup, required if you set resource.storage.type=COS
tencent:
cloud:
access:
key:
id: <your-access-key-id>
secret: <your-access-key-secret>
cos:
# predefined region code: https://cloud.tencent.com/document/product/436/6224
region: ap-nanjing
bucket:
name: dolphinscheduler
```
To enable COS, you also need to configure `api-server/conf/common.properties` and `worker-server/conf/common.properties`. You can refer to the following:
```properties
resource.storage.type=COS
```

29
docs/docs/zh/guide/resource/configuration.md

@ -1,7 +1,7 @@
# 资源中心配置详情
- 资源中心通常用于上传文件以及任务组管理等操作。
- 资源中心可以对接分布式的文件存储系统,如[Hadoop](https://hadoop.apache.org/docs/r2.7.0/)(2.6+)或者[MinIO](https://github.com/minio/minio)集群,也可以对接远端的对象存储,如[AWS S3](https://aws.amazon.com/s3/)或者[阿里云 OSS](https://www.aliyun.com/product/oss),[华为云 OBS](https://support.huaweicloud.com/obs/index.html) 等。
- 资源中心可以对接分布式的文件存储系统,如[Hadoop](https://hadoop.apache.org/docs/r2.7.0/)(2.6+)或者[MinIO](https://github.com/minio/minio)集群,也可以对接远端的对象存储,如[AWS S3](https://aws.amazon.com/s3/)或者[阿里云 OSS](https://www.aliyun.com/product/oss),[华为云 OBS](https://support.huaweicloud.com/obs/index.html),[腾讯云 COS](https://cloud.tencent.com/product/cos) 等。
- 资源中心也可以直接对接本地文件系统。在单机模式下,您无需依赖`Hadoop`或`S3`一类的外部存储系统,可以方便地对接本地文件系统进行体验。
- 除此之外,对于集群模式下的部署,您可以通过使用[S3FS-FUSE](https://github.com/s3fs-fuse/s3fs-fuse)将`S3`挂载到本地,或者使用[JINDO-FUSE](https://help.aliyun.com/document_detail/187410.html)将`OSS`挂载到本地等,再用资源中心对接本地文件系统方式来操作远端对象存储中的文件。
@ -90,3 +90,30 @@ resource.huawei.cloud.obs.endpoint=obs.cn-southwest-2.huaweicloud.com
> * 如果用到资源上传的功能,那么[安装部署](../installation/standalone.md)中,部署用户需要有这部分的操作权限。
> * 如果 Hadoop 集群的 NameNode 配置了 HA 的话,需要开启 HDFS 类型的资源上传,同时需要将 Hadoop 集群下的 `core-site.xml``hdfs-site.xml` 复制到 `worker-server/conf` 以及 `api-server/conf`,非 NameNode HA 跳过此步骤。
## 对接腾讯云 COS
如果需要使用到资源中心的 COS 上传资源,我们需要对以下路径的进行配置:`api-server/conf/resource-center.yaml` 和 `worker-server/conf/resource-center.yaml`。可参考如下:
```yaml
resource:
# 腾讯云 COS 配置
tencent:
cloud:
access:
key:
id: <your-access-key-id>
secret: <your-access-key-secret>
cos:
# COS 区域代码可参考: https://cloud.tencent.com/document/product/436/6224
region: ap-nanjing
bucket:
name: dolphinscheduler
```
为了激活腾讯云存储 COS,还需要对以下路径的进行配置:`api-server/conf/common.properties` 和 `worker-server/conf/common.properties`。可参考如下:
```properties
resource.storage.type=COS
```

17
dolphinscheduler-bom/pom.xml

@ -116,6 +116,7 @@
<azure-sdk-bom.version>1.2.10</azure-sdk-bom.version>
<protobuf.version>3.17.2</protobuf.version>
<esdk-obs.version>3.23.3</esdk-obs.version>
<qcloud-cos.version>5.6.231</qcloud-cos.version>
<system-lambda.version>1.2.1</system-lambda.version>
<zeppelin-client.version>0.10.1</zeppelin-client.version>
<testcontainer.version>1.19.3</testcontainer.version>
@ -927,6 +928,22 @@
</exclusions>
</dependency>
<dependency>
<groupId>com.qcloud</groupId>
<artifactId>cos_api</artifactId>
<version>${qcloud-cos.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.github.stefanbirkner</groupId>
<artifactId>system-lambda</artifactId>

5
dolphinscheduler-common/pom.xml

@ -106,6 +106,11 @@
<artifactId>esdk-obs-java-bundle</artifactId>
</dependency>
<dependency>
<groupId>com.qcloud</groupId>
<artifactId>cos_api</artifactId>
</dependency>
<dependency>
<groupId>com.azure</groupId>
<artifactId>azure-storage-blob</artifactId>

12
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java

@ -36,6 +36,7 @@ public final class Constants {
public static final String REMOTE_LOGGING_YAML_PATH = "/remote-logging.yaml";
public static final String AWS_YAML_PATH = "/aws.yaml";
public static final String RESOURCE_CENTER_YAML_PATH = "/resource-center.yaml";
public static final String FORMAT_SS = "%s%s";
public static final String FORMAT_S_S = "%s/%s";
@ -681,6 +682,17 @@ public final class Constants {
public static final String REMOTE_LOGGING_ABS_ACCOUNT_KEY = "remote.logging.abs.account.key";
public static final String REMOTE_LOGGING_ABS_CONTAINER_NAME = "remote.logging.abs.container.name";
/**
* remote logging for COS
*/
public static final String REMOTE_LOGGING_COS_ACCESS_KEY_ID = "remote.logging.cos.access.key.id";
public static final String REMOTE_LOGGING_COS_ACCESS_KEY_SECRET = "remote.logging.cos.access.key.secret";
public static final String REMOTE_LOGGING_COS_BUCKET_NAME = "remote.logging.cos.bucket.name";
public static final String REMOTE_LOGGING_COS_REGION = "remote.logging.cos.region";
/**
* data quality
*/

112
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/CosRemoteLogHandler.java

@ -0,0 +1,112 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.log.remote;
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.commons.lang3.StringUtils;
import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import lombok.extern.slf4j.Slf4j;
import com.qcloud.cos.COSClient;
import com.qcloud.cos.ClientConfig;
import com.qcloud.cos.auth.BasicCOSCredentials;
import com.qcloud.cos.auth.COSCredentials;
import com.qcloud.cos.http.HttpProtocol;
import com.qcloud.cos.model.GetObjectRequest;
import com.qcloud.cos.model.PutObjectRequest;
import com.qcloud.cos.region.Region;
@Slf4j
public class CosRemoteLogHandler implements RemoteLogHandler, Closeable {
private final COSClient cosClient;
private final String bucketName;
private static CosRemoteLogHandler instance;
private CosRemoteLogHandler() {
String secretId = PropertyUtils.getString(Constants.REMOTE_LOGGING_COS_ACCESS_KEY_ID);
String secretKey = PropertyUtils.getString(Constants.REMOTE_LOGGING_COS_ACCESS_KEY_SECRET);
COSCredentials cosCredentials = new BasicCOSCredentials(secretId, secretKey);
String regionName = PropertyUtils.getString(Constants.REMOTE_LOGGING_COS_REGION);
ClientConfig clientConfig = new ClientConfig(new Region(regionName));
clientConfig.setHttpProtocol(HttpProtocol.https);
this.cosClient = new COSClient(cosCredentials, clientConfig);
this.bucketName = PropertyUtils.getString(Constants.REMOTE_LOGGING_COS_BUCKET_NAME);
checkBucketNameExists(this.bucketName);
}
public static synchronized CosRemoteLogHandler getInstance() {
if (instance == null) {
instance = new CosRemoteLogHandler();
}
return instance;
}
@Override
public void sendRemoteLog(String logPath) {
String objectName = RemoteLogUtils.getObjectNameFromLogPath(logPath);
try {
log.info("send remote log from {} to tencent cos {}", logPath, objectName);
PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, objectName, new File(logPath));
cosClient.putObject(putObjectRequest);
} catch (Exception e) {
log.error("error while sending remote log from {} to tencent cos {}, reason:", logPath, objectName, e);
}
}
@Override
public void getRemoteLog(String logPath) {
String objectName = RemoteLogUtils.getObjectNameFromLogPath(logPath);
try {
log.info("get remote log from tencent cos {} to {}", objectName, logPath);
cosClient.getObject(new GetObjectRequest(bucketName, objectName), new File(logPath));
} catch (Exception e) {
log.error("error while sending remote log from {} to tencent cos {}, reason:", objectName, logPath, e);
}
}
@Override
public void close() throws IOException {
if (cosClient != null) {
cosClient.shutdown();
}
}
private void checkBucketNameExists(String bucketName) {
if (StringUtils.isBlank(bucketName)) {
throw new IllegalArgumentException(Constants.REMOTE_LOGGING_COS_BUCKET_NAME + " is empty");
}
boolean existsBucket = cosClient.doesBucketExist(bucketName);
if (!existsBucket) {
throw new IllegalArgumentException(
"bucketName: " + bucketName + " does not exists, you should create it first");
}
log.debug("tencent cos bucket {} has been found for remote logging", bucketName);
}
}

2
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/RemoteLogHandlerFactory.java

@ -41,6 +41,8 @@ public class RemoteLogHandlerFactory {
return GcsRemoteLogHandler.getInstance();
} else if ("ABS".equals(target)) {
return AbsRemoteLogHandler.getInstance();
} else if ("COS".equals(target)) {
return CosRemoteLogHandler.getInstance();
}
log.error("No suitable remote logging target for {}", target);

3
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java

@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.common.utils;
import static org.apache.dolphinscheduler.common.constants.Constants.AWS_YAML_PATH;
import static org.apache.dolphinscheduler.common.constants.Constants.COMMON_PROPERTIES_PATH;
import static org.apache.dolphinscheduler.common.constants.Constants.REMOTE_LOGGING_YAML_PATH;
import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_CENTER_YAML_PATH;
import org.apache.dolphinscheduler.common.config.ImmutablePriorityPropertyDelegate;
import org.apache.dolphinscheduler.common.config.ImmutablePropertyDelegate;
@ -43,7 +44,7 @@ public class PropertyUtils {
private final ImmutablePriorityPropertyDelegate propertyDelegate =
new ImmutablePriorityPropertyDelegate(
new ImmutablePropertyDelegate(COMMON_PROPERTIES_PATH),
new ImmutableYamlDelegate(REMOTE_LOGGING_YAML_PATH, AWS_YAML_PATH));
new ImmutableYamlDelegate(REMOTE_LOGGING_YAML_PATH, AWS_YAML_PATH, RESOURCE_CENTER_YAML_PATH));
public static String getString(String key) {
return propertyDelegate.get(key.trim());

70
dolphinscheduler-common/src/main/resources/common.properties

@ -21,58 +21,15 @@ data.basedir.path=/tmp/dolphinscheduler
# resource view suffixs
#resource.view.suffixs=txt,log,sh,bat,conf,cfg,py,java,sql,xml,hql,properties,json,yml,yaml,ini,js
# resource storage type: LOCAL, HDFS, S3, OSS, GCS, ABS, OBS. LOCAL type is default type, and it's a specific type of HDFS with "resource.hdfs.fs.defaultFS = file:///" configuration
# resource storage type: LOCAL, HDFS, S3, OSS, GCS, ABS, OBS, COS. LOCAL type is default type, and it's a specific type of HDFS with "resource.hdfs.fs.defaultFS = file:///" configuration
# please notice that LOCAL mode does not support reading and writing in distributed mode, which mean you can only use your resource in one machine, unless
# use shared file mount point
resource.storage.type=LOCAL
# resource store on HDFS/S3 path, resource file will store to this base path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended
resource.storage.upload.base.path=/tmp/dolphinscheduler
# The Azure client ID (Azure Application (client) ID)
resource.azure.client.id=minioadmin
# The Azure client secret in the Azure application
resource.azure.client.secret=minioadmin
# The Azure data factory subscription ID
resource.azure.subId=minioadmin
# The Azure tenant id in the Azure Active Directory
resource.azure.tenant.id=minioadmin
# The query interval
resource.query.interval=10000
# alibaba cloud access key id, required if you set resource.storage.type=OSS
resource.alibaba.cloud.access.key.id=<your-access-key-id>
# alibaba cloud access key secret, required if you set resource.storage.type=OSS
resource.alibaba.cloud.access.key.secret=<your-access-key-secret>
# alibaba cloud region, required if you set resource.storage.type=OSS
resource.alibaba.cloud.region=cn-hangzhou
# oss bucket name, required if you set resource.storage.type=OSS
resource.alibaba.cloud.oss.bucket.name=dolphinscheduler
# oss bucket endpoint, required if you set resource.storage.type=OSS
resource.alibaba.cloud.oss.endpoint=https://oss-cn-hangzhou.aliyuncs.com
# the location of the google cloud credential, required if you set resource.storage.type=GCS
resource.google.cloud.storage.credential=/path/to/credential
# gcs bucket name, required if you set resource.storage.type=GCS
resource.google.cloud.storage.bucket.name=<your-bucket>
# abs container name, required if you set resource.storage.type=ABS
resource.azure.blob.storage.container.name=<your-container>
# abs account name, required if you set resource.storage.type=ABS
resource.azure.blob.storage.account.name=<your-account-name>
# abs connection string, required if you set resource.storage.type=ABS
resource.azure.blob.storage.connection.string=<your-connection-string>
# huawei cloud access key id, required if you set resource.storage.type=OBS
resource.huawei.cloud.access.key.id=<your-access-key-id>
# huawei cloud access key secret, required if you set resource.storage.type=OBS
resource.huawei.cloud.access.key.secret=<your-access-key-secret>
# oss bucket name, required if you set resource.storage.type=OBS
resource.huawei.cloud.obs.bucket.name=dolphinscheduler
# oss bucket endpoint, required if you set resource.storage.type=OBS
resource.huawei.cloud.obs.endpoint=obs.cn-southwest-2.huaweicloud.com
# if resource.storage.type=HDFS, the user must have the permission to create directories under the HDFS root path
resource.hdfs.root.user=hdfs
# if resource.storage.type=S3, the value like: s3a://dolphinscheduler; if resource.storage.type=HDFS and namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir
@ -163,28 +120,3 @@ shell.interceptor.type=bash
# Whether to enable remote logging
remote.logging.enable=false
# if remote.logging.enable = true, set the target of remote logging
remote.logging.target=OSS
# if remote.logging.enable = true, set the log base directory
remote.logging.base.dir=logs
# if remote.logging.enable = true, set the number of threads to send logs to remote storage
remote.logging.thread.pool.size=10
# oss access key id, required if you set remote.logging.target=OSS
remote.logging.oss.access.key.id=<access.key.id>
# oss access key secret, required if you set remote.logging.target=OSS
remote.logging.oss.access.key.secret=<access.key.secret>
# oss bucket name, required if you set remote.logging.target=OSS
remote.logging.oss.bucket.name=<bucket.name>
# oss endpoint, required if you set remote.logging.target=OSS
remote.logging.oss.endpoint=<endpoint>
# the location of the google cloud credential, required if you set remote.logging.target=GCS
remote.logging.google.cloud.storage.credential=/path/to/credential
# gcs bucket name, required if you set remote.logging.target=GCS
remote.logging.google.cloud.storage.bucket.name=<your-bucket>
# abs account name, required if you set resource.storage.type=ABS
remote.logging.abs.account.name=<your-account-name>
# abs account key, required if you set resource.storage.type=ABS
remote.logging.abs.account.key=<your-account-key>
# abs container name, required if you set resource.storage.type=ABS
remote.logging.abs.container.name=<your-container-name>

34
dolphinscheduler-common/src/main/resources/remote-logging.yaml

@ -15,29 +15,27 @@
# limitations under the License.
#
remote-logging:
# Whether to enable remote logging
enable: false
# if remote-logging.enable = true, set the target of remote logging
remote.logging:
# if remote.logging.enable = true, set the target of remote logging
target: OSS
# if remote-logging.enable = true, set the log base directory
# if remote.logging.enable = true, set the log base directory
base.dir: logs
# if remote-logging.enable = true, set the number of threads to send logs to remote storage
# if remote.logging.enable = true, set the number of threads to send logs to remote storage
thread.pool.size: 10
# required if you set remote-logging.target=OSS
# required if you set remote.logging.target=OSS
oss:
# oss access key id, required if you set remote-logging.target=OSS
# oss access key id, required if you set remote.logging.target=OSS
access.key.id: <access.key.id>
# oss access key secret, required if you set remote-logging.target=OSS
# oss access key secret, required if you set remote.logging.target=OSS
access.key.secret: <access.key.secret>
# oss bucket name, required if you set remote-logging.target=OSS
# oss bucket name, required if you set remote.logging.target=OSS
bucket.name: <bucket.name>
# oss endpoint, required if you set remote-logging.target=OSS
# oss endpoint, required if you set remote.logging.target=OSS
endpoint: <endpoint>
google.cloud.storage:
# the location of the google cloud credential, required if you set remote-logging.target=GCS
# the location of the google cloud credential, required if you set remote.logging.target=GCS
credential: /path/to/credential
# gcs bucket name, required if you set remote-logging.target=GCS
# gcs bucket name, required if you set remote.logging.target=GCS
bucket.name: <your-bucket>
abs:
# abs account name, required if you set resource.storage.type=ABS
@ -46,4 +44,12 @@ remote-logging:
account.key: <your-account-key>
# abs container name, required if you set resource.storage.type=ABS
container.name: <your-container-name>
cos:
# cos access key id, required if you set remote-logging.target=COS
access.key.id: <access.key.id>
# cos access key secret, required if you set remote-logging.target=COS
access.key.secret: <access.key.secret>
# cos bucket name, required if you set remote-logging.target=COS
bucket.name: <bucket.name>
# cos region, required if you set remote-logging.target=COS
region: <region>

80
dolphinscheduler-common/src/main/resources/resource-center.yaml

@ -0,0 +1,80 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
resource:
# Tencent Cloud Storage (COS) setup, required if you set resource.storage.type=COS
tencent:
cloud:
access:
key:
id: <your-access-key-id>
secret: <your-access-key-secret>
cos:
# predefined region code: https://cloud.tencent.com/document/product/436/6224
region: ap-nanjing
bucket:
name: dolphinscheduler
# Microsoft Azure Storage (ABS) setup, required if you set resource.storage.type=ABS
azure:
client:
id: minioadmin
secret: minioadmin
subId: minioadmin
tenant:
id: minioadmin
blob:
storage:
connection:
string: <your-connection-string>
account:
name: <your-account-name>
container:
name: <your-container>
# Alibaba Cloud Storage (OSS) setup, required if you set resource.storage.type=OSS
alibaba:
cloud:
access:
key:
id: <your-access-key-id>
secret: <your-access-key-secret>
region: cn-hangzhou
oss:
bucket:
name: dolphinscheduler
endpoint: https://oss-cn-hangzhou.aliyuncs.com
# Google Cloud Storage (GCS) setup, required if you set resource.storage.type=GCS
google:
cloud:
storage:
credential: /path/to/credential
bucket:
name: <your-bucket>
# Huawei Cloud Storage (OBS), required if you set resource.storage.type=OBS
huawei:
cloud:
access:
key:
id: <your-access-key-id>
secret: <your-access-key-secret>
obs:
bucket:
name: dolphinscheduler
endpoint: obs.cn-southwest-2.huaweicloud.com

13
dolphinscheduler-common/src/test/resources/common.properties

@ -27,7 +27,7 @@ data.basedir.path=/tmp/dolphinscheduler
# resource view suffixs
#resource.view.suffixs=txt,log,sh,bat,conf,cfg,py,java,sql,xml,hql,properties,json,yml,yaml,ini,js
# resource storage type: LOCAL, HDFS, S3, OSS, GCS, ABS, OBS. LOCAL type is default type, and it's a specific type of HDFS with "resource.hdfs.fs.defaultFS = file:///" configuration
# resource storage type: LOCAL, HDFS, S3, OSS, GCS, ABS, OBS, COS. LOCAL type is default type, and it's a specific type of HDFS with "resource.hdfs.fs.defaultFS = file:///" configuration
# please notice that LOCAL mode does not support reading and writing in distributed mode, which mean you can only use your resource in one machine, unless
# use shared file mount point
resource.storage.type=LOCAL
@ -83,6 +83,17 @@ resource.azure.blob.storage.account.name=<your-account-name>
# abs connection string, required if you set resource.storage.type=ABS
resource.azure.blob.storage.connection.string=<your-connection-string>
# tencent cloud access key id, required if you set resource.storage.type=COS
resource.tencent.cloud.access.key.id=<your-access-key-id>
# access key secret, required if you set resource.storage.type=COS
resource.tencent.cloud.access.key.secret=<your-access-key-secret>
# cos bucket name, required if you set resource.storage.type=COS
resource.tencent.cloud.cos.bucket.name=dolphinscheduler
# cos bucket region, required if you set resource.storage.type=COS, see: https://cloud.tencent.com/document/product/436/6224
resource.tencent.cloud.cos.region=ap-nanjing
# if resource.storage.type=HDFS, the user must have the permission to create directories under the HDFS root path
resource.hdfs.root.user=hdfs
# if resource.storage.type=S3, the value like: s3a://dolphinscheduler; if resource.storage.type=HDFS and namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir

46
dolphinscheduler-common/src/test/resources/remote-logging.yaml

@ -15,41 +15,27 @@
# limitations under the License.
#
remote-logging:
# Whether to enable remote logging
enable: false
# if remote-logging.enable = true, set the target of remote logging
remote.logging:
# if remote.logging.enable = true, set the target of remote logging
target: OSS
# if remote-logging.enable = true, set the log base directory
# if remote.logging.enable = true, set the log base directory
base.dir: logs
# if remote-logging.enable = true, set the number of threads to send logs to remote storage
# if remote.logging.enable = true, set the number of threads to send logs to remote storage
thread.pool.size: 10
# required if you set remote-logging.target=OSS
# required if you set remote.logging.target=OSS
oss:
# oss access key id, required if you set remote-logging.target=OSS
# oss access key id, required if you set remote.logging.target=OSS
access.key.id: <access.key.id>
# oss access key secret, required if you set remote-logging.target=OSS
# oss access key secret, required if you set remote.logging.target=OSS
access.key.secret: <access.key.secret>
# oss bucket name, required if you set remote-logging.target=OSS
# oss bucket name, required if you set remote.logging.target=OSS
bucket.name: <bucket.name>
# oss endpoint, required if you set remote-logging.target=OSS
# oss endpoint, required if you set remote.logging.target=OSS
endpoint: <endpoint>
# required if you set remote-logging.target=S3
s3:
# s3 access key id, required if you set remote-logging.target=S3
access.key.id: <access.key.id>
# s3 access key secret, required if you set remote-logging.target=S3
access.key.secret: <access.key.secret>
# s3 bucket name, required if you set remote-logging.target=S3
bucket.name: <bucket.name>
# s3 endpoint, required if you set remote-logging.target=S3
endpoint: <endpoint>
# s3 region, required if you set remote-logging.target=S3
region: <region>
google.cloud.storage:
# the location of the google cloud credential, required if you set remote-logging.target=GCS
# the location of the google cloud credential, required if you set remote.logging.target=GCS
credential: /path/to/credential
# gcs bucket name, required if you set remote-logging.target=GCS
# gcs bucket name, required if you set remote.logging.target=GCS
bucket.name: <your-bucket>
abs:
# abs account name, required if you set resource.storage.type=ABS
@ -58,4 +44,12 @@ remote-logging:
account.key: <your-account-key>
# abs container name, required if you set resource.storage.type=ABS
container.name: <your-container-name>
cos:
# cos access key id, required if you set remote-logging.target=COS
access.key.id: <access.key.id>
# cos access key secret, required if you set remote-logging.target=COS
access.key.secret: <access.key.secret>
# cos bucket name, required if you set remote-logging.target=COS
bucket.name: <bucket.name>
# cos region, required if you set remote-logging.target=COS
region: <region>

26
dolphinscheduler-dist/release-docs/LICENSE vendored

@ -555,7 +555,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
proto-google-cloud-storage-v2 2.18.0-alpha: https://mvnrepository.com/artifact/com.google.api.grpc/proto-google-cloud-storage-v2/2.18.0-alpha, Apache 2.0
proto-google-iam-v1 1.9.0: https://mvnrepository.com/artifact/com.google.api.grpc/proto-google-iam-v1/1.9.0, Apache 2.0
sshd-sftp https://mvnrepository.com/artifact/org.apache.sshd/sshd-sftp/2.8.0 Apache 2.0
sshd-scp https://mvnrepository.com/artifact/org.apache.sshd/sshd-scp/2.8.0 Aapache 2.0
sshd-scp https://mvnrepository.com/artifact/org.apache.sshd/sshd-scp/2.8.0 Apache 2.0
casdoor-java-sdk 1.11.0 https://mvnrepository.com/artifact/org.casbin/casdoor-java-sdk/1.11.0 Apache 2.0
casdoor-spring-boot-starter 1.6.0 https://mvnrepository.com/artifact/org.casbin/casdoor-spring-boot-starter/1.6.0 Apache 2.0
org.apache.oltu.oauth2.client 1.0.2 https://mvnrepository.com/artifact/org.apache.oltu.oauth2/org.apache.oltu.oauth2.client/1.0.2 Apache 2.0
@ -572,8 +572,9 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
tea-rpc-util 0.1.3.jar https://github.com/aliyun/aliyun-openapi-java-sdk/blob/master/README.md#license Apache 2.0
tea-util 0.2.13.jar https://github.com/aliyun/aliyun-openapi-java-sdk/blob/master/README.md#license Apache 2.0
delight-nashorn-sandbox 0.3.2 https://github.com/javadelight/delight-nashorn-sandbox/blob/master/README.md#license Apache 2.0
tencentcloud-sdk-java-common 3.1.213 https://mvnrepository.com/artifact/com.tencentcloudapi/tencentcloud-sdk-java-common/3.1.213, Apache 2.0
tencentcloud-sdk-java-kms 3.1.213 https://mvnrepository.com/artifact/com.tencentcloudapi/tencentcloud-sdk-java-kms/3.1.213, Apache 2.0
logging-interceptor 2.7.5 https://mvnrepository.com/artifact/com.squareup.okhttp/logging-interceptor/2.7.5, Apache 2.0
@ -616,6 +617,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
google-auth-library-oauth2-http 1.15.0: https://mvnrepository.com/artifact/com.google.auth/google-auth-library-oauth2-http/1.15.0, BSD 3-clause
threetenbp 1.6.5: https://mvnrepository.com/artifact/org.threeten/threetenbp/1.6.5, BSD 3-clause
dom4j 2.1.3 https://github.com/dom4j/dom4j/blob/master/LICENSE BSD 3-clause
xstream 1.4.20 https://mvnrepository.com/artifact/com.thoughtworks.xstream/xstream/1.4.20, BSD 3-clause
========================================================================
CDDL licenses
@ -713,6 +715,8 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
azure-core-management 1.10.1: https://mvnrepository.com/artifact/com.azure/azure-core-management/1.10.1, MIT
azure-storage-blob 12.21.0: https://mvnrepository.com/artifact/com.azure/azure-storage-blob/12.21.0, MIT
azure-storage-internal-avro 12.6.0: https://mvnrepository.com/artifact/com.azure/azure-storage-internal-avro/12.6.0, MIT
cos_api 5.6.231 https://mvnrepository.com/artifact/com.qcloud/cos_api/5.6.231, MIT
java-jwt 4.4.0 https://mvnrepository.com/artifact/com.auth0/java-jwt/4.4.0, MIT
========================================================================
MPL 1.1 licenses
@ -813,3 +817,19 @@ JDOM licenses
Vertica Client Driver License
========================================
vertica-jdbc 12.0.4-0: https://mvnrepository.com/artifact/com.vertica.jdbc/vertica-jdbc/12.0.4-0, Vertica Client Driver License
========================================================================
Indiana University Extreme! Lab Software License
========================================================================
The following components are provided under a Indiana University Extreme! Lab Software License. See project link for details.
The text of each license is also included at licenses/LICENSE-[project].txt.
mxparser 1.2.2: https://mvnrepository.com/artifact/io.github.x-stream/mxparser/1.2.2, Indiana University Extreme! Lab Software License
========================================================================
Public Domain License
========================================================================
The following components are provided under a Public Domain License. See project link for details.
The text of each license is also included at licenses/LICENSE-[project].txt.
xmlpull 1.1.3.1: https://mvnrepository.com/artifact/xmlpull/xmlpull/1.1.3.1, Public Domain License

22
dolphinscheduler-dist/release-docs/licenses/LICENSE-cos_api.txt vendored

@ -0,0 +1,22 @@
MIT License
Copyright (c) 2017 腾讯云
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

22
dolphinscheduler-dist/release-docs/licenses/LICENSE-java-jwt.txt vendored

@ -0,0 +1,22 @@
The MIT License (MIT)
Copyright (c) 2015 Auth0, Inc. <support@auth0.com> (http://auth0.com)
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

203
dolphinscheduler-dist/release-docs/licenses/LICENSE-logging-interceptor.txt vendored

@ -0,0 +1,203 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

57
dolphinscheduler-dist/release-docs/licenses/LICENSE-mxparser.txt vendored

@ -0,0 +1,57 @@
Indiana University Extreme! Lab Software License, Version 1.2
Copyright (C) 2003 The Trustees of Indiana University.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1) All redistributions of source code must retain the above
copyright notice, the list of authors in the original source
code, this list of conditions and the disclaimer listed in this
license;
2) All redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the disclaimer
listed in this license in the documentation and/or other
materials provided with the distribution;
3) Any documentation included with all redistributions must include
the following acknowledgement:
"This product includes software developed by the Indiana
University Extreme! Lab. For further information please visit
http://www.extreme.indiana.edu/"
Alternatively, this acknowledgment may appear in the software
itself, and wherever such third-party acknowledgments normally
appear.
4) The name "Indiana University" or "Indiana University
Extreme! Lab" shall not be used to endorse or promote
products derived from this software without prior written
permission from Indiana University. For written permission,
please contact http://www.extreme.indiana.edu/.
5) Products derived from this software may not use "Indiana
University" name nor may "Indiana University" appear in their name,
without prior written permission of the Indiana University.
Indiana University provides no reassurances that the source code
provided does not infringe the patent or any other intellectual
property rights of any other entity. Indiana University disclaims any
liability to any recipient for claims brought by any other entity
based on infringement of intellectual property rights or otherwise.
LICENSEE UNDERSTANDS THAT SOFTWARE IS PROVIDED "AS IS" FOR WHICH
NO WARRANTIES AS TO CAPABILITIES OR ACCURACY ARE MADE. INDIANA
UNIVERSITY GIVES NO WARRANTIES AND MAKES NO REPRESENTATION THAT
SOFTWARE IS FREE OF INFRINGEMENT OF THIRD PARTY PATENT, COPYRIGHT, OR
OTHER PROPRIETARY RIGHTS. INDIANA UNIVERSITY MAKES NO WARRANTIES THAT
SOFTWARE IS FREE FROM "BUGS", "VIRUSES", "TROJAN HORSES", "TRAP
DOORS", "WORMS", OR OTHER HARMFUL CODE. LICENSEE ASSUMES THE ENTIRE
RISK AS TO THE PERFORMANCE OF SOFTWARE AND/OR ASSOCIATED MATERIALS,
AND TO THE PERFORMANCE AND VALIDITY OF INFORMATION GENERATED USING
SOFTWARE.

203
dolphinscheduler-dist/release-docs/licenses/LICENSE-tencentcloud-sdk-java-common.txt vendored

@ -0,0 +1,203 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

203
dolphinscheduler-dist/release-docs/licenses/LICENSE-tencentcloud-sdk-java-kms.txt vendored

@ -0,0 +1,203 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

15
dolphinscheduler-dist/release-docs/licenses/LICENSE-xmlpull.txt vendored

@ -0,0 +1,15 @@
XMLPULL API IS FREE
-------------------
All of the XMLPULL API source code, compiled code, and documentation
contained in this distribution *except* for tests (see separate LICENSE_TESTS.txt)
are in the Public Domain.
XMLPULL API comes with NO WARRANTY or guarantee of fitness for any purpose.
Initial authors:
Stefan Haustein
Aleksander Slominski
2001-12-12

29
dolphinscheduler-dist/release-docs/licenses/LICENSE-xstream.txt vendored

@ -0,0 +1,29 @@
Copyright (c) 2003-2006, Joe Walnes
Copyright (c) 2006-2015 XStream Committers
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this list of
conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice, this list of
conditions and the following disclaimer in the documentation and/or other materials provided
with the distribution.
3. Neither the name of XStream nor the names of its contributors may be used to endorse
or promote products derived from this software without specific prior written
permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY
WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
DAMAGE.

5
dolphinscheduler-storage-plugin/dolphinscheduler-storage-all/pom.xml

@ -62,6 +62,11 @@
<artifactId>dolphinscheduler-storage-obs</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-storage-cos</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
</project>

6
dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageType.java

@ -29,7 +29,11 @@ public enum StorageType {
ABS(5, "ABS"),
OBS(6, "OBS");
OBS(6, "OBS"),
COS(7, "COS"),
;
private final int code;
private final String name;

70
dolphinscheduler-storage-plugin/dolphinscheduler-storage-cos/pom.xml

@ -0,0 +1,70 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-storage-plugin</artifactId>
<version>dev-SNAPSHOT</version>
</parent>
<artifactId>dolphinscheduler-storage-cos</artifactId>
<properties>
<plugin.name>storage.cos</plugin.name>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-common</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-storage-api</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.qcloud</groupId>
<artifactId>cos_api</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>shade</goal>
</goals>
<phase>package</phase>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

30
dolphinscheduler-storage-plugin/dolphinscheduler-storage-cos/src/main/java/org/apache/dolphinscheduler/plugin/storage/cos/CosStorageConstants.java

@ -0,0 +1,30 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.storage.cos;
public class CosStorageConstants {
// Tencent Cloud Storage (COS) configuration
public static final String TENCENT_CLOUD_COS_PROPERTY_PREFIX = "resource.tencent.cloud";
public static final String TENCENT_CLOUD_COS_BUCKET_NAME = "resource.tencent.cloud.cos.bucket.name";
public static final String TENCENT_CLOUD_COS_REGION = "resource.tencent.cloud.cos.region";
public static final String TENCENT_CLOUD_ACCESS_KEY_ID = "resource.tencent.cloud.access.key.id";
public static final String TENCENT_CLOUD_ACCESS_KEY_SECRET = "resource.tencent.cloud.access.key.secret";
public static final String DEFAULT_COS_RESOURCE_UPLOAD_PATH = "/dolphinscheduler";
}

338
dolphinscheduler-storage-plugin/dolphinscheduler-storage-cos/src/main/java/org/apache/dolphinscheduler/plugin/storage/cos/CosStorageOperator.java

@ -0,0 +1,338 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.storage.cos;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.plugin.storage.api.AbstractStorageOperator;
import org.apache.dolphinscheduler.plugin.storage.api.ResourceMetadata;
import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.apache.dolphinscheduler.plugin.storage.api.constants.StorageConstants;
import org.apache.commons.lang3.StringUtils;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.stream.Collectors;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import com.qcloud.cos.COSClient;
import com.qcloud.cos.ClientConfig;
import com.qcloud.cos.auth.BasicCOSCredentials;
import com.qcloud.cos.auth.COSCredentials;
import com.qcloud.cos.exception.CosServiceException;
import com.qcloud.cos.http.HttpProtocol;
import com.qcloud.cos.model.COSObject;
import com.qcloud.cos.model.COSObjectSummary;
import com.qcloud.cos.model.CopyObjectRequest;
import com.qcloud.cos.model.CopyResult;
import com.qcloud.cos.model.GetObjectRequest;
import com.qcloud.cos.model.ListObjectsRequest;
import com.qcloud.cos.model.ObjectListing;
import com.qcloud.cos.model.ObjectMetadata;
import com.qcloud.cos.model.PutObjectRequest;
import com.qcloud.cos.model.UploadResult;
import com.qcloud.cos.region.Region;
import com.qcloud.cos.transfer.Copy;
import com.qcloud.cos.transfer.Download;
import com.qcloud.cos.transfer.TransferManager;
import com.qcloud.cos.transfer.TransferManagerConfiguration;
import com.qcloud.cos.transfer.Upload;
@Slf4j
public class CosStorageOperator extends AbstractStorageOperator implements Closeable, StorageOperator {
private static final int TRANSFER_THREAD_POOL_SIZE = 16;
private static final long MULTIPART_UPLOAD_BYTES_THRESHOLD = 5 * 1024 * 1024L;
private static final long MIN_UPLOAD_PART_BYTES = 1024 * 1024L;
private final String bucketName;
private final COSClient cosClient;
private final TransferManager cosTransferManager;
public CosStorageOperator(CosStorageProperties cosStorageProperties) {
super(cosStorageProperties.getResourceUploadPath());
String secretId = cosStorageProperties.getAccessKeyId();
String secretKey = cosStorageProperties.getAccessKeySecret();
COSCredentials cosCredentials = new BasicCOSCredentials(secretId, secretKey);
String regionName = cosStorageProperties.getRegion();
ClientConfig clientConfig = new ClientConfig(new Region(regionName));
clientConfig.setHttpProtocol(HttpProtocol.https);
this.cosClient = new COSClient(cosCredentials, clientConfig);
this.bucketName = cosStorageProperties.getBucketName();
ensureBucketSuccessfullyCreated(bucketName);
this.cosTransferManager = getCosTransferManager();
}
@Override
public void close() throws IOException {
this.cosTransferManager.shutdownNow(true);
}
@Override
public String getStorageBaseDirectory() {
// All directory should end with File.separator
if (resourceBaseAbsolutePath.startsWith(File.separator)) {
String warnMessage =
String.format("%s -> %s should not start with %s in tencent cos",
StorageConstants.RESOURCE_UPLOAD_PATH,
resourceBaseAbsolutePath, File.separator);
log.warn(warnMessage);
return resourceBaseAbsolutePath.substring(1);
}
return resourceBaseAbsolutePath;
}
@SneakyThrows
@Override
public void createStorageDir(String directoryAbsolutePath) {
String cosKey = transformAbsolutePathToCOSKey(directoryAbsolutePath);
if (cosClient.doesObjectExist(bucketName, cosKey)) {
throw new FileAlreadyExistsException("directory: " + cosKey + " already exists");
}
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(0L);
InputStream emptyContent = new ByteArrayInputStream(new byte[0]);
PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, cosKey, emptyContent, metadata);
cosClient.putObject(putObjectRequest);
}
@SneakyThrows
@Override
public void download(String srcFilePath, String dstFilePath, boolean overwrite) {
String cosKey = transformAbsolutePathToCOSKey(srcFilePath);
Path dsTempFolder = Paths.get(FileUtils.DATA_BASEDIR).normalize().toAbsolutePath();
Path fileDownloadPathNormalized = dsTempFolder.resolve(dstFilePath).normalize().toAbsolutePath();
if (!fileDownloadPathNormalized.startsWith(dsTempFolder)) {
// if the destination file path is NOT in DS temp folder (e.g., '/tmp/dolphinscheduler'),
// an IllegalArgumentException should be thrown.
throw new IllegalArgumentException("failed to download to " + fileDownloadPathNormalized);
}
File dstFile = fileDownloadPathNormalized.toFile();
if (dstFile.isDirectory()) {
Files.delete(dstFile.toPath());
} else {
FileUtils.createDirectoryWith755(dstFile.getParentFile().toPath());
}
GetObjectRequest getObjectRequest = new GetObjectRequest(bucketName, cosKey);
Download download = cosTransferManager.download(getObjectRequest, dstFile);
download.waitForCompletion();
}
@Override
public boolean exists(String fileName) {
String cosKey = transformAbsolutePathToCOSKey(fileName);
return cosClient.doesObjectExist(bucketName, cosKey);
}
@Override
public void delete(String filePath, boolean recursive) {
String cosKey = transformAbsolutePathToCOSKey(filePath);
cosClient.deleteObject(bucketName, cosKey);
}
@SneakyThrows
@Override
public void copy(String srcPath, String dstPath, boolean deleteSource, boolean overwrite) {
String srcCosKey = transformAbsolutePathToCOSKey(srcPath);
String destCosKey = transformAbsolutePathToCOSKey(dstPath);
CopyObjectRequest copyObjectRequest = new CopyObjectRequest(bucketName, srcCosKey, bucketName, destCosKey);
Copy copy = cosTransferManager.copy(copyObjectRequest, cosClient, null);
CopyResult copyResult = copy.waitForCopyResult();
if (copyResult != null && deleteSource) {
cosClient.deleteObject(bucketName, srcPath);
}
}
@SneakyThrows
@Override
public void upload(String srcFile, String dstPath, boolean deleteSource, boolean overwrite) {
dstPath = transformAbsolutePathToCOSKey(dstPath);
if (cosClient.doesObjectExist(bucketName, dstPath)) {
if (!overwrite) {
throw new CosServiceException("file: " + dstPath + " already exists");
} else {
cosClient.deleteObject(bucketName, dstPath);
}
}
PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, dstPath, new File(srcFile));
Upload upload = cosTransferManager.upload(putObjectRequest);
UploadResult uploadResult = upload.waitForUploadResult();
if (uploadResult != null && deleteSource) {
Files.delete(Paths.get(srcFile));
}
}
@SneakyThrows
@Override
public List<String> fetchFileContent(String filePath, int skipLineNums, int limit) {
String cosKey = transformAbsolutePathToCOSKey(filePath);
COSObject cosObject = cosClient.getObject(bucketName, cosKey);
try (
InputStreamReader inputStreamReader = new InputStreamReader(cosObject.getObjectContent());
BufferedReader bufferedReader = new BufferedReader(inputStreamReader)) {
return bufferedReader
.lines()
.skip(skipLineNums)
.limit(limit)
.collect(Collectors.toList());
}
}
@Override
public List<StorageEntity> listStorageEntity(String resourceAbsolutePath) {
resourceAbsolutePath = transformCOSKeyToAbsolutePath(resourceAbsolutePath);
ListObjectsRequest request = new ListObjectsRequest();
request.setBucketName(bucketName);
request.setPrefix(resourceAbsolutePath);
request.setDelimiter(File.separator);
ObjectListing result = cosClient.listObjects(request);
return result.getObjectSummaries()
.stream()
.map((COSObjectSummary summary) -> {
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(summary.getSize());
metadata.setLastModified(summary.getLastModified());
COSObject object = new COSObject();
object.setObjectMetadata(metadata);
object.setKey(summary.getKey());
return transformCOSObjectToStorageEntity(object);
})
.collect(Collectors.toList());
}
@Override
public List<StorageEntity> listFileStorageEntityRecursively(String resourceAbsolutePath) {
resourceAbsolutePath = transformCOSKeyToAbsolutePath(resourceAbsolutePath);
Set<String> visited = new HashSet<>();
List<StorageEntity> storageEntityList = new ArrayList<>();
LinkedList<String> foldersToFetch = new LinkedList<>();
foldersToFetch.addLast(resourceAbsolutePath);
while (!foldersToFetch.isEmpty()) {
String pathToExplore = foldersToFetch.pop();
visited.add(pathToExplore);
List<StorageEntity> storageEntities = listStorageEntity(pathToExplore);
for (StorageEntity entity : storageEntities) {
if (entity.isDirectory()) {
if (visited.contains(entity.getFullName())) {
continue;
}
foldersToFetch.add(entity.getFullName());
}
}
storageEntityList.addAll(storageEntities);
}
return storageEntityList;
}
@Override
public StorageEntity getStorageEntity(String resourceAbsolutePath) {
String cosKey = transformCOSKeyToAbsolutePath(resourceAbsolutePath);
COSObject object = cosClient.getObject(bucketName, cosKey);
return transformCOSObjectToStorageEntity(object);
}
public void ensureBucketSuccessfullyCreated(String bucketName) {
if (StringUtils.isBlank(bucketName)) {
throw new IllegalArgumentException(CosStorageConstants.TENCENT_CLOUD_COS_BUCKET_NAME + " is empty");
}
if (!cosClient.doesBucketExist(bucketName)) {
throw new IllegalArgumentException(
"bucketName: " + bucketName + " is not exists, you need to create them by yourself");
}
log.info("bucketName: {} has been found", bucketName);
}
protected StorageEntity transformCOSObjectToStorageEntity(COSObject object) {
ObjectMetadata metadata = object.getObjectMetadata();
String fileAbsolutePath = transformCOSKeyToAbsolutePath(object.getKey());
ResourceMetadata resourceMetaData = getResourceMetaData(fileAbsolutePath);
String fileExtension = com.google.common.io.Files.getFileExtension(resourceMetaData.getResourceAbsolutePath());
return StorageEntity.builder()
.fileName(new File(fileAbsolutePath).getName())
.fullName(fileAbsolutePath)
.pfullName(resourceMetaData.getResourceParentAbsolutePath())
.type(resourceMetaData.getResourceType())
.isDirectory(StringUtils.isEmpty(fileExtension))
.size(metadata.getContentLength())
.createTime(metadata.getLastModified())
.updateTime(metadata.getLastModified())
.build();
}
private String transformAbsolutePathToCOSKey(String absolutePath) {
ResourceMetadata resourceMetaData = getResourceMetaData(absolutePath);
if (resourceMetaData.isDirectory()) {
return FileUtils.concatFilePath(absolutePath, File.separator);
}
return absolutePath;
}
private String transformCOSKeyToAbsolutePath(String cosKey) {
if (cosKey.endsWith(File.separator)) {
return cosKey.substring(0, cosKey.length() - 1);
}
return cosKey;
}
private TransferManager getCosTransferManager() {
ExecutorService threadPool = Executors.newFixedThreadPool(TRANSFER_THREAD_POOL_SIZE);
TransferManager transferManager = new TransferManager(cosClient, threadPool);
TransferManagerConfiguration transferManagerConfiguration = new TransferManagerConfiguration();
transferManagerConfiguration.setMultipartUploadThreshold(MULTIPART_UPLOAD_BYTES_THRESHOLD);
transferManagerConfiguration.setMinimumUploadPartSize(MIN_UPLOAD_PART_BYTES);
transferManager.setConfiguration(transferManagerConfiguration);
return transferManager;
}
}

59
dolphinscheduler-storage-plugin/dolphinscheduler-storage-cos/src/main/java/org/apache/dolphinscheduler/plugin/storage/cos/CosStorageOperatorFactory.java

@ -0,0 +1,59 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.storage.cos;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperatorFactory;
import org.apache.dolphinscheduler.plugin.storage.api.StorageType;
import org.apache.dolphinscheduler.plugin.storage.api.constants.StorageConstants;
import java.util.Map;
import com.google.auto.service.AutoService;
@AutoService(StorageOperatorFactory.class)
public class CosStorageOperatorFactory implements StorageOperatorFactory {
@Override
public StorageOperator createStorageOperate() {
final CosStorageProperties cosStorageProperties = getCosStorageProperties();
return new CosStorageOperator(cosStorageProperties);
}
@Override
public StorageType getStorageOperate() {
return StorageType.COS;
}
private CosStorageProperties getCosStorageProperties() {
Map<String, String> cosPropertiesMap =
PropertyUtils.getByPrefix(CosStorageConstants.TENCENT_CLOUD_COS_PROPERTY_PREFIX);
return CosStorageProperties.builder()
.region(cosPropertiesMap.get(CosStorageConstants.TENCENT_CLOUD_COS_REGION))
.accessKeyId(cosPropertiesMap.get(CosStorageConstants.TENCENT_CLOUD_ACCESS_KEY_ID))
.accessKeySecret(cosPropertiesMap.get(CosStorageConstants.TENCENT_CLOUD_ACCESS_KEY_SECRET))
.bucketName(cosPropertiesMap.get(CosStorageConstants.TENCENT_CLOUD_COS_BUCKET_NAME))
.resourceUploadPath(
cosPropertiesMap.getOrDefault(StorageConstants.RESOURCE_UPLOAD_PATH,
CosStorageConstants.DEFAULT_COS_RESOURCE_UPLOAD_PATH))
.build();
}
}

40
dolphinscheduler-storage-plugin/dolphinscheduler-storage-cos/src/main/java/org/apache/dolphinscheduler/plugin/storage/cos/CosStorageProperties.java

@ -0,0 +1,40 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.storage.cos;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class CosStorageProperties {
private String region;
private String accessKeyId;
private String accessKeySecret;
private String bucketName;
private String resourceUploadPath;
}

21
dolphinscheduler-storage-plugin/dolphinscheduler-storage-cos/src/test/resources/logback.xml

@ -0,0 +1,21 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<configuration>
<logger name="*" level="ERROR"/>
</configuration>

1
dolphinscheduler-storage-plugin/pom.xml

@ -36,6 +36,7 @@
<module>dolphinscheduler-storage-gcs</module>
<module>dolphinscheduler-storage-abs</module>
<module>dolphinscheduler-storage-obs</module>
<module>dolphinscheduler-storage-cos</module>
</modules>
<dependencyManagement>

9
tools/dependencies/known-dependencies.txt

@ -24,7 +24,6 @@ aws-java-sdk-dms-1.12.300.jar
aws-json-protocol-2.17.282.jar
bcpkix-jdk15on-1.69.jar
bcprov-ext-jdk15on-1.69.jar
bcprov-jdk15on-1.69.jar
bcutil-jdk15on-1.69.jar
bonecp-0.8.0.RELEASE.jar
bucket4j-core-6.2.0.jar
@ -483,6 +482,14 @@ azure-storage-blob-12.21.0.jar
azure-storage-internal-avro-12.6.0.jar
vertica-jdbc-12.0.4-0.jar
esdk-obs-java-bundle-3.23.3.jar
cos_api-5.6.231.jar
java-jwt-4.4.0.jar
logging-interceptor-2.7.5.jar
mxparser-1.2.2.jar
tencentcloud-sdk-java-common-3.1.213.jar
tencentcloud-sdk-java-kms-3.1.213.jar
xmlpull-1.1.3.1.jar
xstream-1.4.20.jar
dyvmsapi20170525-2.1.4.jar
alibabacloud-gateway-spi-0.0.1.jar
credentials-java-0.2.2.jar

Loading…
Cancel
Save