Mighten Dai
2 months ago
committed by
GitHub
32 changed files with 1695 additions and 118 deletions
@ -0,0 +1,112 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.common.log.remote; |
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.common.constants.Constants; |
||||||
|
import org.apache.dolphinscheduler.common.utils.PropertyUtils; |
||||||
|
|
||||||
|
import org.apache.commons.lang3.StringUtils; |
||||||
|
|
||||||
|
import java.io.Closeable; |
||||||
|
import java.io.File; |
||||||
|
import java.io.IOException; |
||||||
|
|
||||||
|
import lombok.extern.slf4j.Slf4j; |
||||||
|
|
||||||
|
import com.qcloud.cos.COSClient; |
||||||
|
import com.qcloud.cos.ClientConfig; |
||||||
|
import com.qcloud.cos.auth.BasicCOSCredentials; |
||||||
|
import com.qcloud.cos.auth.COSCredentials; |
||||||
|
import com.qcloud.cos.http.HttpProtocol; |
||||||
|
import com.qcloud.cos.model.GetObjectRequest; |
||||||
|
import com.qcloud.cos.model.PutObjectRequest; |
||||||
|
import com.qcloud.cos.region.Region; |
||||||
|
|
||||||
|
@Slf4j |
||||||
|
public class CosRemoteLogHandler implements RemoteLogHandler, Closeable { |
||||||
|
|
||||||
|
private final COSClient cosClient; |
||||||
|
|
||||||
|
private final String bucketName; |
||||||
|
|
||||||
|
private static CosRemoteLogHandler instance; |
||||||
|
|
||||||
|
private CosRemoteLogHandler() { |
||||||
|
String secretId = PropertyUtils.getString(Constants.REMOTE_LOGGING_COS_ACCESS_KEY_ID); |
||||||
|
String secretKey = PropertyUtils.getString(Constants.REMOTE_LOGGING_COS_ACCESS_KEY_SECRET); |
||||||
|
COSCredentials cosCredentials = new BasicCOSCredentials(secretId, secretKey); |
||||||
|
String regionName = PropertyUtils.getString(Constants.REMOTE_LOGGING_COS_REGION); |
||||||
|
ClientConfig clientConfig = new ClientConfig(new Region(regionName)); |
||||||
|
clientConfig.setHttpProtocol(HttpProtocol.https); |
||||||
|
this.cosClient = new COSClient(cosCredentials, clientConfig); |
||||||
|
this.bucketName = PropertyUtils.getString(Constants.REMOTE_LOGGING_COS_BUCKET_NAME); |
||||||
|
checkBucketNameExists(this.bucketName); |
||||||
|
} |
||||||
|
|
||||||
|
public static synchronized CosRemoteLogHandler getInstance() { |
||||||
|
if (instance == null) { |
||||||
|
instance = new CosRemoteLogHandler(); |
||||||
|
} |
||||||
|
return instance; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public void sendRemoteLog(String logPath) { |
||||||
|
String objectName = RemoteLogUtils.getObjectNameFromLogPath(logPath); |
||||||
|
|
||||||
|
try { |
||||||
|
log.info("send remote log from {} to tencent cos {}", logPath, objectName); |
||||||
|
PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, objectName, new File(logPath)); |
||||||
|
cosClient.putObject(putObjectRequest); |
||||||
|
} catch (Exception e) { |
||||||
|
log.error("error while sending remote log from {} to tencent cos {}, reason:", logPath, objectName, e); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public void getRemoteLog(String logPath) { |
||||||
|
String objectName = RemoteLogUtils.getObjectNameFromLogPath(logPath); |
||||||
|
|
||||||
|
try { |
||||||
|
log.info("get remote log from tencent cos {} to {}", objectName, logPath); |
||||||
|
cosClient.getObject(new GetObjectRequest(bucketName, objectName), new File(logPath)); |
||||||
|
} catch (Exception e) { |
||||||
|
log.error("error while sending remote log from {} to tencent cos {}, reason:", objectName, logPath, e); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public void close() throws IOException { |
||||||
|
if (cosClient != null) { |
||||||
|
cosClient.shutdown(); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
private void checkBucketNameExists(String bucketName) { |
||||||
|
if (StringUtils.isBlank(bucketName)) { |
||||||
|
throw new IllegalArgumentException(Constants.REMOTE_LOGGING_COS_BUCKET_NAME + " is empty"); |
||||||
|
} |
||||||
|
boolean existsBucket = cosClient.doesBucketExist(bucketName); |
||||||
|
if (!existsBucket) { |
||||||
|
throw new IllegalArgumentException( |
||||||
|
"bucketName: " + bucketName + " does not exists, you should create it first"); |
||||||
|
} |
||||||
|
|
||||||
|
log.debug("tencent cos bucket {} has been found for remote logging", bucketName); |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,80 @@ |
|||||||
|
# |
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
# contributor license agreements. See the NOTICE file distributed with |
||||||
|
# this work for additional information regarding copyright ownership. |
||||||
|
# The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
# (the "License"); you may not use this file except in compliance with |
||||||
|
# the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, software |
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
# See the License for the specific language governing permissions and |
||||||
|
# limitations under the License. |
||||||
|
# |
||||||
|
|
||||||
|
resource: |
||||||
|
# Tencent Cloud Storage (COS) setup, required if you set resource.storage.type=COS |
||||||
|
tencent: |
||||||
|
cloud: |
||||||
|
access: |
||||||
|
key: |
||||||
|
id: <your-access-key-id> |
||||||
|
secret: <your-access-key-secret> |
||||||
|
cos: |
||||||
|
# predefined region code: https://cloud.tencent.com/document/product/436/6224 |
||||||
|
region: ap-nanjing |
||||||
|
bucket: |
||||||
|
name: dolphinscheduler |
||||||
|
|
||||||
|
# Microsoft Azure Storage (ABS) setup, required if you set resource.storage.type=ABS |
||||||
|
azure: |
||||||
|
client: |
||||||
|
id: minioadmin |
||||||
|
secret: minioadmin |
||||||
|
subId: minioadmin |
||||||
|
tenant: |
||||||
|
id: minioadmin |
||||||
|
blob: |
||||||
|
storage: |
||||||
|
connection: |
||||||
|
string: <your-connection-string> |
||||||
|
account: |
||||||
|
name: <your-account-name> |
||||||
|
container: |
||||||
|
name: <your-container> |
||||||
|
|
||||||
|
# Alibaba Cloud Storage (OSS) setup, required if you set resource.storage.type=OSS |
||||||
|
alibaba: |
||||||
|
cloud: |
||||||
|
access: |
||||||
|
key: |
||||||
|
id: <your-access-key-id> |
||||||
|
secret: <your-access-key-secret> |
||||||
|
region: cn-hangzhou |
||||||
|
oss: |
||||||
|
bucket: |
||||||
|
name: dolphinscheduler |
||||||
|
endpoint: https://oss-cn-hangzhou.aliyuncs.com |
||||||
|
|
||||||
|
# Google Cloud Storage (GCS) setup, required if you set resource.storage.type=GCS |
||||||
|
google: |
||||||
|
cloud: |
||||||
|
storage: |
||||||
|
credential: /path/to/credential |
||||||
|
bucket: |
||||||
|
name: <your-bucket> |
||||||
|
|
||||||
|
# Huawei Cloud Storage (OBS), required if you set resource.storage.type=OBS |
||||||
|
huawei: |
||||||
|
cloud: |
||||||
|
access: |
||||||
|
key: |
||||||
|
id: <your-access-key-id> |
||||||
|
secret: <your-access-key-secret> |
||||||
|
obs: |
||||||
|
bucket: |
||||||
|
name: dolphinscheduler |
||||||
|
endpoint: obs.cn-southwest-2.huaweicloud.com |
||||||
@ -0,0 +1,22 @@ |
|||||||
|
MIT License |
||||||
|
|
||||||
|
Copyright (c) 2017 腾讯云 |
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy |
||||||
|
of this software and associated documentation files (the "Software"), to deal |
||||||
|
in the Software without restriction, including without limitation the rights |
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
||||||
|
copies of the Software, and to permit persons to whom the Software is |
||||||
|
furnished to do so, subject to the following conditions: |
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all |
||||||
|
copies or substantial portions of the Software. |
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
||||||
|
SOFTWARE. |
||||||
|
|
||||||
@ -0,0 +1,22 @@ |
|||||||
|
The MIT License (MIT) |
||||||
|
|
||||||
|
Copyright (c) 2015 Auth0, Inc. <support@auth0.com> (http://auth0.com) |
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy |
||||||
|
of this software and associated documentation files (the "Software"), to deal |
||||||
|
in the Software without restriction, including without limitation the rights |
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
||||||
|
copies of the Software, and to permit persons to whom the Software is |
||||||
|
furnished to do so, subject to the following conditions: |
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all |
||||||
|
copies or substantial portions of the Software. |
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
||||||
|
SOFTWARE. |
||||||
|
|
||||||
@ -0,0 +1,203 @@ |
|||||||
|
|
||||||
|
Apache License |
||||||
|
Version 2.0, January 2004 |
||||||
|
http://www.apache.org/licenses/ |
||||||
|
|
||||||
|
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION |
||||||
|
|
||||||
|
1. Definitions. |
||||||
|
|
||||||
|
"License" shall mean the terms and conditions for use, reproduction, |
||||||
|
and distribution as defined by Sections 1 through 9 of this document. |
||||||
|
|
||||||
|
"Licensor" shall mean the copyright owner or entity authorized by |
||||||
|
the copyright owner that is granting the License. |
||||||
|
|
||||||
|
"Legal Entity" shall mean the union of the acting entity and all |
||||||
|
other entities that control, are controlled by, or are under common |
||||||
|
control with that entity. For the purposes of this definition, |
||||||
|
"control" means (i) the power, direct or indirect, to cause the |
||||||
|
direction or management of such entity, whether by contract or |
||||||
|
otherwise, or (ii) ownership of fifty percent (50%) or more of the |
||||||
|
outstanding shares, or (iii) beneficial ownership of such entity. |
||||||
|
|
||||||
|
"You" (or "Your") shall mean an individual or Legal Entity |
||||||
|
exercising permissions granted by this License. |
||||||
|
|
||||||
|
"Source" form shall mean the preferred form for making modifications, |
||||||
|
including but not limited to software source code, documentation |
||||||
|
source, and configuration files. |
||||||
|
|
||||||
|
"Object" form shall mean any form resulting from mechanical |
||||||
|
transformation or translation of a Source form, including but |
||||||
|
not limited to compiled object code, generated documentation, |
||||||
|
and conversions to other media types. |
||||||
|
|
||||||
|
"Work" shall mean the work of authorship, whether in Source or |
||||||
|
Object form, made available under the License, as indicated by a |
||||||
|
copyright notice that is included in or attached to the work |
||||||
|
(an example is provided in the Appendix below). |
||||||
|
|
||||||
|
"Derivative Works" shall mean any work, whether in Source or Object |
||||||
|
form, that is based on (or derived from) the Work and for which the |
||||||
|
editorial revisions, annotations, elaborations, or other modifications |
||||||
|
represent, as a whole, an original work of authorship. For the purposes |
||||||
|
of this License, Derivative Works shall not include works that remain |
||||||
|
separable from, or merely link (or bind by name) to the interfaces of, |
||||||
|
the Work and Derivative Works thereof. |
||||||
|
|
||||||
|
"Contribution" shall mean any work of authorship, including |
||||||
|
the original version of the Work and any modifications or additions |
||||||
|
to that Work or Derivative Works thereof, that is intentionally |
||||||
|
submitted to Licensor for inclusion in the Work by the copyright owner |
||||||
|
or by an individual or Legal Entity authorized to submit on behalf of |
||||||
|
the copyright owner. For the purposes of this definition, "submitted" |
||||||
|
means any form of electronic, verbal, or written communication sent |
||||||
|
to the Licensor or its representatives, including but not limited to |
||||||
|
communication on electronic mailing lists, source code control systems, |
||||||
|
and issue tracking systems that are managed by, or on behalf of, the |
||||||
|
Licensor for the purpose of discussing and improving the Work, but |
||||||
|
excluding communication that is conspicuously marked or otherwise |
||||||
|
designated in writing by the copyright owner as "Not a Contribution." |
||||||
|
|
||||||
|
"Contributor" shall mean Licensor and any individual or Legal Entity |
||||||
|
on behalf of whom a Contribution has been received by Licensor and |
||||||
|
subsequently incorporated within the Work. |
||||||
|
|
||||||
|
2. Grant of Copyright License. Subject to the terms and conditions of |
||||||
|
this License, each Contributor hereby grants to You a perpetual, |
||||||
|
worldwide, non-exclusive, no-charge, royalty-free, irrevocable |
||||||
|
copyright license to reproduce, prepare Derivative Works of, |
||||||
|
publicly display, publicly perform, sublicense, and distribute the |
||||||
|
Work and such Derivative Works in Source or Object form. |
||||||
|
|
||||||
|
3. Grant of Patent License. Subject to the terms and conditions of |
||||||
|
this License, each Contributor hereby grants to You a perpetual, |
||||||
|
worldwide, non-exclusive, no-charge, royalty-free, irrevocable |
||||||
|
(except as stated in this section) patent license to make, have made, |
||||||
|
use, offer to sell, sell, import, and otherwise transfer the Work, |
||||||
|
where such license applies only to those patent claims licensable |
||||||
|
by such Contributor that are necessarily infringed by their |
||||||
|
Contribution(s) alone or by combination of their Contribution(s) |
||||||
|
with the Work to which such Contribution(s) was submitted. If You |
||||||
|
institute patent litigation against any entity (including a |
||||||
|
cross-claim or counterclaim in a lawsuit) alleging that the Work |
||||||
|
or a Contribution incorporated within the Work constitutes direct |
||||||
|
or contributory patent infringement, then any patent licenses |
||||||
|
granted to You under this License for that Work shall terminate |
||||||
|
as of the date such litigation is filed. |
||||||
|
|
||||||
|
4. Redistribution. You may reproduce and distribute copies of the |
||||||
|
Work or Derivative Works thereof in any medium, with or without |
||||||
|
modifications, and in Source or Object form, provided that You |
||||||
|
meet the following conditions: |
||||||
|
|
||||||
|
(a) You must give any other recipients of the Work or |
||||||
|
Derivative Works a copy of this License; and |
||||||
|
|
||||||
|
(b) You must cause any modified files to carry prominent notices |
||||||
|
stating that You changed the files; and |
||||||
|
|
||||||
|
(c) You must retain, in the Source form of any Derivative Works |
||||||
|
that You distribute, all copyright, patent, trademark, and |
||||||
|
attribution notices from the Source form of the Work, |
||||||
|
excluding those notices that do not pertain to any part of |
||||||
|
the Derivative Works; and |
||||||
|
|
||||||
|
(d) If the Work includes a "NOTICE" text file as part of its |
||||||
|
distribution, then any Derivative Works that You distribute must |
||||||
|
include a readable copy of the attribution notices contained |
||||||
|
within such NOTICE file, excluding those notices that do not |
||||||
|
pertain to any part of the Derivative Works, in at least one |
||||||
|
of the following places: within a NOTICE text file distributed |
||||||
|
as part of the Derivative Works; within the Source form or |
||||||
|
documentation, if provided along with the Derivative Works; or, |
||||||
|
within a display generated by the Derivative Works, if and |
||||||
|
wherever such third-party notices normally appear. The contents |
||||||
|
of the NOTICE file are for informational purposes only and |
||||||
|
do not modify the License. You may add Your own attribution |
||||||
|
notices within Derivative Works that You distribute, alongside |
||||||
|
or as an addendum to the NOTICE text from the Work, provided |
||||||
|
that such additional attribution notices cannot be construed |
||||||
|
as modifying the License. |
||||||
|
|
||||||
|
You may add Your own copyright statement to Your modifications and |
||||||
|
may provide additional or different license terms and conditions |
||||||
|
for use, reproduction, or distribution of Your modifications, or |
||||||
|
for any such Derivative Works as a whole, provided Your use, |
||||||
|
reproduction, and distribution of the Work otherwise complies with |
||||||
|
the conditions stated in this License. |
||||||
|
|
||||||
|
5. Submission of Contributions. Unless You explicitly state otherwise, |
||||||
|
any Contribution intentionally submitted for inclusion in the Work |
||||||
|
by You to the Licensor shall be under the terms and conditions of |
||||||
|
this License, without any additional terms or conditions. |
||||||
|
Notwithstanding the above, nothing herein shall supersede or modify |
||||||
|
the terms of any separate license agreement you may have executed |
||||||
|
with Licensor regarding such Contributions. |
||||||
|
|
||||||
|
6. Trademarks. This License does not grant permission to use the trade |
||||||
|
names, trademarks, service marks, or product names of the Licensor, |
||||||
|
except as required for reasonable and customary use in describing the |
||||||
|
origin of the Work and reproducing the content of the NOTICE file. |
||||||
|
|
||||||
|
7. Disclaimer of Warranty. Unless required by applicable law or |
||||||
|
agreed to in writing, Licensor provides the Work (and each |
||||||
|
Contributor provides its Contributions) on an "AS IS" BASIS, |
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or |
||||||
|
implied, including, without limitation, any warranties or conditions |
||||||
|
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A |
||||||
|
PARTICULAR PURPOSE. You are solely responsible for determining the |
||||||
|
appropriateness of using or redistributing the Work and assume any |
||||||
|
risks associated with Your exercise of permissions under this License. |
||||||
|
|
||||||
|
8. Limitation of Liability. In no event and under no legal theory, |
||||||
|
whether in tort (including negligence), contract, or otherwise, |
||||||
|
unless required by applicable law (such as deliberate and grossly |
||||||
|
negligent acts) or agreed to in writing, shall any Contributor be |
||||||
|
liable to You for damages, including any direct, indirect, special, |
||||||
|
incidental, or consequential damages of any character arising as a |
||||||
|
result of this License or out of the use or inability to use the |
||||||
|
Work (including but not limited to damages for loss of goodwill, |
||||||
|
work stoppage, computer failure or malfunction, or any and all |
||||||
|
other commercial damages or losses), even if such Contributor |
||||||
|
has been advised of the possibility of such damages. |
||||||
|
|
||||||
|
9. Accepting Warranty or Additional Liability. While redistributing |
||||||
|
the Work or Derivative Works thereof, You may choose to offer, |
||||||
|
and charge a fee for, acceptance of support, warranty, indemnity, |
||||||
|
or other liability obligations and/or rights consistent with this |
||||||
|
License. However, in accepting such obligations, You may act only |
||||||
|
on Your own behalf and on Your sole responsibility, not on behalf |
||||||
|
of any other Contributor, and only if You agree to indemnify, |
||||||
|
defend, and hold each Contributor harmless for any liability |
||||||
|
incurred by, or claims asserted against, such Contributor by reason |
||||||
|
of your accepting any such warranty or additional liability. |
||||||
|
|
||||||
|
END OF TERMS AND CONDITIONS |
||||||
|
|
||||||
|
APPENDIX: How to apply the Apache License to your work. |
||||||
|
|
||||||
|
To apply the Apache License to your work, attach the following |
||||||
|
boilerplate notice, with the fields enclosed by brackets "[]" |
||||||
|
replaced with your own identifying information. (Don't include |
||||||
|
the brackets!) The text should be enclosed in the appropriate |
||||||
|
comment syntax for the file format. We also recommend that a |
||||||
|
file or class name and description of purpose be included on the |
||||||
|
same "printed page" as the copyright notice for easier |
||||||
|
identification within third-party archives. |
||||||
|
|
||||||
|
Copyright [yyyy] [name of copyright owner] |
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License"); |
||||||
|
you may not use this file except in compliance with the License. |
||||||
|
You may obtain a copy of the License at |
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software |
||||||
|
distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
See the License for the specific language governing permissions and |
||||||
|
limitations under the License. |
||||||
|
|
||||||
@ -0,0 +1,57 @@ |
|||||||
|
Indiana University Extreme! Lab Software License, Version 1.2 |
||||||
|
|
||||||
|
Copyright (C) 2003 The Trustees of Indiana University. |
||||||
|
All rights reserved. |
||||||
|
|
||||||
|
Redistribution and use in source and binary forms, with or without |
||||||
|
modification, are permitted provided that the following conditions are |
||||||
|
met: |
||||||
|
|
||||||
|
1) All redistributions of source code must retain the above |
||||||
|
copyright notice, the list of authors in the original source |
||||||
|
code, this list of conditions and the disclaimer listed in this |
||||||
|
license; |
||||||
|
|
||||||
|
2) All redistributions in binary form must reproduce the above |
||||||
|
copyright notice, this list of conditions and the disclaimer |
||||||
|
listed in this license in the documentation and/or other |
||||||
|
materials provided with the distribution; |
||||||
|
|
||||||
|
3) Any documentation included with all redistributions must include |
||||||
|
the following acknowledgement: |
||||||
|
|
||||||
|
"This product includes software developed by the Indiana |
||||||
|
University Extreme! Lab. For further information please visit |
||||||
|
http://www.extreme.indiana.edu/" |
||||||
|
|
||||||
|
Alternatively, this acknowledgment may appear in the software |
||||||
|
itself, and wherever such third-party acknowledgments normally |
||||||
|
appear. |
||||||
|
|
||||||
|
4) The name "Indiana University" or "Indiana University |
||||||
|
Extreme! Lab" shall not be used to endorse or promote |
||||||
|
products derived from this software without prior written |
||||||
|
permission from Indiana University. For written permission, |
||||||
|
please contact http://www.extreme.indiana.edu/. |
||||||
|
|
||||||
|
5) Products derived from this software may not use "Indiana |
||||||
|
University" name nor may "Indiana University" appear in their name, |
||||||
|
without prior written permission of the Indiana University. |
||||||
|
|
||||||
|
Indiana University provides no reassurances that the source code |
||||||
|
provided does not infringe the patent or any other intellectual |
||||||
|
property rights of any other entity. Indiana University disclaims any |
||||||
|
liability to any recipient for claims brought by any other entity |
||||||
|
based on infringement of intellectual property rights or otherwise. |
||||||
|
|
||||||
|
LICENSEE UNDERSTANDS THAT SOFTWARE IS PROVIDED "AS IS" FOR WHICH |
||||||
|
NO WARRANTIES AS TO CAPABILITIES OR ACCURACY ARE MADE. INDIANA |
||||||
|
UNIVERSITY GIVES NO WARRANTIES AND MAKES NO REPRESENTATION THAT |
||||||
|
SOFTWARE IS FREE OF INFRINGEMENT OF THIRD PARTY PATENT, COPYRIGHT, OR |
||||||
|
OTHER PROPRIETARY RIGHTS. INDIANA UNIVERSITY MAKES NO WARRANTIES THAT |
||||||
|
SOFTWARE IS FREE FROM "BUGS", "VIRUSES", "TROJAN HORSES", "TRAP |
||||||
|
DOORS", "WORMS", OR OTHER HARMFUL CODE. LICENSEE ASSUMES THE ENTIRE |
||||||
|
RISK AS TO THE PERFORMANCE OF SOFTWARE AND/OR ASSOCIATED MATERIALS, |
||||||
|
AND TO THE PERFORMANCE AND VALIDITY OF INFORMATION GENERATED USING |
||||||
|
SOFTWARE. |
||||||
|
|
||||||
@ -0,0 +1,203 @@ |
|||||||
|
|
||||||
|
Apache License |
||||||
|
Version 2.0, January 2004 |
||||||
|
http://www.apache.org/licenses/ |
||||||
|
|
||||||
|
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION |
||||||
|
|
||||||
|
1. Definitions. |
||||||
|
|
||||||
|
"License" shall mean the terms and conditions for use, reproduction, |
||||||
|
and distribution as defined by Sections 1 through 9 of this document. |
||||||
|
|
||||||
|
"Licensor" shall mean the copyright owner or entity authorized by |
||||||
|
the copyright owner that is granting the License. |
||||||
|
|
||||||
|
"Legal Entity" shall mean the union of the acting entity and all |
||||||
|
other entities that control, are controlled by, or are under common |
||||||
|
control with that entity. For the purposes of this definition, |
||||||
|
"control" means (i) the power, direct or indirect, to cause the |
||||||
|
direction or management of such entity, whether by contract or |
||||||
|
otherwise, or (ii) ownership of fifty percent (50%) or more of the |
||||||
|
outstanding shares, or (iii) beneficial ownership of such entity. |
||||||
|
|
||||||
|
"You" (or "Your") shall mean an individual or Legal Entity |
||||||
|
exercising permissions granted by this License. |
||||||
|
|
||||||
|
"Source" form shall mean the preferred form for making modifications, |
||||||
|
including but not limited to software source code, documentation |
||||||
|
source, and configuration files. |
||||||
|
|
||||||
|
"Object" form shall mean any form resulting from mechanical |
||||||
|
transformation or translation of a Source form, including but |
||||||
|
not limited to compiled object code, generated documentation, |
||||||
|
and conversions to other media types. |
||||||
|
|
||||||
|
"Work" shall mean the work of authorship, whether in Source or |
||||||
|
Object form, made available under the License, as indicated by a |
||||||
|
copyright notice that is included in or attached to the work |
||||||
|
(an example is provided in the Appendix below). |
||||||
|
|
||||||
|
"Derivative Works" shall mean any work, whether in Source or Object |
||||||
|
form, that is based on (or derived from) the Work and for which the |
||||||
|
editorial revisions, annotations, elaborations, or other modifications |
||||||
|
represent, as a whole, an original work of authorship. For the purposes |
||||||
|
of this License, Derivative Works shall not include works that remain |
||||||
|
separable from, or merely link (or bind by name) to the interfaces of, |
||||||
|
the Work and Derivative Works thereof. |
||||||
|
|
||||||
|
"Contribution" shall mean any work of authorship, including |
||||||
|
the original version of the Work and any modifications or additions |
||||||
|
to that Work or Derivative Works thereof, that is intentionally |
||||||
|
submitted to Licensor for inclusion in the Work by the copyright owner |
||||||
|
or by an individual or Legal Entity authorized to submit on behalf of |
||||||
|
the copyright owner. For the purposes of this definition, "submitted" |
||||||
|
means any form of electronic, verbal, or written communication sent |
||||||
|
to the Licensor or its representatives, including but not limited to |
||||||
|
communication on electronic mailing lists, source code control systems, |
||||||
|
and issue tracking systems that are managed by, or on behalf of, the |
||||||
|
Licensor for the purpose of discussing and improving the Work, but |
||||||
|
excluding communication that is conspicuously marked or otherwise |
||||||
|
designated in writing by the copyright owner as "Not a Contribution." |
||||||
|
|
||||||
|
"Contributor" shall mean Licensor and any individual or Legal Entity |
||||||
|
on behalf of whom a Contribution has been received by Licensor and |
||||||
|
subsequently incorporated within the Work. |
||||||
|
|
||||||
|
2. Grant of Copyright License. Subject to the terms and conditions of |
||||||
|
this License, each Contributor hereby grants to You a perpetual, |
||||||
|
worldwide, non-exclusive, no-charge, royalty-free, irrevocable |
||||||
|
copyright license to reproduce, prepare Derivative Works of, |
||||||
|
publicly display, publicly perform, sublicense, and distribute the |
||||||
|
Work and such Derivative Works in Source or Object form. |
||||||
|
|
||||||
|
3. Grant of Patent License. Subject to the terms and conditions of |
||||||
|
this License, each Contributor hereby grants to You a perpetual, |
||||||
|
worldwide, non-exclusive, no-charge, royalty-free, irrevocable |
||||||
|
(except as stated in this section) patent license to make, have made, |
||||||
|
use, offer to sell, sell, import, and otherwise transfer the Work, |
||||||
|
where such license applies only to those patent claims licensable |
||||||
|
by such Contributor that are necessarily infringed by their |
||||||
|
Contribution(s) alone or by combination of their Contribution(s) |
||||||
|
with the Work to which such Contribution(s) was submitted. If You |
||||||
|
institute patent litigation against any entity (including a |
||||||
|
cross-claim or counterclaim in a lawsuit) alleging that the Work |
||||||
|
or a Contribution incorporated within the Work constitutes direct |
||||||
|
or contributory patent infringement, then any patent licenses |
||||||
|
granted to You under this License for that Work shall terminate |
||||||
|
as of the date such litigation is filed. |
||||||
|
|
||||||
|
4. Redistribution. You may reproduce and distribute copies of the |
||||||
|
Work or Derivative Works thereof in any medium, with or without |
||||||
|
modifications, and in Source or Object form, provided that You |
||||||
|
meet the following conditions: |
||||||
|
|
||||||
|
(a) You must give any other recipients of the Work or |
||||||
|
Derivative Works a copy of this License; and |
||||||
|
|
||||||
|
(b) You must cause any modified files to carry prominent notices |
||||||
|
stating that You changed the files; and |
||||||
|
|
||||||
|
(c) You must retain, in the Source form of any Derivative Works |
||||||
|
that You distribute, all copyright, patent, trademark, and |
||||||
|
attribution notices from the Source form of the Work, |
||||||
|
excluding those notices that do not pertain to any part of |
||||||
|
the Derivative Works; and |
||||||
|
|
||||||
|
(d) If the Work includes a "NOTICE" text file as part of its |
||||||
|
distribution, then any Derivative Works that You distribute must |
||||||
|
include a readable copy of the attribution notices contained |
||||||
|
within such NOTICE file, excluding those notices that do not |
||||||
|
pertain to any part of the Derivative Works, in at least one |
||||||
|
of the following places: within a NOTICE text file distributed |
||||||
|
as part of the Derivative Works; within the Source form or |
||||||
|
documentation, if provided along with the Derivative Works; or, |
||||||
|
within a display generated by the Derivative Works, if and |
||||||
|
wherever such third-party notices normally appear. The contents |
||||||
|
of the NOTICE file are for informational purposes only and |
||||||
|
do not modify the License. You may add Your own attribution |
||||||
|
notices within Derivative Works that You distribute, alongside |
||||||
|
or as an addendum to the NOTICE text from the Work, provided |
||||||
|
that such additional attribution notices cannot be construed |
||||||
|
as modifying the License. |
||||||
|
|
||||||
|
You may add Your own copyright statement to Your modifications and |
||||||
|
may provide additional or different license terms and conditions |
||||||
|
for use, reproduction, or distribution of Your modifications, or |
||||||
|
for any such Derivative Works as a whole, provided Your use, |
||||||
|
reproduction, and distribution of the Work otherwise complies with |
||||||
|
the conditions stated in this License. |
||||||
|
|
||||||
|
5. Submission of Contributions. Unless You explicitly state otherwise, |
||||||
|
any Contribution intentionally submitted for inclusion in the Work |
||||||
|
by You to the Licensor shall be under the terms and conditions of |
||||||
|
this License, without any additional terms or conditions. |
||||||
|
Notwithstanding the above, nothing herein shall supersede or modify |
||||||
|
the terms of any separate license agreement you may have executed |
||||||
|
with Licensor regarding such Contributions. |
||||||
|
|
||||||
|
6. Trademarks. This License does not grant permission to use the trade |
||||||
|
names, trademarks, service marks, or product names of the Licensor, |
||||||
|
except as required for reasonable and customary use in describing the |
||||||
|
origin of the Work and reproducing the content of the NOTICE file. |
||||||
|
|
||||||
|
7. Disclaimer of Warranty. Unless required by applicable law or |
||||||
|
agreed to in writing, Licensor provides the Work (and each |
||||||
|
Contributor provides its Contributions) on an "AS IS" BASIS, |
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or |
||||||
|
implied, including, without limitation, any warranties or conditions |
||||||
|
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A |
||||||
|
PARTICULAR PURPOSE. You are solely responsible for determining the |
||||||
|
appropriateness of using or redistributing the Work and assume any |
||||||
|
risks associated with Your exercise of permissions under this License. |
||||||
|
|
||||||
|
8. Limitation of Liability. In no event and under no legal theory, |
||||||
|
whether in tort (including negligence), contract, or otherwise, |
||||||
|
unless required by applicable law (such as deliberate and grossly |
||||||
|
negligent acts) or agreed to in writing, shall any Contributor be |
||||||
|
liable to You for damages, including any direct, indirect, special, |
||||||
|
incidental, or consequential damages of any character arising as a |
||||||
|
result of this License or out of the use or inability to use the |
||||||
|
Work (including but not limited to damages for loss of goodwill, |
||||||
|
work stoppage, computer failure or malfunction, or any and all |
||||||
|
other commercial damages or losses), even if such Contributor |
||||||
|
has been advised of the possibility of such damages. |
||||||
|
|
||||||
|
9. Accepting Warranty or Additional Liability. While redistributing |
||||||
|
the Work or Derivative Works thereof, You may choose to offer, |
||||||
|
and charge a fee for, acceptance of support, warranty, indemnity, |
||||||
|
or other liability obligations and/or rights consistent with this |
||||||
|
License. However, in accepting such obligations, You may act only |
||||||
|
on Your own behalf and on Your sole responsibility, not on behalf |
||||||
|
of any other Contributor, and only if You agree to indemnify, |
||||||
|
defend, and hold each Contributor harmless for any liability |
||||||
|
incurred by, or claims asserted against, such Contributor by reason |
||||||
|
of your accepting any such warranty or additional liability. |
||||||
|
|
||||||
|
END OF TERMS AND CONDITIONS |
||||||
|
|
||||||
|
APPENDIX: How to apply the Apache License to your work. |
||||||
|
|
||||||
|
To apply the Apache License to your work, attach the following |
||||||
|
boilerplate notice, with the fields enclosed by brackets "[]" |
||||||
|
replaced with your own identifying information. (Don't include |
||||||
|
the brackets!) The text should be enclosed in the appropriate |
||||||
|
comment syntax for the file format. We also recommend that a |
||||||
|
file or class name and description of purpose be included on the |
||||||
|
same "printed page" as the copyright notice for easier |
||||||
|
identification within third-party archives. |
||||||
|
|
||||||
|
Copyright [yyyy] [name of copyright owner] |
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License"); |
||||||
|
you may not use this file except in compliance with the License. |
||||||
|
You may obtain a copy of the License at |
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software |
||||||
|
distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
See the License for the specific language governing permissions and |
||||||
|
limitations under the License. |
||||||
|
|
||||||
@ -0,0 +1,203 @@ |
|||||||
|
|
||||||
|
Apache License |
||||||
|
Version 2.0, January 2004 |
||||||
|
http://www.apache.org/licenses/ |
||||||
|
|
||||||
|
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION |
||||||
|
|
||||||
|
1. Definitions. |
||||||
|
|
||||||
|
"License" shall mean the terms and conditions for use, reproduction, |
||||||
|
and distribution as defined by Sections 1 through 9 of this document. |
||||||
|
|
||||||
|
"Licensor" shall mean the copyright owner or entity authorized by |
||||||
|
the copyright owner that is granting the License. |
||||||
|
|
||||||
|
"Legal Entity" shall mean the union of the acting entity and all |
||||||
|
other entities that control, are controlled by, or are under common |
||||||
|
control with that entity. For the purposes of this definition, |
||||||
|
"control" means (i) the power, direct or indirect, to cause the |
||||||
|
direction or management of such entity, whether by contract or |
||||||
|
otherwise, or (ii) ownership of fifty percent (50%) or more of the |
||||||
|
outstanding shares, or (iii) beneficial ownership of such entity. |
||||||
|
|
||||||
|
"You" (or "Your") shall mean an individual or Legal Entity |
||||||
|
exercising permissions granted by this License. |
||||||
|
|
||||||
|
"Source" form shall mean the preferred form for making modifications, |
||||||
|
including but not limited to software source code, documentation |
||||||
|
source, and configuration files. |
||||||
|
|
||||||
|
"Object" form shall mean any form resulting from mechanical |
||||||
|
transformation or translation of a Source form, including but |
||||||
|
not limited to compiled object code, generated documentation, |
||||||
|
and conversions to other media types. |
||||||
|
|
||||||
|
"Work" shall mean the work of authorship, whether in Source or |
||||||
|
Object form, made available under the License, as indicated by a |
||||||
|
copyright notice that is included in or attached to the work |
||||||
|
(an example is provided in the Appendix below). |
||||||
|
|
||||||
|
"Derivative Works" shall mean any work, whether in Source or Object |
||||||
|
form, that is based on (or derived from) the Work and for which the |
||||||
|
editorial revisions, annotations, elaborations, or other modifications |
||||||
|
represent, as a whole, an original work of authorship. For the purposes |
||||||
|
of this License, Derivative Works shall not include works that remain |
||||||
|
separable from, or merely link (or bind by name) to the interfaces of, |
||||||
|
the Work and Derivative Works thereof. |
||||||
|
|
||||||
|
"Contribution" shall mean any work of authorship, including |
||||||
|
the original version of the Work and any modifications or additions |
||||||
|
to that Work or Derivative Works thereof, that is intentionally |
||||||
|
submitted to Licensor for inclusion in the Work by the copyright owner |
||||||
|
or by an individual or Legal Entity authorized to submit on behalf of |
||||||
|
the copyright owner. For the purposes of this definition, "submitted" |
||||||
|
means any form of electronic, verbal, or written communication sent |
||||||
|
to the Licensor or its representatives, including but not limited to |
||||||
|
communication on electronic mailing lists, source code control systems, |
||||||
|
and issue tracking systems that are managed by, or on behalf of, the |
||||||
|
Licensor for the purpose of discussing and improving the Work, but |
||||||
|
excluding communication that is conspicuously marked or otherwise |
||||||
|
designated in writing by the copyright owner as "Not a Contribution." |
||||||
|
|
||||||
|
"Contributor" shall mean Licensor and any individual or Legal Entity |
||||||
|
on behalf of whom a Contribution has been received by Licensor and |
||||||
|
subsequently incorporated within the Work. |
||||||
|
|
||||||
|
2. Grant of Copyright License. Subject to the terms and conditions of |
||||||
|
this License, each Contributor hereby grants to You a perpetual, |
||||||
|
worldwide, non-exclusive, no-charge, royalty-free, irrevocable |
||||||
|
copyright license to reproduce, prepare Derivative Works of, |
||||||
|
publicly display, publicly perform, sublicense, and distribute the |
||||||
|
Work and such Derivative Works in Source or Object form. |
||||||
|
|
||||||
|
3. Grant of Patent License. Subject to the terms and conditions of |
||||||
|
this License, each Contributor hereby grants to You a perpetual, |
||||||
|
worldwide, non-exclusive, no-charge, royalty-free, irrevocable |
||||||
|
(except as stated in this section) patent license to make, have made, |
||||||
|
use, offer to sell, sell, import, and otherwise transfer the Work, |
||||||
|
where such license applies only to those patent claims licensable |
||||||
|
by such Contributor that are necessarily infringed by their |
||||||
|
Contribution(s) alone or by combination of their Contribution(s) |
||||||
|
with the Work to which such Contribution(s) was submitted. If You |
||||||
|
institute patent litigation against any entity (including a |
||||||
|
cross-claim or counterclaim in a lawsuit) alleging that the Work |
||||||
|
or a Contribution incorporated within the Work constitutes direct |
||||||
|
or contributory patent infringement, then any patent licenses |
||||||
|
granted to You under this License for that Work shall terminate |
||||||
|
as of the date such litigation is filed. |
||||||
|
|
||||||
|
4. Redistribution. You may reproduce and distribute copies of the |
||||||
|
Work or Derivative Works thereof in any medium, with or without |
||||||
|
modifications, and in Source or Object form, provided that You |
||||||
|
meet the following conditions: |
||||||
|
|
||||||
|
(a) You must give any other recipients of the Work or |
||||||
|
Derivative Works a copy of this License; and |
||||||
|
|
||||||
|
(b) You must cause any modified files to carry prominent notices |
||||||
|
stating that You changed the files; and |
||||||
|
|
||||||
|
(c) You must retain, in the Source form of any Derivative Works |
||||||
|
that You distribute, all copyright, patent, trademark, and |
||||||
|
attribution notices from the Source form of the Work, |
||||||
|
excluding those notices that do not pertain to any part of |
||||||
|
the Derivative Works; and |
||||||
|
|
||||||
|
(d) If the Work includes a "NOTICE" text file as part of its |
||||||
|
distribution, then any Derivative Works that You distribute must |
||||||
|
include a readable copy of the attribution notices contained |
||||||
|
within such NOTICE file, excluding those notices that do not |
||||||
|
pertain to any part of the Derivative Works, in at least one |
||||||
|
of the following places: within a NOTICE text file distributed |
||||||
|
as part of the Derivative Works; within the Source form or |
||||||
|
documentation, if provided along with the Derivative Works; or, |
||||||
|
within a display generated by the Derivative Works, if and |
||||||
|
wherever such third-party notices normally appear. The contents |
||||||
|
of the NOTICE file are for informational purposes only and |
||||||
|
do not modify the License. You may add Your own attribution |
||||||
|
notices within Derivative Works that You distribute, alongside |
||||||
|
or as an addendum to the NOTICE text from the Work, provided |
||||||
|
that such additional attribution notices cannot be construed |
||||||
|
as modifying the License. |
||||||
|
|
||||||
|
You may add Your own copyright statement to Your modifications and |
||||||
|
may provide additional or different license terms and conditions |
||||||
|
for use, reproduction, or distribution of Your modifications, or |
||||||
|
for any such Derivative Works as a whole, provided Your use, |
||||||
|
reproduction, and distribution of the Work otherwise complies with |
||||||
|
the conditions stated in this License. |
||||||
|
|
||||||
|
5. Submission of Contributions. Unless You explicitly state otherwise, |
||||||
|
any Contribution intentionally submitted for inclusion in the Work |
||||||
|
by You to the Licensor shall be under the terms and conditions of |
||||||
|
this License, without any additional terms or conditions. |
||||||
|
Notwithstanding the above, nothing herein shall supersede or modify |
||||||
|
the terms of any separate license agreement you may have executed |
||||||
|
with Licensor regarding such Contributions. |
||||||
|
|
||||||
|
6. Trademarks. This License does not grant permission to use the trade |
||||||
|
names, trademarks, service marks, or product names of the Licensor, |
||||||
|
except as required for reasonable and customary use in describing the |
||||||
|
origin of the Work and reproducing the content of the NOTICE file. |
||||||
|
|
||||||
|
7. Disclaimer of Warranty. Unless required by applicable law or |
||||||
|
agreed to in writing, Licensor provides the Work (and each |
||||||
|
Contributor provides its Contributions) on an "AS IS" BASIS, |
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or |
||||||
|
implied, including, without limitation, any warranties or conditions |
||||||
|
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A |
||||||
|
PARTICULAR PURPOSE. You are solely responsible for determining the |
||||||
|
appropriateness of using or redistributing the Work and assume any |
||||||
|
risks associated with Your exercise of permissions under this License. |
||||||
|
|
||||||
|
8. Limitation of Liability. In no event and under no legal theory, |
||||||
|
whether in tort (including negligence), contract, or otherwise, |
||||||
|
unless required by applicable law (such as deliberate and grossly |
||||||
|
negligent acts) or agreed to in writing, shall any Contributor be |
||||||
|
liable to You for damages, including any direct, indirect, special, |
||||||
|
incidental, or consequential damages of any character arising as a |
||||||
|
result of this License or out of the use or inability to use the |
||||||
|
Work (including but not limited to damages for loss of goodwill, |
||||||
|
work stoppage, computer failure or malfunction, or any and all |
||||||
|
other commercial damages or losses), even if such Contributor |
||||||
|
has been advised of the possibility of such damages. |
||||||
|
|
||||||
|
9. Accepting Warranty or Additional Liability. While redistributing |
||||||
|
the Work or Derivative Works thereof, You may choose to offer, |
||||||
|
and charge a fee for, acceptance of support, warranty, indemnity, |
||||||
|
or other liability obligations and/or rights consistent with this |
||||||
|
License. However, in accepting such obligations, You may act only |
||||||
|
on Your own behalf and on Your sole responsibility, not on behalf |
||||||
|
of any other Contributor, and only if You agree to indemnify, |
||||||
|
defend, and hold each Contributor harmless for any liability |
||||||
|
incurred by, or claims asserted against, such Contributor by reason |
||||||
|
of your accepting any such warranty or additional liability. |
||||||
|
|
||||||
|
END OF TERMS AND CONDITIONS |
||||||
|
|
||||||
|
APPENDIX: How to apply the Apache License to your work. |
||||||
|
|
||||||
|
To apply the Apache License to your work, attach the following |
||||||
|
boilerplate notice, with the fields enclosed by brackets "[]" |
||||||
|
replaced with your own identifying information. (Don't include |
||||||
|
the brackets!) The text should be enclosed in the appropriate |
||||||
|
comment syntax for the file format. We also recommend that a |
||||||
|
file or class name and description of purpose be included on the |
||||||
|
same "printed page" as the copyright notice for easier |
||||||
|
identification within third-party archives. |
||||||
|
|
||||||
|
Copyright [yyyy] [name of copyright owner] |
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License"); |
||||||
|
you may not use this file except in compliance with the License. |
||||||
|
You may obtain a copy of the License at |
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software |
||||||
|
distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
See the License for the specific language governing permissions and |
||||||
|
limitations under the License. |
||||||
|
|
||||||
@ -0,0 +1,15 @@ |
|||||||
|
XMLPULL API IS FREE |
||||||
|
------------------- |
||||||
|
|
||||||
|
All of the XMLPULL API source code, compiled code, and documentation |
||||||
|
contained in this distribution *except* for tests (see separate LICENSE_TESTS.txt) |
||||||
|
are in the Public Domain. |
||||||
|
|
||||||
|
XMLPULL API comes with NO WARRANTY or guarantee of fitness for any purpose. |
||||||
|
|
||||||
|
Initial authors: |
||||||
|
|
||||||
|
Stefan Haustein |
||||||
|
Aleksander Slominski |
||||||
|
|
||||||
|
2001-12-12 |
||||||
@ -0,0 +1,29 @@ |
|||||||
|
Copyright (c) 2003-2006, Joe Walnes |
||||||
|
Copyright (c) 2006-2015 XStream Committers |
||||||
|
All rights reserved. |
||||||
|
|
||||||
|
Redistribution and use in source and binary forms, with or without |
||||||
|
modification, are permitted provided that the following conditions are met: |
||||||
|
|
||||||
|
1. Redistributions of source code must retain the above copyright notice, this list of |
||||||
|
conditions and the following disclaimer. |
||||||
|
|
||||||
|
2. Redistributions in binary form must reproduce the above copyright notice, this list of |
||||||
|
conditions and the following disclaimer in the documentation and/or other materials provided |
||||||
|
with the distribution. |
||||||
|
|
||||||
|
3. Neither the name of XStream nor the names of its contributors may be used to endorse |
||||||
|
or promote products derived from this software without specific prior written |
||||||
|
permission. |
||||||
|
|
||||||
|
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY |
||||||
|
EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
||||||
|
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT |
||||||
|
SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, |
||||||
|
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED |
||||||
|
TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR |
||||||
|
BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
||||||
|
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY |
||||||
|
WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH |
||||||
|
DAMAGE. |
||||||
|
|
||||||
@ -0,0 +1,70 @@ |
|||||||
|
<?xml version="1.0" encoding="UTF-8"?> |
||||||
|
<!-- |
||||||
|
~ Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
~ contributor license agreements. See the NOTICE file distributed with |
||||||
|
~ this work for additional information regarding copyright ownership. |
||||||
|
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
~ (the "License"); you may not use this file except in compliance with |
||||||
|
~ the License. You may obtain a copy of the License at |
||||||
|
~ |
||||||
|
~ http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
~ |
||||||
|
~ Unless required by applicable law or agreed to in writing, software |
||||||
|
~ distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
~ See the License for the specific language governing permissions and |
||||||
|
~ limitations under the License. |
||||||
|
--> |
||||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" |
||||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> |
||||||
|
<modelVersion>4.0.0</modelVersion> |
||||||
|
<parent> |
||||||
|
<groupId>org.apache.dolphinscheduler</groupId> |
||||||
|
<artifactId>dolphinscheduler-storage-plugin</artifactId> |
||||||
|
<version>dev-SNAPSHOT</version> |
||||||
|
</parent> |
||||||
|
|
||||||
|
<artifactId>dolphinscheduler-storage-cos</artifactId> |
||||||
|
|
||||||
|
<properties> |
||||||
|
<plugin.name>storage.cos</plugin.name> |
||||||
|
</properties> |
||||||
|
|
||||||
|
<dependencies> |
||||||
|
<dependency> |
||||||
|
<groupId>org.apache.dolphinscheduler</groupId> |
||||||
|
<artifactId>dolphinscheduler-common</artifactId> |
||||||
|
<version>${project.version}</version> |
||||||
|
<scope>provided</scope> |
||||||
|
</dependency> |
||||||
|
|
||||||
|
<dependency> |
||||||
|
<groupId>org.apache.dolphinscheduler</groupId> |
||||||
|
<artifactId>dolphinscheduler-storage-api</artifactId> |
||||||
|
<version>${project.version}</version> |
||||||
|
<scope>provided</scope> |
||||||
|
</dependency> |
||||||
|
|
||||||
|
<dependency> |
||||||
|
<groupId>com.qcloud</groupId> |
||||||
|
<artifactId>cos_api</artifactId> |
||||||
|
</dependency> |
||||||
|
</dependencies> |
||||||
|
|
||||||
|
<build> |
||||||
|
<plugins> |
||||||
|
<plugin> |
||||||
|
<groupId>org.apache.maven.plugins</groupId> |
||||||
|
<artifactId>maven-shade-plugin</artifactId> |
||||||
|
<executions> |
||||||
|
<execution> |
||||||
|
<goals> |
||||||
|
<goal>shade</goal> |
||||||
|
</goals> |
||||||
|
<phase>package</phase> |
||||||
|
</execution> |
||||||
|
</executions> |
||||||
|
</plugin> |
||||||
|
</plugins> |
||||||
|
</build> |
||||||
|
</project> |
||||||
@ -0,0 +1,30 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.plugin.storage.cos; |
||||||
|
|
||||||
|
public class CosStorageConstants { |
||||||
|
|
||||||
|
// Tencent Cloud Storage (COS) configuration
|
||||||
|
public static final String TENCENT_CLOUD_COS_PROPERTY_PREFIX = "resource.tencent.cloud"; |
||||||
|
public static final String TENCENT_CLOUD_COS_BUCKET_NAME = "resource.tencent.cloud.cos.bucket.name"; |
||||||
|
public static final String TENCENT_CLOUD_COS_REGION = "resource.tencent.cloud.cos.region"; |
||||||
|
public static final String TENCENT_CLOUD_ACCESS_KEY_ID = "resource.tencent.cloud.access.key.id"; |
||||||
|
public static final String TENCENT_CLOUD_ACCESS_KEY_SECRET = "resource.tencent.cloud.access.key.secret"; |
||||||
|
|
||||||
|
public static final String DEFAULT_COS_RESOURCE_UPLOAD_PATH = "/dolphinscheduler"; |
||||||
|
} |
||||||
@ -0,0 +1,338 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.plugin.storage.cos; |
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.common.utils.FileUtils; |
||||||
|
import org.apache.dolphinscheduler.plugin.storage.api.AbstractStorageOperator; |
||||||
|
import org.apache.dolphinscheduler.plugin.storage.api.ResourceMetadata; |
||||||
|
import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity; |
||||||
|
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; |
||||||
|
import org.apache.dolphinscheduler.plugin.storage.api.constants.StorageConstants; |
||||||
|
|
||||||
|
import org.apache.commons.lang3.StringUtils; |
||||||
|
|
||||||
|
import java.io.BufferedReader; |
||||||
|
import java.io.ByteArrayInputStream; |
||||||
|
import java.io.Closeable; |
||||||
|
import java.io.File; |
||||||
|
import java.io.IOException; |
||||||
|
import java.io.InputStream; |
||||||
|
import java.io.InputStreamReader; |
||||||
|
import java.nio.file.FileAlreadyExistsException; |
||||||
|
import java.nio.file.Files; |
||||||
|
import java.nio.file.Path; |
||||||
|
import java.nio.file.Paths; |
||||||
|
import java.util.ArrayList; |
||||||
|
import java.util.HashSet; |
||||||
|
import java.util.LinkedList; |
||||||
|
import java.util.List; |
||||||
|
import java.util.Set; |
||||||
|
import java.util.concurrent.ExecutorService; |
||||||
|
import java.util.concurrent.Executors; |
||||||
|
import java.util.stream.Collectors; |
||||||
|
|
||||||
|
import lombok.SneakyThrows; |
||||||
|
import lombok.extern.slf4j.Slf4j; |
||||||
|
|
||||||
|
import com.qcloud.cos.COSClient; |
||||||
|
import com.qcloud.cos.ClientConfig; |
||||||
|
import com.qcloud.cos.auth.BasicCOSCredentials; |
||||||
|
import com.qcloud.cos.auth.COSCredentials; |
||||||
|
import com.qcloud.cos.exception.CosServiceException; |
||||||
|
import com.qcloud.cos.http.HttpProtocol; |
||||||
|
import com.qcloud.cos.model.COSObject; |
||||||
|
import com.qcloud.cos.model.COSObjectSummary; |
||||||
|
import com.qcloud.cos.model.CopyObjectRequest; |
||||||
|
import com.qcloud.cos.model.CopyResult; |
||||||
|
import com.qcloud.cos.model.GetObjectRequest; |
||||||
|
import com.qcloud.cos.model.ListObjectsRequest; |
||||||
|
import com.qcloud.cos.model.ObjectListing; |
||||||
|
import com.qcloud.cos.model.ObjectMetadata; |
||||||
|
import com.qcloud.cos.model.PutObjectRequest; |
||||||
|
import com.qcloud.cos.model.UploadResult; |
||||||
|
import com.qcloud.cos.region.Region; |
||||||
|
import com.qcloud.cos.transfer.Copy; |
||||||
|
import com.qcloud.cos.transfer.Download; |
||||||
|
import com.qcloud.cos.transfer.TransferManager; |
||||||
|
import com.qcloud.cos.transfer.TransferManagerConfiguration; |
||||||
|
import com.qcloud.cos.transfer.Upload; |
||||||
|
|
||||||
|
@Slf4j |
||||||
|
public class CosStorageOperator extends AbstractStorageOperator implements Closeable, StorageOperator { |
||||||
|
|
||||||
|
private static final int TRANSFER_THREAD_POOL_SIZE = 16; |
||||||
|
|
||||||
|
private static final long MULTIPART_UPLOAD_BYTES_THRESHOLD = 5 * 1024 * 1024L; |
||||||
|
private static final long MIN_UPLOAD_PART_BYTES = 1024 * 1024L; |
||||||
|
|
||||||
|
private final String bucketName; |
||||||
|
|
||||||
|
private final COSClient cosClient; |
||||||
|
|
||||||
|
private final TransferManager cosTransferManager; |
||||||
|
|
||||||
|
public CosStorageOperator(CosStorageProperties cosStorageProperties) { |
||||||
|
super(cosStorageProperties.getResourceUploadPath()); |
||||||
|
String secretId = cosStorageProperties.getAccessKeyId(); |
||||||
|
String secretKey = cosStorageProperties.getAccessKeySecret(); |
||||||
|
COSCredentials cosCredentials = new BasicCOSCredentials(secretId, secretKey); |
||||||
|
String regionName = cosStorageProperties.getRegion(); |
||||||
|
ClientConfig clientConfig = new ClientConfig(new Region(regionName)); |
||||||
|
clientConfig.setHttpProtocol(HttpProtocol.https); |
||||||
|
this.cosClient = new COSClient(cosCredentials, clientConfig); |
||||||
|
this.bucketName = cosStorageProperties.getBucketName(); |
||||||
|
ensureBucketSuccessfullyCreated(bucketName); |
||||||
|
this.cosTransferManager = getCosTransferManager(); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public void close() throws IOException { |
||||||
|
this.cosTransferManager.shutdownNow(true); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public String getStorageBaseDirectory() { |
||||||
|
// All directory should end with File.separator
|
||||||
|
if (resourceBaseAbsolutePath.startsWith(File.separator)) { |
||||||
|
String warnMessage = |
||||||
|
String.format("%s -> %s should not start with %s in tencent cos", |
||||||
|
StorageConstants.RESOURCE_UPLOAD_PATH, |
||||||
|
resourceBaseAbsolutePath, File.separator); |
||||||
|
log.warn(warnMessage); |
||||||
|
return resourceBaseAbsolutePath.substring(1); |
||||||
|
} |
||||||
|
return resourceBaseAbsolutePath; |
||||||
|
} |
||||||
|
|
||||||
|
@SneakyThrows |
||||||
|
@Override |
||||||
|
public void createStorageDir(String directoryAbsolutePath) { |
||||||
|
String cosKey = transformAbsolutePathToCOSKey(directoryAbsolutePath); |
||||||
|
if (cosClient.doesObjectExist(bucketName, cosKey)) { |
||||||
|
throw new FileAlreadyExistsException("directory: " + cosKey + " already exists"); |
||||||
|
} |
||||||
|
ObjectMetadata metadata = new ObjectMetadata(); |
||||||
|
metadata.setContentLength(0L); |
||||||
|
InputStream emptyContent = new ByteArrayInputStream(new byte[0]); |
||||||
|
PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, cosKey, emptyContent, metadata); |
||||||
|
cosClient.putObject(putObjectRequest); |
||||||
|
} |
||||||
|
|
||||||
|
@SneakyThrows |
||||||
|
@Override |
||||||
|
public void download(String srcFilePath, String dstFilePath, boolean overwrite) { |
||||||
|
String cosKey = transformAbsolutePathToCOSKey(srcFilePath); |
||||||
|
Path dsTempFolder = Paths.get(FileUtils.DATA_BASEDIR).normalize().toAbsolutePath(); |
||||||
|
Path fileDownloadPathNormalized = dsTempFolder.resolve(dstFilePath).normalize().toAbsolutePath(); |
||||||
|
if (!fileDownloadPathNormalized.startsWith(dsTempFolder)) { |
||||||
|
// if the destination file path is NOT in DS temp folder (e.g., '/tmp/dolphinscheduler'),
|
||||||
|
// an IllegalArgumentException should be thrown.
|
||||||
|
throw new IllegalArgumentException("failed to download to " + fileDownloadPathNormalized); |
||||||
|
} |
||||||
|
File dstFile = fileDownloadPathNormalized.toFile(); |
||||||
|
if (dstFile.isDirectory()) { |
||||||
|
Files.delete(dstFile.toPath()); |
||||||
|
} else { |
||||||
|
FileUtils.createDirectoryWith755(dstFile.getParentFile().toPath()); |
||||||
|
} |
||||||
|
|
||||||
|
GetObjectRequest getObjectRequest = new GetObjectRequest(bucketName, cosKey); |
||||||
|
Download download = cosTransferManager.download(getObjectRequest, dstFile); |
||||||
|
download.waitForCompletion(); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public boolean exists(String fileName) { |
||||||
|
String cosKey = transformAbsolutePathToCOSKey(fileName); |
||||||
|
return cosClient.doesObjectExist(bucketName, cosKey); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public void delete(String filePath, boolean recursive) { |
||||||
|
String cosKey = transformAbsolutePathToCOSKey(filePath); |
||||||
|
cosClient.deleteObject(bucketName, cosKey); |
||||||
|
} |
||||||
|
|
||||||
|
@SneakyThrows |
||||||
|
@Override |
||||||
|
public void copy(String srcPath, String dstPath, boolean deleteSource, boolean overwrite) { |
||||||
|
String srcCosKey = transformAbsolutePathToCOSKey(srcPath); |
||||||
|
String destCosKey = transformAbsolutePathToCOSKey(dstPath); |
||||||
|
CopyObjectRequest copyObjectRequest = new CopyObjectRequest(bucketName, srcCosKey, bucketName, destCosKey); |
||||||
|
Copy copy = cosTransferManager.copy(copyObjectRequest, cosClient, null); |
||||||
|
|
||||||
|
CopyResult copyResult = copy.waitForCopyResult(); |
||||||
|
if (copyResult != null && deleteSource) { |
||||||
|
cosClient.deleteObject(bucketName, srcPath); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
@SneakyThrows |
||||||
|
@Override |
||||||
|
public void upload(String srcFile, String dstPath, boolean deleteSource, boolean overwrite) { |
||||||
|
dstPath = transformAbsolutePathToCOSKey(dstPath); |
||||||
|
|
||||||
|
if (cosClient.doesObjectExist(bucketName, dstPath)) { |
||||||
|
if (!overwrite) { |
||||||
|
throw new CosServiceException("file: " + dstPath + " already exists"); |
||||||
|
} else { |
||||||
|
cosClient.deleteObject(bucketName, dstPath); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, dstPath, new File(srcFile)); |
||||||
|
Upload upload = cosTransferManager.upload(putObjectRequest); |
||||||
|
UploadResult uploadResult = upload.waitForUploadResult(); |
||||||
|
if (uploadResult != null && deleteSource) { |
||||||
|
Files.delete(Paths.get(srcFile)); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
@SneakyThrows |
||||||
|
@Override |
||||||
|
public List<String> fetchFileContent(String filePath, int skipLineNums, int limit) { |
||||||
|
String cosKey = transformAbsolutePathToCOSKey(filePath); |
||||||
|
|
||||||
|
COSObject cosObject = cosClient.getObject(bucketName, cosKey); |
||||||
|
try ( |
||||||
|
InputStreamReader inputStreamReader = new InputStreamReader(cosObject.getObjectContent()); |
||||||
|
BufferedReader bufferedReader = new BufferedReader(inputStreamReader)) { |
||||||
|
return bufferedReader |
||||||
|
.lines() |
||||||
|
.skip(skipLineNums) |
||||||
|
.limit(limit) |
||||||
|
.collect(Collectors.toList()); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public List<StorageEntity> listStorageEntity(String resourceAbsolutePath) { |
||||||
|
resourceAbsolutePath = transformCOSKeyToAbsolutePath(resourceAbsolutePath); |
||||||
|
|
||||||
|
ListObjectsRequest request = new ListObjectsRequest(); |
||||||
|
request.setBucketName(bucketName); |
||||||
|
request.setPrefix(resourceAbsolutePath); |
||||||
|
request.setDelimiter(File.separator); |
||||||
|
|
||||||
|
ObjectListing result = cosClient.listObjects(request); |
||||||
|
|
||||||
|
return result.getObjectSummaries() |
||||||
|
.stream() |
||||||
|
.map((COSObjectSummary summary) -> { |
||||||
|
ObjectMetadata metadata = new ObjectMetadata(); |
||||||
|
metadata.setContentLength(summary.getSize()); |
||||||
|
metadata.setLastModified(summary.getLastModified()); |
||||||
|
COSObject object = new COSObject(); |
||||||
|
object.setObjectMetadata(metadata); |
||||||
|
object.setKey(summary.getKey()); |
||||||
|
return transformCOSObjectToStorageEntity(object); |
||||||
|
}) |
||||||
|
.collect(Collectors.toList()); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public List<StorageEntity> listFileStorageEntityRecursively(String resourceAbsolutePath) { |
||||||
|
resourceAbsolutePath = transformCOSKeyToAbsolutePath(resourceAbsolutePath); |
||||||
|
|
||||||
|
Set<String> visited = new HashSet<>(); |
||||||
|
List<StorageEntity> storageEntityList = new ArrayList<>(); |
||||||
|
LinkedList<String> foldersToFetch = new LinkedList<>(); |
||||||
|
foldersToFetch.addLast(resourceAbsolutePath); |
||||||
|
|
||||||
|
while (!foldersToFetch.isEmpty()) { |
||||||
|
String pathToExplore = foldersToFetch.pop(); |
||||||
|
visited.add(pathToExplore); |
||||||
|
List<StorageEntity> storageEntities = listStorageEntity(pathToExplore); |
||||||
|
for (StorageEntity entity : storageEntities) { |
||||||
|
if (entity.isDirectory()) { |
||||||
|
if (visited.contains(entity.getFullName())) { |
||||||
|
continue; |
||||||
|
} |
||||||
|
foldersToFetch.add(entity.getFullName()); |
||||||
|
} |
||||||
|
} |
||||||
|
storageEntityList.addAll(storageEntities); |
||||||
|
} |
||||||
|
|
||||||
|
return storageEntityList; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public StorageEntity getStorageEntity(String resourceAbsolutePath) { |
||||||
|
String cosKey = transformCOSKeyToAbsolutePath(resourceAbsolutePath); |
||||||
|
|
||||||
|
COSObject object = cosClient.getObject(bucketName, cosKey); |
||||||
|
return transformCOSObjectToStorageEntity(object); |
||||||
|
} |
||||||
|
|
||||||
|
public void ensureBucketSuccessfullyCreated(String bucketName) { |
||||||
|
if (StringUtils.isBlank(bucketName)) { |
||||||
|
throw new IllegalArgumentException(CosStorageConstants.TENCENT_CLOUD_COS_BUCKET_NAME + " is empty"); |
||||||
|
} |
||||||
|
|
||||||
|
if (!cosClient.doesBucketExist(bucketName)) { |
||||||
|
throw new IllegalArgumentException( |
||||||
|
"bucketName: " + bucketName + " is not exists, you need to create them by yourself"); |
||||||
|
} |
||||||
|
|
||||||
|
log.info("bucketName: {} has been found", bucketName); |
||||||
|
} |
||||||
|
|
||||||
|
protected StorageEntity transformCOSObjectToStorageEntity(COSObject object) { |
||||||
|
ObjectMetadata metadata = object.getObjectMetadata(); |
||||||
|
String fileAbsolutePath = transformCOSKeyToAbsolutePath(object.getKey()); |
||||||
|
ResourceMetadata resourceMetaData = getResourceMetaData(fileAbsolutePath); |
||||||
|
String fileExtension = com.google.common.io.Files.getFileExtension(resourceMetaData.getResourceAbsolutePath()); |
||||||
|
|
||||||
|
return StorageEntity.builder() |
||||||
|
.fileName(new File(fileAbsolutePath).getName()) |
||||||
|
.fullName(fileAbsolutePath) |
||||||
|
.pfullName(resourceMetaData.getResourceParentAbsolutePath()) |
||||||
|
.type(resourceMetaData.getResourceType()) |
||||||
|
.isDirectory(StringUtils.isEmpty(fileExtension)) |
||||||
|
.size(metadata.getContentLength()) |
||||||
|
.createTime(metadata.getLastModified()) |
||||||
|
.updateTime(metadata.getLastModified()) |
||||||
|
.build(); |
||||||
|
} |
||||||
|
|
||||||
|
private String transformAbsolutePathToCOSKey(String absolutePath) { |
||||||
|
ResourceMetadata resourceMetaData = getResourceMetaData(absolutePath); |
||||||
|
if (resourceMetaData.isDirectory()) { |
||||||
|
return FileUtils.concatFilePath(absolutePath, File.separator); |
||||||
|
} |
||||||
|
return absolutePath; |
||||||
|
} |
||||||
|
|
||||||
|
private String transformCOSKeyToAbsolutePath(String cosKey) { |
||||||
|
if (cosKey.endsWith(File.separator)) { |
||||||
|
return cosKey.substring(0, cosKey.length() - 1); |
||||||
|
} |
||||||
|
return cosKey; |
||||||
|
} |
||||||
|
|
||||||
|
private TransferManager getCosTransferManager() { |
||||||
|
ExecutorService threadPool = Executors.newFixedThreadPool(TRANSFER_THREAD_POOL_SIZE); |
||||||
|
TransferManager transferManager = new TransferManager(cosClient, threadPool); |
||||||
|
TransferManagerConfiguration transferManagerConfiguration = new TransferManagerConfiguration(); |
||||||
|
transferManagerConfiguration.setMultipartUploadThreshold(MULTIPART_UPLOAD_BYTES_THRESHOLD); |
||||||
|
transferManagerConfiguration.setMinimumUploadPartSize(MIN_UPLOAD_PART_BYTES); |
||||||
|
transferManager.setConfiguration(transferManagerConfiguration); |
||||||
|
return transferManager; |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,59 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.plugin.storage.cos; |
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.common.utils.PropertyUtils; |
||||||
|
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; |
||||||
|
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperatorFactory; |
||||||
|
import org.apache.dolphinscheduler.plugin.storage.api.StorageType; |
||||||
|
import org.apache.dolphinscheduler.plugin.storage.api.constants.StorageConstants; |
||||||
|
|
||||||
|
import java.util.Map; |
||||||
|
|
||||||
|
import com.google.auto.service.AutoService; |
||||||
|
|
||||||
|
@AutoService(StorageOperatorFactory.class) |
||||||
|
public class CosStorageOperatorFactory implements StorageOperatorFactory { |
||||||
|
|
||||||
|
@Override |
||||||
|
public StorageOperator createStorageOperate() { |
||||||
|
final CosStorageProperties cosStorageProperties = getCosStorageProperties(); |
||||||
|
return new CosStorageOperator(cosStorageProperties); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public StorageType getStorageOperate() { |
||||||
|
return StorageType.COS; |
||||||
|
} |
||||||
|
|
||||||
|
private CosStorageProperties getCosStorageProperties() { |
||||||
|
|
||||||
|
Map<String, String> cosPropertiesMap = |
||||||
|
PropertyUtils.getByPrefix(CosStorageConstants.TENCENT_CLOUD_COS_PROPERTY_PREFIX); |
||||||
|
|
||||||
|
return CosStorageProperties.builder() |
||||||
|
.region(cosPropertiesMap.get(CosStorageConstants.TENCENT_CLOUD_COS_REGION)) |
||||||
|
.accessKeyId(cosPropertiesMap.get(CosStorageConstants.TENCENT_CLOUD_ACCESS_KEY_ID)) |
||||||
|
.accessKeySecret(cosPropertiesMap.get(CosStorageConstants.TENCENT_CLOUD_ACCESS_KEY_SECRET)) |
||||||
|
.bucketName(cosPropertiesMap.get(CosStorageConstants.TENCENT_CLOUD_COS_BUCKET_NAME)) |
||||||
|
.resourceUploadPath( |
||||||
|
cosPropertiesMap.getOrDefault(StorageConstants.RESOURCE_UPLOAD_PATH, |
||||||
|
CosStorageConstants.DEFAULT_COS_RESOURCE_UPLOAD_PATH)) |
||||||
|
.build(); |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,40 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.plugin.storage.cos; |
||||||
|
|
||||||
|
import lombok.AllArgsConstructor; |
||||||
|
import lombok.Builder; |
||||||
|
import lombok.Data; |
||||||
|
import lombok.NoArgsConstructor; |
||||||
|
|
||||||
|
@Data |
||||||
|
@Builder |
||||||
|
@AllArgsConstructor |
||||||
|
@NoArgsConstructor |
||||||
|
public class CosStorageProperties { |
||||||
|
|
||||||
|
private String region; |
||||||
|
|
||||||
|
private String accessKeyId; |
||||||
|
|
||||||
|
private String accessKeySecret; |
||||||
|
|
||||||
|
private String bucketName; |
||||||
|
|
||||||
|
private String resourceUploadPath; |
||||||
|
} |
||||||
@ -0,0 +1,21 @@ |
|||||||
|
<?xml version="1.0" encoding="UTF-8"?> |
||||||
|
<!-- |
||||||
|
~ Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
~ contributor license agreements. See the NOTICE file distributed with |
||||||
|
~ this work for additional information regarding copyright ownership. |
||||||
|
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
~ (the "License"); you may not use this file except in compliance with |
||||||
|
~ the License. You may obtain a copy of the License at |
||||||
|
~ |
||||||
|
~ http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
~ |
||||||
|
~ Unless required by applicable law or agreed to in writing, software |
||||||
|
~ distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
~ See the License for the specific language governing permissions and |
||||||
|
~ limitations under the License. |
||||||
|
--> |
||||||
|
|
||||||
|
<configuration> |
||||||
|
<logger name="*" level="ERROR"/> |
||||||
|
</configuration> |
||||||
Loading…
Reference in new issue