Rick Cheng
1 year ago
committed by
GitHub
18 changed files with 652 additions and 6 deletions
@ -0,0 +1,35 @@
|
||||
# Remote Logging |
||||
|
||||
Apache DolphinScheduler supports writing task logs to remote storage. When remote logging is enabled, DolphinScheduler will send the task logs to the specified remote storage asynchronously after the task ends. In addition, when the user views or downloads the task log, if the log file does not exist locally, DolphinScheduler will download the corresponding log file from the remote storage to the local file system. |
||||
|
||||
## Enabling remote logging |
||||
|
||||
If you deploy DolphinScheduler in `Cluster` or `Pseudo-Cluster` mode, you need to configure `api-server/conf/common.properties`, `master-server/conf/common.properties` and `worker-server/conf/common.properties`. |
||||
If you deploy DolphinScheduler in `Standalone` mode, you only need to configure `standalone-server/conf/common.properties` as follows: |
||||
|
||||
```properties |
||||
# Whether to enable remote logging |
||||
remote.logging.enable=false |
||||
# if remote.logging.enable = true, set the target of remote logging |
||||
remote.logging.target=OSS |
||||
# if remote.logging.enable = true, set the log base directory |
||||
remote.logging.base.dir=logs |
||||
# if remote.logging.enable = true, set the number of threads to send logs to remote storage |
||||
remote.logging.thread.pool.size=10 |
||||
``` |
||||
|
||||
## Writing task logs to [Aliyun Object Storage Service (OSS)](https://www.aliyun.com/product/oss) |
||||
|
||||
Configure `common.properties` as follows: |
||||
|
||||
```properties |
||||
# oss access key id, required if you set remote.logging.target=OSS |
||||
remote.logging.oss.access.key.id=<access.key.id> |
||||
# oss access key secret, required if you set remote.logging.target=OSS |
||||
remote.logging.oss.access.key.secret=<access.key.secret> |
||||
# oss bucket name, required if you set remote.logging.target=OSS |
||||
remote.logging.oss.bucket.name=<bucket.name> |
||||
# oss endpoint, required if you set remote.logging.target=OSS |
||||
remote.logging.oss.endpoint=<endpoint> |
||||
``` |
||||
|
||||
@ -0,0 +1,35 @@
|
||||
# 远程日志存储（Remote Logging） |
||||
|
||||
Apache DolphinScheduler支持将任务日志传输到远端存储上。当配置开启远程日志存储后，DolphinScheduler将在任务结束后，将对应的任务日志异步地发送到指定的远端存储上。此外，用户在查看或下载任务日志时，若本地没有该日志文件，DolphinScheduler将从远端存储上下载对应的日志文件到本地文件系统。 |
||||
|
||||
## 开启远程日志存储 |
||||
|
||||
如果您以 `集群` 模式或者 `伪集群` 模式部署DolphinScheduler，您需要对以下路径的文件进行配置：`api-server/conf/common.properties`，`master-server/conf/common.properties`和 `worker-server/conf/common.properties`； |
||||
若您以 `单机` 模式部署DolphinScheduler，您只需要配置 `standalone-server/conf/common.properties`，具体配置如下： |
||||
|
||||
```properties |
||||
# 是否开启远程日志存储 |
||||
remote.logging.enable=true |
||||
# 任务日志写入的远端存储，目前仅支持OSS |
||||
remote.logging.target=OSS |
||||
# 任务日志在远端存储上的目录 |
||||
remote.logging.base.dir=logs |
||||
# 设置向远端存储异步发送日志的线程池大小 |
||||
remote.logging.thread.pool.size=10 |
||||
``` |
||||
|
||||
## 将任务日志写入[阿里云对象存储（OSS）](https://www.aliyun.com/product/oss) |
||||
|
||||
配置`common.propertis`如下： |
||||
|
||||
```properties |
||||
# oss access key id, required if you set remote.logging.target=OSS |
||||
remote.logging.oss.access.key.id=<access.key.id> |
||||
# oss access key secret, required if you set remote.logging.target=OSS |
||||
remote.logging.oss.access.key.secret=<access.key.secret> |
||||
# oss bucket name, required if you set remote.logging.target=OSS |
||||
remote.logging.oss.bucket.name=<bucket.name> |
||||
# oss endpoint, required if you set remote.logging.target=OSS |
||||
remote.logging.oss.endpoint=<endpoint> |
||||
``` |
||||
|
||||
@ -0,0 +1,143 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.log.remote; |
||||
|
||||
import org.apache.dolphinscheduler.common.constants.Constants; |
||||
import org.apache.dolphinscheduler.common.factory.OssClientFactory; |
||||
import org.apache.dolphinscheduler.common.model.OssConnection; |
||||
import org.apache.dolphinscheduler.common.utils.PropertyUtils; |
||||
|
||||
import org.apache.commons.lang3.StringUtils; |
||||
|
||||
import java.io.Closeable; |
||||
import java.io.File; |
||||
import java.io.IOException; |
||||
import java.nio.file.Path; |
||||
import java.nio.file.Paths; |
||||
|
||||
import lombok.extern.slf4j.Slf4j; |
||||
|
||||
import com.aliyun.oss.OSS; |
||||
import com.aliyun.oss.model.Bucket; |
||||
import com.aliyun.oss.model.GetObjectRequest; |
||||
import com.aliyun.oss.model.PutObjectRequest; |
||||
|
||||
@Slf4j |
||||
public class OssRemoteLogHandler implements RemoteLogHandler, Closeable { |
||||
|
||||
private static final int OBJECT_NAME_COUNT = 2; |
||||
|
||||
private OSS ossClient; |
||||
|
||||
private String bucketName; |
||||
|
||||
public OssRemoteLogHandler() { |
||||
} |
||||
|
||||
public void init() { |
||||
String accessKeyId = readOssAccessKeyId(); |
||||
String accessKeySecret = readOssAccessKeySecret(); |
||||
String endpoint = readOssEndpoint(); |
||||
ossClient = OssClientFactory.buildOssClient(new OssConnection(accessKeyId, accessKeySecret, endpoint)); |
||||
|
||||
bucketName = readOssBucketName(); |
||||
checkBucketNameExists(bucketName); |
||||
} |
||||
|
||||
@Override |
||||
public void sendRemoteLog(String logPath) { |
||||
String objectName = getObjectNameFromLogPath(logPath); |
||||
|
||||
try { |
||||
log.info("send remote log {} to OSS {}", logPath, objectName); |
||||
PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, objectName, new File(logPath)); |
||||
ossClient.putObject(putObjectRequest); |
||||
} catch (Exception e) { |
||||
log.error("error while sending remote log {} to OSS {}", logPath, objectName, e); |
||||
} |
||||
} |
||||
|
||||
@Override |
||||
public void getRemoteLog(String logPath) { |
||||
String objectName = getObjectNameFromLogPath(logPath); |
||||
|
||||
try { |
||||
log.info("get remote log on OSS {} to {}", objectName, logPath); |
||||
ossClient.getObject(new GetObjectRequest(bucketName, objectName), new File(logPath)); |
||||
} catch (Exception e) { |
||||
log.error("error while getting remote log on OSS {} to {}", objectName, logPath, e); |
||||
} |
||||
} |
||||
|
||||
@Override |
||||
public void close() throws IOException { |
||||
if (ossClient != null) { |
||||
ossClient.shutdown(); |
||||
} |
||||
} |
||||
|
||||
private String getObjectNameFromLogPath(String logPath) { |
||||
Path path = Paths.get(logPath); |
||||
int nameCount = path.getNameCount(); |
||||
|
||||
if (nameCount < OBJECT_NAME_COUNT) { |
||||
return Paths.get(readOssBaseDir(), logPath).toString(); |
||||
} else { |
||||
return Paths.get(readOssBaseDir(), path.subpath(nameCount - OBJECT_NAME_COUNT, nameCount).toString()) |
||||
.toString(); |
||||
} |
||||
} |
||||
|
||||
private void checkBucketNameExists(String bucketName) { |
||||
if (StringUtils.isBlank(bucketName)) { |
||||
throw new IllegalArgumentException(Constants.REMOTE_LOGGING_OSS_BUCKET_NAME + " is empty"); |
||||
} |
||||
|
||||
Bucket existsBucket = ossClient.listBuckets() |
||||
.stream() |
||||
.filter( |
||||
bucket -> bucket.getName().equals(bucketName)) |
||||
.findFirst() |
||||
.orElseThrow(() -> { |
||||
return new IllegalArgumentException( |
||||
"bucketName: " + bucketName + " does not exist, you need to create them by yourself"); |
||||
}); |
||||
|
||||
log.info("bucketName: {} has been found", existsBucket.getName()); |
||||
} |
||||
|
||||
private String readOssAccessKeyId() { |
||||
return PropertyUtils.getString(Constants.REMOTE_LOGGING_OSS_ACCESS_KEY_ID); |
||||
} |
||||
|
||||
private String readOssAccessKeySecret() { |
||||
return PropertyUtils.getString(Constants.REMOTE_LOGGING_OSS_ACCESS_KEY_SECRET); |
||||
} |
||||
|
||||
private String readOssEndpoint() { |
||||
return PropertyUtils.getString(Constants.REMOTE_LOGGING_OSS_ENDPOINT); |
||||
} |
||||
|
||||
private String readOssBucketName() { |
||||
return PropertyUtils.getString(Constants.REMOTE_LOGGING_OSS_BUCKET_NAME); |
||||
} |
||||
|
||||
private String readOssBaseDir() { |
||||
return PropertyUtils.getString(Constants.REMOTE_LOGGING_BASE_DIR); |
||||
} |
||||
} |
||||
@ -0,0 +1,44 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.log.remote; |
||||
|
||||
import org.apache.dolphinscheduler.common.constants.Constants; |
||||
import org.apache.dolphinscheduler.common.utils.PropertyUtils; |
||||
|
||||
import java.util.concurrent.Executor; |
||||
|
||||
import org.springframework.context.annotation.Bean; |
||||
import org.springframework.context.annotation.Configuration; |
||||
import org.springframework.scheduling.annotation.EnableAsync; |
||||
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; |
||||
|
||||
@Configuration |
||||
@EnableAsync |
||||
public class RemoteLogHandleThreadPool { |
||||
|
||||
@Bean |
||||
public Executor remoteLogHandleExecutor() { |
||||
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); |
||||
executor.setCorePoolSize(PropertyUtils.getInt(Constants.REMOTE_LOGGING_THREAD_POOL_SIZE, 10)); |
||||
executor.setMaxPoolSize(PropertyUtils.getInt(Constants.REMOTE_LOGGING_THREAD_POOL_SIZE, 10)); |
||||
executor.setThreadNamePrefix("remote-logging-"); |
||||
executor.initialize(); |
||||
|
||||
return executor; |
||||
} |
||||
} |
||||
@ -0,0 +1,25 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.log.remote; |
||||
|
||||
public interface RemoteLogHandler { |
||||
|
||||
void sendRemoteLog(String logPath); |
||||
|
||||
void getRemoteLog(String logPath); |
||||
} |
||||
@ -0,0 +1,39 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.log.remote; |
||||
|
||||
import org.apache.dolphinscheduler.common.constants.Constants; |
||||
import org.apache.dolphinscheduler.common.utils.PropertyUtils; |
||||
|
||||
import lombok.experimental.UtilityClass; |
||||
|
||||
@UtilityClass |
||||
public class RemoteLogHandlerFactory { |
||||
|
||||
public RemoteLogHandler getRemoteLogHandler() { |
||||
if (!RemoteLogUtils.isRemoteLoggingEnable()) { |
||||
return null; |
||||
} |
||||
if (!"OSS".equals(PropertyUtils.getUpperCaseString(Constants.REMOTE_LOGGING_TARGET))) { |
||||
return null; |
||||
} |
||||
OssRemoteLogHandler ossRemoteLogHandler = new OssRemoteLogHandler(); |
||||
ossRemoteLogHandler.init(); |
||||
return ossRemoteLogHandler; |
||||
} |
||||
} |
||||
@ -0,0 +1,48 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.log.remote; |
||||
|
||||
import org.apache.dolphinscheduler.common.constants.Constants; |
||||
import org.apache.dolphinscheduler.common.utils.PropertyUtils; |
||||
|
||||
import lombok.extern.slf4j.Slf4j; |
||||
|
||||
import org.springframework.scheduling.annotation.Async; |
||||
import org.springframework.stereotype.Component; |
||||
|
||||
@Component |
||||
@Slf4j |
||||
public class RemoteLogService { |
||||
|
||||
@Async("remoteLogHandleExecutor") |
||||
public void asyncSendRemoteLog(String logPath) { |
||||
if (RemoteLogUtils.isRemoteLoggingEnable()) { |
||||
log.info("Start to send log {} to remote target {}", logPath, |
||||
PropertyUtils.getString(Constants.REMOTE_LOGGING_TARGET)); |
||||
|
||||
RemoteLogHandler remoteLogHandler = RemoteLogHandlerFactory.getRemoteLogHandler(); |
||||
if (remoteLogHandler == null) { |
||||
log.error("remote log handler is null"); |
||||
return; |
||||
} |
||||
remoteLogHandler.sendRemoteLog(logPath); |
||||
log.info("Succeed to send log {} to remote target {}", logPath, |
||||
PropertyUtils.getString(Constants.REMOTE_LOGGING_TARGET)); |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,79 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.log.remote; |
||||
|
||||
import org.apache.dolphinscheduler.common.constants.Constants; |
||||
import org.apache.dolphinscheduler.common.utils.PropertyUtils; |
||||
|
||||
import java.nio.file.Path; |
||||
import java.nio.file.Paths; |
||||
|
||||
import javax.annotation.PostConstruct; |
||||
|
||||
import lombok.extern.slf4j.Slf4j; |
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired; |
||||
import org.springframework.stereotype.Component; |
||||
|
||||
@Component |
||||
@Slf4j |
||||
public class RemoteLogUtils { |
||||
|
||||
private static RemoteLogService remoteLogService; |
||||
|
||||
@Autowired |
||||
private RemoteLogService autowiredRemoteLogService; |
||||
|
||||
@PostConstruct |
||||
private void init() { |
||||
remoteLogService = autowiredRemoteLogService; |
||||
} |
||||
|
||||
public static void sendRemoteLog(String logPath) { |
||||
if (isRemoteLoggingEnable()) { |
||||
// send task logs to remote storage asynchronously
|
||||
remoteLogService.asyncSendRemoteLog(logPath); |
||||
} |
||||
} |
||||
|
||||
public static void getRemoteLog(String logPath) { |
||||
if (isRemoteLoggingEnable()) { |
||||
log.info("Start to get log {} from remote target {}", logPath, |
||||
PropertyUtils.getString(Constants.REMOTE_LOGGING_TARGET)); |
||||
|
||||
mkdirOfLog(logPath); |
||||
RemoteLogHandler remoteLogHandler = RemoteLogHandlerFactory.getRemoteLogHandler(); |
||||
if (remoteLogHandler == null) { |
||||
log.error("remote log handler is null"); |
||||
return; |
||||
} |
||||
remoteLogHandler.getRemoteLog(logPath); |
||||
log.info("End get log {} from remote target {}", logPath, |
||||
PropertyUtils.getString(Constants.REMOTE_LOGGING_TARGET)); |
||||
} |
||||
} |
||||
|
||||
private static void mkdirOfLog(String logPath) { |
||||
Path directory = Paths.get(logPath).getParent(); |
||||
directory.toFile().mkdirs(); |
||||
} |
||||
|
||||
public static boolean isRemoteLoggingEnable() { |
||||
return PropertyUtils.getBoolean(Constants.REMOTE_LOGGING_ENABLE, Boolean.FALSE); |
||||
} |
||||
} |
||||
@ -0,0 +1,53 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.log.remote; |
||||
|
||||
import org.apache.dolphinscheduler.common.constants.Constants; |
||||
import org.apache.dolphinscheduler.common.utils.PropertyUtils; |
||||
|
||||
import java.lang.reflect.Method; |
||||
|
||||
import org.junit.jupiter.api.Assertions; |
||||
import org.junit.jupiter.api.Test; |
||||
import org.junit.jupiter.api.extension.ExtendWith; |
||||
import org.mockito.MockedStatic; |
||||
import org.mockito.Mockito; |
||||
import org.mockito.junit.jupiter.MockitoExtension; |
||||
|
||||
@ExtendWith(MockitoExtension.class) |
||||
public class OssRemoteLogHandlerTest { |
||||
|
||||
@Test |
||||
public void testGetObjectNameFromLogPath() throws Exception { |
||||
final String logPath = "/path/to/dolphinscheduler/logs/20230116/8245922982496_1-1-3.log"; |
||||
final String expectedObjectName = "logs/20230116/8245922982496_1-1-3.log"; |
||||
|
||||
OssRemoteLogHandler ossRemoteLogHandler = new OssRemoteLogHandler(); |
||||
|
||||
try (MockedStatic<PropertyUtils> propertyUtilsMockedStatic = Mockito.mockStatic(PropertyUtils.class)) { |
||||
propertyUtilsMockedStatic.when(() -> PropertyUtils.getString(Constants.REMOTE_LOGGING_BASE_DIR)) |
||||
.thenReturn("logs"); |
||||
|
||||
Method method = OssRemoteLogHandler.class.getDeclaredMethod("getObjectNameFromLogPath", String.class); |
||||
method.setAccessible(true); |
||||
String objectName = (String) method.invoke(ossRemoteLogHandler, logPath); |
||||
|
||||
Assertions.assertEquals(expectedObjectName, objectName); |
||||
} |
||||
} |
||||
} |
||||
Loading…
Reference in new issue