Browse Source
* cherry-pick [Bug] [Worker] Optimize the getAppId method to avoid worker OOM when kill task Co-authored-by: Wenjun Ruan <wenjun@apache.org>3.0.1-release
Kerwin
2 years ago
committed by
GitHub
17 changed files with 341 additions and 128 deletions
@ -0,0 +1,44 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.remote.command.log; |
||||
|
||||
import lombok.AllArgsConstructor; |
||||
import lombok.Data; |
||||
import lombok.NoArgsConstructor; |
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils; |
||||
import org.apache.dolphinscheduler.remote.command.Command; |
||||
import org.apache.dolphinscheduler.remote.command.CommandType; |
||||
|
||||
import java.io.Serializable; |
||||
|
||||
@Data |
||||
@AllArgsConstructor |
||||
@NoArgsConstructor |
||||
public class GetAppIdRequestCommand implements Serializable { |
||||
|
||||
private String logPath; |
||||
|
||||
public Command convert2Command() { |
||||
Command command = new Command(); |
||||
command.setType(CommandType.GET_APP_ID_REQUEST); |
||||
byte[] body = JSONUtils.toJsonByteArray(this); |
||||
command.setBody(body); |
||||
return command; |
||||
} |
||||
|
||||
} |
||||
@ -0,0 +1,44 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.remote.command.log; |
||||
|
||||
import lombok.AllArgsConstructor; |
||||
import lombok.Data; |
||||
import lombok.NoArgsConstructor; |
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils; |
||||
import org.apache.dolphinscheduler.remote.command.Command; |
||||
import org.apache.dolphinscheduler.remote.command.CommandType; |
||||
|
||||
import java.io.Serializable; |
||||
import java.util.List; |
||||
|
||||
@Data |
||||
@AllArgsConstructor |
||||
@NoArgsConstructor |
||||
public class GetAppIdResponseCommand implements Serializable { |
||||
|
||||
private List<String> appIds; |
||||
|
||||
public Command convert2Command(long opaque) { |
||||
Command command = new Command(opaque); |
||||
command.setType(CommandType.GET_APP_ID_RESPONSE); |
||||
byte[] body = JSONUtils.toJsonByteArray(this); |
||||
command.setBody(body); |
||||
return command; |
||||
} |
||||
} |
||||
@ -0,0 +1,75 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.plugin.task.api.utils; |
||||
|
||||
import lombok.NonNull; |
||||
import lombok.experimental.UtilityClass; |
||||
import lombok.extern.slf4j.Slf4j; |
||||
import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; |
||||
import org.slf4j.Logger; |
||||
|
||||
import java.io.File; |
||||
import java.io.IOException; |
||||
import java.nio.file.Files; |
||||
import java.nio.file.Paths; |
||||
import java.util.ArrayList; |
||||
import java.util.Collections; |
||||
import java.util.HashSet; |
||||
import java.util.List; |
||||
import java.util.Set; |
||||
import java.util.regex.Matcher; |
||||
import java.util.regex.Pattern; |
||||
import java.util.stream.Stream; |
||||
|
||||
@Slf4j |
||||
@UtilityClass |
||||
public class LogUtils { |
||||
|
||||
private static final Pattern APPLICATION_REGEX = Pattern.compile(TaskConstants.YARN_APPLICATION_REGEX); |
||||
|
||||
public List<String> getAppIdsFromLogFile(@NonNull String logPath) { |
||||
return getAppIdsFromLogFile(logPath, log); |
||||
} |
||||
|
||||
public List<String> getAppIdsFromLogFile(@NonNull String logPath, Logger logger) { |
||||
File logFile = new File(logPath); |
||||
if (!logFile.exists() || !logFile.isFile()) { |
||||
return Collections.emptyList(); |
||||
} |
||||
Set<String> appIds = new HashSet<>(); |
||||
try (Stream<String> stream = Files.lines(Paths.get(logPath))) { |
||||
stream.filter(line -> { |
||||
Matcher matcher = APPLICATION_REGEX.matcher(line); |
||||
return matcher.find(); |
||||
} |
||||
).forEach(line -> { |
||||
Matcher matcher = APPLICATION_REGEX.matcher(line); |
||||
if (matcher.find()) { |
||||
String appId = matcher.group(); |
||||
if (appIds.add(appId)) { |
||||
logger.info("Find appId: {} from {}", appId, logPath); |
||||
} |
||||
} |
||||
}); |
||||
return new ArrayList<>(appIds); |
||||
} catch (IOException e) { |
||||
logger.error("Get appId from log file erro, logPath: {}", logPath, e); |
||||
return Collections.emptyList(); |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,36 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.plugin.task.api.utils; |
||||
|
||||
import com.google.common.collect.Lists; |
||||
import org.junit.Assert; |
||||
import org.junit.Test; |
||||
|
||||
import java.util.List; |
||||
|
||||
public class LogUtilsTest { |
||||
|
||||
private static final String APP_ID_FILE = LogUtilsTest.class.getResource("/appId.txt") |
||||
.getFile(); |
||||
|
||||
@Test |
||||
public void getAppIdsFromLogFile() { |
||||
List<String> appIds = LogUtils.getAppIdsFromLogFile(APP_ID_FILE); |
||||
Assert.assertEquals(Lists.newArrayList("application_1548381669007_1234"), appIds); |
||||
} |
||||
} |
||||
@ -0,0 +1,29 @@
|
||||
# |
||||
# Licensed to the Apache Software Foundation (ASF) under one or more |
||||
# contributor license agreements. See the NOTICE file distributed with |
||||
# this work for additional information regarding copyright ownership. |
||||
# The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
# (the "License"); you may not use this file except in compliance with |
||||
# the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
# |
||||
|
||||
19/04/02 11:40:22 INFO yarn.Client: Application report for application_1548381669007_1234 (state: ACCEPTED) |
||||
19/04/02 11:40:23 INFO yarn.Client: Application report for application_1548381669007_1234 (state: ACCEPTED) |
||||
19/04/02 11:40:24 INFO yarn.Client: Application report for application_1548381669007_1234 (state: ACCEPTED) |
||||
19/04/02 11:40:25 INFO yarn.Client: Application report for application_1548381669007_1234 (state: ACCEPTED) |
||||
19/04/02 11:40:26 INFO yarn.Client: Application report for application_1548381669007_1234 (state: ACCEPTED) |
||||
19/04/02 11:40:27 INFO yarn.Client: Application report for application_1548381669007_1234 (state: ACCEPTED) |
||||
19/04/02 11:40:28 INFO yarn.Client: Application report for application_1548381669007_1234 (state: ACCEPTED) |
||||
19/04/02 11:40:29 INFO yarn.Client: Application report for application_1548381669007_1234 (state: ACCEPTED) |
||||
19/04/02 11:40:30 INFO yarn.Client: Application report for application_1548381669007_1234 (state: ACCEPTED) |
||||
19/04/02 11:40:31 INFO yarn.Client: Application report for application_1548381669007_1234 (state: ACCEPTED) |
||||
19/04/02 11:40:32 INFO yarn.Client: Application report for application_1548381669007_1234 (state: RUNNING) |
||||
19/04/02 11:40:33 INFO yarn.Client: Application report for application_1548381669007_1234 (state: RUNNING) |
||||
Loading…
Reference in new issue