Wenjun Ruan
2 years ago
committed by
GitHub
128 changed files with 1479 additions and 1214 deletions
@ -1,60 +0,0 @@ |
|||||||
/* |
|
||||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
|
||||||
* contributor license agreements. See the NOTICE file distributed with |
|
||||||
* this work for additional information regarding copyright ownership. |
|
||||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
|
||||||
* (the "License"); you may not use this file except in compliance with |
|
||||||
* the License. You may obtain a copy of the License at |
|
||||||
* |
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
* |
|
||||||
* Unless required by applicable law or agreed to in writing, software |
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
* See the License for the specific language governing permissions and |
|
||||||
* limitations under the License. |
|
||||||
*/ |
|
||||||
|
|
||||||
package org.apache.dolphinscheduler.service.storage; |
|
||||||
|
|
||||||
import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_STORAGE_TYPE; |
|
||||||
import static org.apache.dolphinscheduler.common.constants.Constants.STORAGE_HDFS; |
|
||||||
import static org.apache.dolphinscheduler.common.constants.Constants.STORAGE_OSS; |
|
||||||
import static org.apache.dolphinscheduler.common.constants.Constants.STORAGE_S3; |
|
||||||
|
|
||||||
import org.apache.dolphinscheduler.common.utils.PropertyUtils; |
|
||||||
import org.apache.dolphinscheduler.service.storage.impl.HadoopUtils; |
|
||||||
import org.apache.dolphinscheduler.service.storage.impl.OssOperator; |
|
||||||
import org.apache.dolphinscheduler.service.storage.impl.S3Utils; |
|
||||||
|
|
||||||
import org.springframework.context.annotation.Bean; |
|
||||||
import org.springframework.context.annotation.Configuration; |
|
||||||
import org.springframework.stereotype.Component; |
|
||||||
|
|
||||||
/** |
|
||||||
* choose the impl of storage by RESOURCE_STORAGE_TYPE |
|
||||||
*/ |
|
||||||
|
|
||||||
@Component |
|
||||||
@Configuration |
|
||||||
public class StoreConfiguration { |
|
||||||
|
|
||||||
@Bean |
|
||||||
public StorageOperate storageOperate() { |
|
||||||
switch (PropertyUtils.getUpperCaseString(RESOURCE_STORAGE_TYPE)) { |
|
||||||
case STORAGE_OSS: |
|
||||||
OssOperator ossOperator = new OssOperator(); |
|
||||||
// TODO: change to use ossOperator.init(ossConnection) after DS supports Configuration / Connection
|
|
||||||
// Center
|
|
||||||
ossOperator.init(); |
|
||||||
return ossOperator; |
|
||||||
case STORAGE_S3: |
|
||||||
return S3Utils.getInstance(); |
|
||||||
case STORAGE_HDFS: |
|
||||||
return HadoopUtils.getInstance(); |
|
||||||
default: |
|
||||||
return null; |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
} |
|
||||||
@ -1,56 +0,0 @@ |
|||||||
/* |
|
||||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
|
||||||
* contributor license agreements. See the NOTICE file distributed with |
|
||||||
* this work for additional information regarding copyright ownership. |
|
||||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
|
||||||
* (the "License"); you may not use this file except in compliance with |
|
||||||
* the License. You may obtain a copy of the License at |
|
||||||
* |
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
* |
|
||||||
* Unless required by applicable law or agreed to in writing, software |
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
* See the License for the specific language governing permissions and |
|
||||||
* limitations under the License. |
|
||||||
*/ |
|
||||||
package org.apache.dolphinscheduler.service.log; |
|
||||||
|
|
||||||
import org.apache.dolphinscheduler.common.constants.Constants; |
|
||||||
|
|
||||||
import org.junit.jupiter.api.Assertions; |
|
||||||
import org.junit.jupiter.api.Test; |
|
||||||
|
|
||||||
import ch.qos.logback.classic.Level; |
|
||||||
import ch.qos.logback.classic.spi.LoggingEvent; |
|
||||||
import ch.qos.logback.core.spi.FilterReply; |
|
||||||
|
|
||||||
public class MasterLogFilterTest { |
|
||||||
|
|
||||||
@Test |
|
||||||
public void decide() { |
|
||||||
MasterLogFilter masterLogFilter = new MasterLogFilter(); |
|
||||||
|
|
||||||
FilterReply filterReply = masterLogFilter.decide(new LoggingEvent() { |
|
||||||
|
|
||||||
@Override |
|
||||||
public String getThreadName() { |
|
||||||
return Constants.THREAD_NAME_MASTER_SERVER; |
|
||||||
} |
|
||||||
|
|
||||||
@Override |
|
||||||
public Level getLevel() { |
|
||||||
return Level.INFO; |
|
||||||
} |
|
||||||
|
|
||||||
@Override |
|
||||||
public String getMessage() { |
|
||||||
return "master insert into queue success, task : shell2"; |
|
||||||
} |
|
||||||
|
|
||||||
}); |
|
||||||
|
|
||||||
Assertions.assertEquals(FilterReply.ACCEPT, filterReply); |
|
||||||
|
|
||||||
} |
|
||||||
} |
|
||||||
@ -1,66 +0,0 @@ |
|||||||
/* |
|
||||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
|
||||||
* contributor license agreements. See the NOTICE file distributed with |
|
||||||
* this work for additional information regarding copyright ownership. |
|
||||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
|
||||||
* (the "License"); you may not use this file except in compliance with |
|
||||||
* the License. You may obtain a copy of the License at |
|
||||||
* |
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
* |
|
||||||
* Unless required by applicable law or agreed to in writing, software |
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
* See the License for the specific language governing permissions and |
|
||||||
* limitations under the License. |
|
||||||
*/ |
|
||||||
package org.apache.dolphinscheduler.service.log; |
|
||||||
|
|
||||||
import org.apache.dolphinscheduler.common.constants.Constants; |
|
||||||
|
|
||||||
import org.junit.jupiter.api.Assertions; |
|
||||||
import org.junit.jupiter.api.Test; |
|
||||||
|
|
||||||
import ch.qos.logback.classic.Level; |
|
||||||
import ch.qos.logback.classic.spi.LoggingEvent; |
|
||||||
import ch.qos.logback.core.spi.FilterReply; |
|
||||||
|
|
||||||
public class WorkerLogFilterTest { |
|
||||||
|
|
||||||
@Test |
|
||||||
public void decide() { |
|
||||||
WorkerLogFilter workerLogFilter = new WorkerLogFilter(); |
|
||||||
|
|
||||||
FilterReply filterReply = workerLogFilter.decide(new LoggingEvent() { |
|
||||||
|
|
||||||
@Override |
|
||||||
public String getThreadName() { |
|
||||||
return Constants.THREAD_NAME_WORKER_SERVER; |
|
||||||
} |
|
||||||
|
|
||||||
@Override |
|
||||||
public Level getLevel() { |
|
||||||
return Level.INFO; |
|
||||||
} |
|
||||||
|
|
||||||
@Override |
|
||||||
public String getMessage() { |
|
||||||
return "consume tasks: [2_177_2_704_-1],there still have 0 tasks need to be executed"; |
|
||||||
} |
|
||||||
|
|
||||||
@Override |
|
||||||
public Object[] getArgumentArray() { |
|
||||||
return new Object[0]; |
|
||||||
} |
|
||||||
|
|
||||||
@Override |
|
||||||
public String getFormattedMessage() { |
|
||||||
return "consume tasks: [2_177_2_704_-1],there still have 0 tasks need to be executed"; |
|
||||||
} |
|
||||||
|
|
||||||
}); |
|
||||||
|
|
||||||
Assertions.assertEquals(FilterReply.ACCEPT, filterReply); |
|
||||||
|
|
||||||
} |
|
||||||
} |
|
||||||
@ -0,0 +1,52 @@ |
|||||||
|
<?xml version="1.0" encoding="UTF-8"?> |
||||||
|
<!-- |
||||||
|
~ Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
~ contributor license agreements. See the NOTICE file distributed with |
||||||
|
~ this work for additional information regarding copyright ownership. |
||||||
|
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
~ (the "License"); you may not use this file except in compliance with |
||||||
|
~ the License. You may obtain a copy of the License at |
||||||
|
~ |
||||||
|
~ http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
~ |
||||||
|
~ Unless required by applicable law or agreed to in writing, software |
||||||
|
~ distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
~ See the License for the specific language governing permissions and |
||||||
|
~ limitations under the License. |
||||||
|
--> |
||||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" |
||||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> |
||||||
|
<modelVersion>4.0.0</modelVersion> |
||||||
|
<parent> |
||||||
|
<groupId>org.apache.dolphinscheduler</groupId> |
||||||
|
<artifactId>dolphinscheduler-storage-plugin</artifactId> |
||||||
|
<version>dev-SNAPSHOT</version> |
||||||
|
</parent> |
||||||
|
|
||||||
|
<artifactId>dolphinscheduler-storage-all</artifactId> |
||||||
|
|
||||||
|
<dependencies> |
||||||
|
<dependency> |
||||||
|
<groupId>org.apache.dolphinscheduler</groupId> |
||||||
|
<artifactId>dolphinscheduler-storage-api</artifactId> |
||||||
|
<version>${project.version}</version> |
||||||
|
</dependency> |
||||||
|
<dependency> |
||||||
|
<groupId>org.apache.dolphinscheduler</groupId> |
||||||
|
<artifactId>dolphinscheduler-storage-s3</artifactId> |
||||||
|
<version>${project.version}</version> |
||||||
|
</dependency> |
||||||
|
<dependency> |
||||||
|
<groupId>org.apache.dolphinscheduler</groupId> |
||||||
|
<artifactId>dolphinscheduler-storage-hdfs</artifactId> |
||||||
|
<version>${project.version}</version> |
||||||
|
</dependency> |
||||||
|
<dependency> |
||||||
|
<groupId>org.apache.dolphinscheduler</groupId> |
||||||
|
<artifactId>dolphinscheduler-storage-oss</artifactId> |
||||||
|
<version>${project.version}</version> |
||||||
|
</dependency> |
||||||
|
</dependencies> |
||||||
|
|
||||||
|
</project> |
||||||
@ -0,0 +1,36 @@ |
|||||||
|
<?xml version="1.0" encoding="UTF-8"?> |
||||||
|
<!-- |
||||||
|
~ Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
~ contributor license agreements. See the NOTICE file distributed with |
||||||
|
~ this work for additional information regarding copyright ownership. |
||||||
|
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
~ (the "License"); you may not use this file except in compliance with |
||||||
|
~ the License. You may obtain a copy of the License at |
||||||
|
~ |
||||||
|
~ http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
~ |
||||||
|
~ Unless required by applicable law or agreed to in writing, software |
||||||
|
~ distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
~ See the License for the specific language governing permissions and |
||||||
|
~ limitations under the License. |
||||||
|
--> |
||||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" |
||||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> |
||||||
|
<modelVersion>4.0.0</modelVersion> |
||||||
|
<parent> |
||||||
|
<groupId>org.apache.dolphinscheduler</groupId> |
||||||
|
<artifactId>dolphinscheduler-storage-plugin</artifactId> |
||||||
|
<version>dev-SNAPSHOT</version> |
||||||
|
</parent> |
||||||
|
|
||||||
|
<artifactId>dolphinscheduler-storage-api</artifactId> |
||||||
|
|
||||||
|
<dependencies> |
||||||
|
<dependency> |
||||||
|
<groupId>org.apache.dolphinscheduler</groupId> |
||||||
|
<artifactId>dolphinscheduler-spi</artifactId> |
||||||
|
<version>${project.version}</version> |
||||||
|
</dependency> |
||||||
|
</dependencies> |
||||||
|
</project> |
||||||
@ -0,0 +1,50 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.plugin.storage.api; |
||||||
|
|
||||||
|
import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_STORAGE_TYPE; |
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.common.utils.PropertyUtils; |
||||||
|
|
||||||
|
import java.util.Optional; |
||||||
|
import java.util.ServiceLoader; |
||||||
|
|
||||||
|
import org.springframework.context.annotation.Bean; |
||||||
|
import org.springframework.context.annotation.Configuration; |
||||||
|
|
||||||
|
// todo: If we move the config to yaml
|
||||||
|
@Configuration |
||||||
|
public class StorageConfiguration { |
||||||
|
|
||||||
|
@Bean |
||||||
|
public StorageOperate storageOperate() { |
||||||
|
Optional<StorageType> storageTypeOptional = |
||||||
|
StorageType.getStorageType(PropertyUtils.getUpperCaseString(RESOURCE_STORAGE_TYPE)); |
||||||
|
Optional<StorageOperate> storageOperate = storageTypeOptional.map(storageType -> { |
||||||
|
ServiceLoader<StorageOperateFactory> storageOperateFactories = |
||||||
|
ServiceLoader.load(StorageOperateFactory.class); |
||||||
|
for (StorageOperateFactory storageOperateFactory : storageOperateFactories) { |
||||||
|
if (storageOperateFactory.getStorageOperate() == storageType) { |
||||||
|
return storageOperateFactory.createStorageOperate(); |
||||||
|
} |
||||||
|
} |
||||||
|
return null; |
||||||
|
}); |
||||||
|
return storageOperate.orElse(null); |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,53 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.plugin.storage.api; |
||||||
|
|
||||||
|
import java.util.Optional; |
||||||
|
|
||||||
|
public enum StorageType { |
||||||
|
|
||||||
|
HDFS(0, "HDFS"), |
||||||
|
OSS(1, "OSS"), |
||||||
|
S3(2, "S3"), |
||||||
|
; |
||||||
|
|
||||||
|
private final int code; |
||||||
|
private final String name; |
||||||
|
|
||||||
|
StorageType(int code, String name) { |
||||||
|
this.code = code; |
||||||
|
this.name = name; |
||||||
|
} |
||||||
|
|
||||||
|
public int getCode() { |
||||||
|
return code; |
||||||
|
} |
||||||
|
|
||||||
|
public String getName() { |
||||||
|
return name; |
||||||
|
} |
||||||
|
|
||||||
|
public static Optional<StorageType> getStorageType(String name) { |
||||||
|
for (StorageType storageType : StorageType.values()) { |
||||||
|
if (storageType.getName().equals(name)) { |
||||||
|
return Optional.of(storageType); |
||||||
|
} |
||||||
|
} |
||||||
|
return Optional.empty(); |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,232 @@ |
|||||||
|
<?xml version="1.0" encoding="UTF-8"?> |
||||||
|
<!-- |
||||||
|
~ Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
~ contributor license agreements. See the NOTICE file distributed with |
||||||
|
~ this work for additional information regarding copyright ownership. |
||||||
|
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
~ (the "License"); you may not use this file except in compliance with |
||||||
|
~ the License. You may obtain a copy of the License at |
||||||
|
~ |
||||||
|
~ http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
~ |
||||||
|
~ Unless required by applicable law or agreed to in writing, software |
||||||
|
~ distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
~ See the License for the specific language governing permissions and |
||||||
|
~ limitations under the License. |
||||||
|
--> |
||||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" |
||||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> |
||||||
|
<modelVersion>4.0.0</modelVersion> |
||||||
|
<parent> |
||||||
|
<groupId>org.apache.dolphinscheduler</groupId> |
||||||
|
<artifactId>dolphinscheduler-storage-plugin</artifactId> |
||||||
|
<version>dev-SNAPSHOT</version> |
||||||
|
</parent> |
||||||
|
|
||||||
|
<artifactId>dolphinscheduler-storage-hdfs</artifactId> |
||||||
|
|
||||||
|
<dependencies> |
||||||
|
<!-- This is used to load Kerberos Conf --> |
||||||
|
<dependency> |
||||||
|
<groupId>org.apache.dolphinscheduler</groupId> |
||||||
|
<artifactId>dolphinscheduler-datasource-api</artifactId> |
||||||
|
</dependency> |
||||||
|
<dependency> |
||||||
|
<groupId>org.apache.dolphinscheduler</groupId> |
||||||
|
<artifactId>dolphinscheduler-storage-api</artifactId> |
||||||
|
</dependency> |
||||||
|
<dependency> |
||||||
|
<groupId>org.apache.hadoop</groupId> |
||||||
|
<artifactId>hadoop-common</artifactId> |
||||||
|
<scope>provided</scope> |
||||||
|
<exclusions> |
||||||
|
<exclusion> |
||||||
|
<groupId>org.slf4j</groupId> |
||||||
|
<artifactId>slf4j-log4j12</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>jdk.tools</groupId> |
||||||
|
<artifactId>jdk.tools</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>javax.servlet</groupId> |
||||||
|
<artifactId>servlet-api</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>javax.servlet</groupId> |
||||||
|
<artifactId>servlet-api</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>log4j</groupId> |
||||||
|
<artifactId>log4j</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>org.apache.curator</groupId> |
||||||
|
<artifactId>curator-client</artifactId> |
||||||
|
</exclusion> |
||||||
|
|
||||||
|
<exclusion> |
||||||
|
<groupId>commons-configuration</groupId> |
||||||
|
<artifactId>commons-configuration</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>io.grpc</groupId> |
||||||
|
<artifactId>grpc-protobuf</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>io.netty</groupId> |
||||||
|
<artifactId>netty</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>org.codehaus.jackson</groupId> |
||||||
|
<artifactId>jackson-core-asl</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>org.codehaus.jackson</groupId> |
||||||
|
<artifactId>jackson-mapper-asl</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>com.google.protobuf</groupId> |
||||||
|
<artifactId>jackson-mapper-asl</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>com.google.code.gson</groupId> |
||||||
|
<artifactId>gson</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>xmlenc</groupId> |
||||||
|
<artifactId>xmlenc</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>commons-net</groupId> |
||||||
|
<artifactId>commons-net</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>org.apache.avro</groupId> |
||||||
|
<artifactId>avro</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>org.apache.zookeeper</groupId> |
||||||
|
<artifactId>zookeeper</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>javax.servlet.jsp</groupId> |
||||||
|
<artifactId>jsp-api</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>com.sun.jersey</groupId> |
||||||
|
<artifactId>jersey-json</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>com.sun.jersey</groupId> |
||||||
|
<artifactId>jersey-server</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>com.sun.jersey</groupId> |
||||||
|
<artifactId>jersey-core</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>org.mortbay.jetty</groupId> |
||||||
|
<artifactId>jetty</artifactId> |
||||||
|
</exclusion> |
||||||
|
</exclusions> |
||||||
|
</dependency> |
||||||
|
<dependency> |
||||||
|
<groupId>org.apache.hadoop</groupId> |
||||||
|
<artifactId>hadoop-client</artifactId> |
||||||
|
<scope>provided</scope> |
||||||
|
<exclusions> |
||||||
|
<exclusion> |
||||||
|
<groupId>org.slf4j</groupId> |
||||||
|
<artifactId>slf4j-log4j12</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>javax.servlet</groupId> |
||||||
|
<artifactId>servlet-api</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>org.codehaus.jackson</groupId> |
||||||
|
<artifactId>jackson-jaxrs</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>org.codehaus.jackson</groupId> |
||||||
|
<artifactId>jackson-xc</artifactId> |
||||||
|
</exclusion> |
||||||
|
|
||||||
|
<exclusion> |
||||||
|
<groupId>org.fusesource.leveldbjni</groupId> |
||||||
|
<artifactId>leveldbjni-all</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>org.apache.zookeeper</groupId> |
||||||
|
<artifactId>zookeeper</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>org.apache.hadoop</groupId> |
||||||
|
<artifactId>hadoop-mapreduce-client-shuffle</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>com.sun.jersey</groupId> |
||||||
|
<artifactId>jersey-client</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>com.sun.jersey</groupId> |
||||||
|
<artifactId>jersey-core</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>javax.xml.bind</groupId> |
||||||
|
<artifactId>jaxb-api</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>log4j</groupId> |
||||||
|
<artifactId>log4j</artifactId> |
||||||
|
</exclusion> |
||||||
|
</exclusions> |
||||||
|
</dependency> |
||||||
|
|
||||||
|
<dependency> |
||||||
|
<groupId>org.apache.hadoop</groupId> |
||||||
|
<artifactId>hadoop-hdfs</artifactId> |
||||||
|
<scope>provided</scope> |
||||||
|
<exclusions> |
||||||
|
<exclusion> |
||||||
|
<groupId>javax.servlet</groupId> |
||||||
|
<artifactId>servlet-api</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>io.netty</groupId> |
||||||
|
<artifactId>netty</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>com.google.protobuf</groupId> |
||||||
|
<artifactId>protobuf-java</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>xmlenc</groupId> |
||||||
|
<artifactId>xmlenc</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>io.netty</groupId> |
||||||
|
<artifactId>netty-all</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>org.fusesource.leveldbjni</groupId> |
||||||
|
<artifactId>leveldbjni-all</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>com.sun.jersey</groupId> |
||||||
|
<artifactId>jersey-core</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>com.sun.jersey</groupId> |
||||||
|
<artifactId>jersey-server</artifactId> |
||||||
|
</exclusion> |
||||||
|
<exclusion> |
||||||
|
<groupId>log4j</groupId> |
||||||
|
<artifactId>log4j</artifactId> |
||||||
|
</exclusion> |
||||||
|
</exclusions> |
||||||
|
</dependency> |
||||||
|
</dependencies> |
||||||
|
</project> |
||||||
@ -0,0 +1,42 @@ |
|||||||
|
<?xml version="1.0" encoding="UTF-8"?> |
||||||
|
<!-- |
||||||
|
~ Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
~ contributor license agreements. See the NOTICE file distributed with |
||||||
|
~ this work for additional information regarding copyright ownership. |
||||||
|
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
~ (the "License"); you may not use this file except in compliance with |
||||||
|
~ the License. You may obtain a copy of the License at |
||||||
|
~ |
||||||
|
~ http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
~ |
||||||
|
~ Unless required by applicable law or agreed to in writing, software |
||||||
|
~ distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
~ See the License for the specific language governing permissions and |
||||||
|
~ limitations under the License. |
||||||
|
--> |
||||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" |
||||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> |
||||||
|
<modelVersion>4.0.0</modelVersion> |
||||||
|
<parent> |
||||||
|
<groupId>org.apache.dolphinscheduler</groupId> |
||||||
|
<artifactId>dolphinscheduler-storage-plugin</artifactId> |
||||||
|
<version>dev-SNAPSHOT</version> |
||||||
|
</parent> |
||||||
|
|
||||||
|
<artifactId>dolphinscheduler-storage-oss</artifactId> |
||||||
|
|
||||||
|
<dependencies> |
||||||
|
<dependency> |
||||||
|
<groupId>org.apache.dolphinscheduler</groupId> |
||||||
|
<artifactId>dolphinscheduler-storage-api</artifactId> |
||||||
|
<version>${project.version}</version> |
||||||
|
</dependency> |
||||||
|
|
||||||
|
<dependency> |
||||||
|
<groupId>org.apache.dolphinscheduler</groupId> |
||||||
|
<artifactId>dolphinscheduler-task-api</artifactId> |
||||||
|
<version>${project.version}</version> |
||||||
|
</dependency> |
||||||
|
</dependencies> |
||||||
|
</project> |
||||||
@ -0,0 +1,46 @@ |
|||||||
|
<?xml version="1.0" encoding="UTF-8"?> |
||||||
|
<!-- |
||||||
|
~ Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
~ contributor license agreements. See the NOTICE file distributed with |
||||||
|
~ this work for additional information regarding copyright ownership. |
||||||
|
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
~ (the "License"); you may not use this file except in compliance with |
||||||
|
~ the License. You may obtain a copy of the License at |
||||||
|
~ |
||||||
|
~ http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
~ |
||||||
|
~ Unless required by applicable law or agreed to in writing, software |
||||||
|
~ distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
~ See the License for the specific language governing permissions and |
||||||
|
~ limitations under the License. |
||||||
|
--> |
||||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" |
||||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> |
||||||
|
<modelVersion>4.0.0</modelVersion> |
||||||
|
<parent> |
||||||
|
<groupId>org.apache.dolphinscheduler</groupId> |
||||||
|
<artifactId>dolphinscheduler-storage-plugin</artifactId> |
||||||
|
<version>dev-SNAPSHOT</version> |
||||||
|
</parent> |
||||||
|
|
||||||
|
<artifactId>dolphinscheduler-storage-s3</artifactId> |
||||||
|
|
||||||
|
<dependencies> |
||||||
|
<dependency> |
||||||
|
<groupId>org.apache.dolphinscheduler</groupId> |
||||||
|
<artifactId>dolphinscheduler-task-api</artifactId> |
||||||
|
</dependency> |
||||||
|
|
||||||
|
<dependency> |
||||||
|
<groupId>org.apache.dolphinscheduler</groupId> |
||||||
|
<artifactId>dolphinscheduler-storage-api</artifactId> |
||||||
|
<version>${project.version}</version> |
||||||
|
</dependency> |
||||||
|
|
||||||
|
<dependency> |
||||||
|
<groupId>com.amazonaws</groupId> |
||||||
|
<artifactId>aws-java-sdk-s3</artifactId> |
||||||
|
</dependency> |
||||||
|
</dependencies> |
||||||
|
</project> |
||||||
@ -0,0 +1,49 @@ |
|||||||
|
<?xml version="1.0" encoding="UTF-8"?> |
||||||
|
<!-- |
||||||
|
~ Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
~ contributor license agreements. See the NOTICE file distributed with |
||||||
|
~ this work for additional information regarding copyright ownership. |
||||||
|
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
~ (the "License"); you may not use this file except in compliance with |
||||||
|
~ the License. You may obtain a copy of the License at |
||||||
|
~ |
||||||
|
~ http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
~ |
||||||
|
~ Unless required by applicable law or agreed to in writing, software |
||||||
|
~ distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
~ See the License for the specific language governing permissions and |
||||||
|
~ limitations under the License. |
||||||
|
--> |
||||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" |
||||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> |
||||||
|
<modelVersion>4.0.0</modelVersion> |
||||||
|
<parent> |
||||||
|
<groupId>org.apache.dolphinscheduler</groupId> |
||||||
|
<artifactId>dolphinscheduler</artifactId> |
||||||
|
<version>dev-SNAPSHOT</version> |
||||||
|
</parent> |
||||||
|
|
||||||
|
<artifactId>dolphinscheduler-storage-plugin</artifactId> |
||||||
|
<packaging>pom</packaging> |
||||||
|
|
||||||
|
<modules> |
||||||
|
<module>dolphinscheduler-storage-api</module> |
||||||
|
<module>dolphinscheduler-storage-all</module> |
||||||
|
<module>dolphinscheduler-storage-s3</module> |
||||||
|
<module>dolphinscheduler-storage-hdfs</module> |
||||||
|
<module>dolphinscheduler-storage-oss</module> |
||||||
|
</modules> |
||||||
|
|
||||||
|
<dependencyManagement> |
||||||
|
<dependencies> |
||||||
|
<dependency> |
||||||
|
<groupId>org.apache.dolphinscheduler</groupId> |
||||||
|
<artifactId>dolphinscheduler-bom</artifactId> |
||||||
|
<version>${project.version}</version> |
||||||
|
<type>pom</type> |
||||||
|
<scope>import</scope> |
||||||
|
</dependency> |
||||||
|
</dependencies> |
||||||
|
</dependencyManagement> |
||||||
|
</project> |
||||||
@ -1,106 +0,0 @@ |
|||||||
/* |
|
||||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
|
||||||
* contributor license agreements. See the NOTICE file distributed with |
|
||||||
* this work for additional information regarding copyright ownership. |
|
||||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
|
||||||
* (the "License"); you may not use this file except in compliance with |
|
||||||
* the License. You may obtain a copy of the License at |
|
||||||
* |
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
* |
|
||||||
* Unless required by applicable law or agreed to in writing, software |
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
* See the License for the specific language governing permissions and |
|
||||||
* limitations under the License. |
|
||||||
*/ |
|
||||||
|
|
||||||
package org.apache.dolphinscheduler.plugin.task.api; |
|
||||||
|
|
||||||
import org.apache.dolphinscheduler.plugin.task.api.utils.OSUtils; |
|
||||||
|
|
||||||
import org.apache.commons.lang3.SystemUtils; |
|
||||||
|
|
||||||
import java.util.regex.Matcher; |
|
||||||
import java.util.regex.Pattern; |
|
||||||
|
|
||||||
import lombok.NonNull; |
|
||||||
|
|
||||||
import org.slf4j.Logger; |
|
||||||
import org.slf4j.LoggerFactory; |
|
||||||
|
|
||||||
public final class ProcessUtils { |
|
||||||
|
|
||||||
private static final Logger logger = LoggerFactory.getLogger(ProcessUtils.class); |
|
||||||
|
|
||||||
private ProcessUtils() { |
|
||||||
throw new IllegalStateException("Utility class"); |
|
||||||
} |
|
||||||
|
|
||||||
/** |
|
||||||
* Initialization regularization, solve the problem of pre-compilation performance, |
|
||||||
* avoid the thread safety problem of multi-thread operation |
|
||||||
*/ |
|
||||||
private static final Pattern MACPATTERN = Pattern.compile("-[+|-]-\\s(\\d+)"); |
|
||||||
|
|
||||||
/** |
|
||||||
* Expression of PID recognition in Windows scene |
|
||||||
*/ |
|
||||||
private static final Pattern WINDOWSATTERN = Pattern.compile("(\\d+)"); |
|
||||||
|
|
||||||
/** |
|
||||||
* kill tasks according to different task types. |
|
||||||
*/ |
|
||||||
public static boolean kill(@NonNull TaskExecutionContext request) { |
|
||||||
try { |
|
||||||
logger.info("Begin kill task instance, processId: {}", request.getProcessId()); |
|
||||||
int processId = request.getProcessId(); |
|
||||||
if (processId == 0) { |
|
||||||
logger.error("Task instance kill failed, processId is not exist"); |
|
||||||
return false; |
|
||||||
} |
|
||||||
|
|
||||||
String cmd = String.format("kill -9 %s", getPidsStr(processId)); |
|
||||||
cmd = OSUtils.getSudoCmd(request.getTenantCode(), cmd); |
|
||||||
logger.info("process id:{}, cmd:{}", processId, cmd); |
|
||||||
|
|
||||||
OSUtils.exeCmd(cmd); |
|
||||||
logger.info("Success kill task instance, processId: {}", request.getProcessId()); |
|
||||||
return true; |
|
||||||
} catch (Exception e) { |
|
||||||
logger.error("Kill task instance error, processId: {}", request.getProcessId(), e); |
|
||||||
return false; |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
/** |
|
||||||
* get pids str. |
|
||||||
* |
|
||||||
* @param processId process id |
|
||||||
* @return pids pid String |
|
||||||
* @throws Exception exception |
|
||||||
*/ |
|
||||||
public static String getPidsStr(int processId) throws Exception { |
|
||||||
StringBuilder sb = new StringBuilder(); |
|
||||||
Matcher mat = null; |
|
||||||
// pstree pid get sub pids
|
|
||||||
if (SystemUtils.IS_OS_MAC) { |
|
||||||
String pids = OSUtils.exeCmd(String.format("%s -sp %d", TaskConstants.PSTREE, processId)); |
|
||||||
if (null != pids) { |
|
||||||
mat = MACPATTERN.matcher(pids); |
|
||||||
} |
|
||||||
} else { |
|
||||||
String pids = OSUtils.exeCmd(String.format("%s -p %d", TaskConstants.PSTREE, processId)); |
|
||||||
mat = WINDOWSATTERN.matcher(pids); |
|
||||||
} |
|
||||||
|
|
||||||
if (null != mat) { |
|
||||||
while (mat.find()) { |
|
||||||
sb.append(mat.group(1)).append(" "); |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
return sb.toString().trim(); |
|
||||||
} |
|
||||||
|
|
||||||
} |
|
||||||
@ -0,0 +1,402 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.plugin.task.api.utils; |
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.common.constants.Constants; |
||||||
|
import org.apache.dolphinscheduler.common.exception.BaseException; |
||||||
|
import org.apache.dolphinscheduler.common.utils.HttpUtils; |
||||||
|
import org.apache.dolphinscheduler.common.utils.JSONUtils; |
||||||
|
import org.apache.dolphinscheduler.common.utils.KerberosHttpClient; |
||||||
|
import org.apache.dolphinscheduler.common.utils.PropertyUtils; |
||||||
|
import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; |
||||||
|
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; |
||||||
|
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; |
||||||
|
|
||||||
|
import org.apache.commons.lang3.StringUtils; |
||||||
|
import org.apache.commons.lang3.SystemUtils; |
||||||
|
|
||||||
|
import java.io.File; |
||||||
|
import java.nio.charset.StandardCharsets; |
||||||
|
import java.util.List; |
||||||
|
import java.util.regex.Matcher; |
||||||
|
import java.util.regex.Pattern; |
||||||
|
|
||||||
|
import lombok.NonNull; |
||||||
|
|
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
import com.fasterxml.jackson.databind.node.ObjectNode; |
||||||
|
|
||||||
|
public final class ProcessUtils { |
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(ProcessUtils.class); |
||||||
|
|
||||||
|
private ProcessUtils() { |
||||||
|
throw new IllegalStateException("Utility class"); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Initialization regularization, solve the problem of pre-compilation performance, |
||||||
|
* avoid the thread safety problem of multi-thread operation |
||||||
|
*/ |
||||||
|
private static final Pattern MACPATTERN = Pattern.compile("-[+|-]-\\s(\\d+)"); |
||||||
|
|
||||||
|
/** |
||||||
|
* Expression of PID recognition in Windows scene |
||||||
|
*/ |
||||||
|
private static final Pattern WINDOWSATTERN = Pattern.compile("(\\d+)"); |
||||||
|
|
||||||
|
private static final String RM_HA_IDS = PropertyUtils.getString(Constants.YARN_RESOURCEMANAGER_HA_RM_IDS); |
||||||
|
private static final String APP_ADDRESS = PropertyUtils.getString(Constants.YARN_APPLICATION_STATUS_ADDRESS); |
||||||
|
private static final String JOB_HISTORY_ADDRESS = |
||||||
|
PropertyUtils.getString(Constants.YARN_JOB_HISTORY_STATUS_ADDRESS); |
||||||
|
private static final int HADOOP_RESOURCE_MANAGER_HTTP_ADDRESS_PORT_VALUE = |
||||||
|
PropertyUtils.getInt(Constants.HADOOP_RESOURCE_MANAGER_HTTPADDRESS_PORT, 8088); |
||||||
|
|
||||||
|
/** |
||||||
|
* kill tasks according to different task types. |
||||||
|
*/ |
||||||
|
public static boolean kill(@NonNull TaskExecutionContext request) { |
||||||
|
try { |
||||||
|
logger.info("Begin kill task instance, processId: {}", request.getProcessId()); |
||||||
|
int processId = request.getProcessId(); |
||||||
|
if (processId == 0) { |
||||||
|
logger.error("Task instance kill failed, processId is not exist"); |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
String cmd = String.format("kill -9 %s", getPidsStr(processId)); |
||||||
|
cmd = OSUtils.getSudoCmd(request.getTenantCode(), cmd); |
||||||
|
logger.info("process id:{}, cmd:{}", processId, cmd); |
||||||
|
|
||||||
|
OSUtils.exeCmd(cmd); |
||||||
|
logger.info("Success kill task instance, processId: {}", request.getProcessId()); |
||||||
|
return true; |
||||||
|
} catch (Exception e) { |
||||||
|
logger.error("Kill task instance error, processId: {}", request.getProcessId(), e); |
||||||
|
return false; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get pids str. |
||||||
|
* |
||||||
|
* @param processId process id |
||||||
|
* @return pids pid String |
||||||
|
* @throws Exception exception |
||||||
|
*/ |
||||||
|
public static String getPidsStr(int processId) throws Exception { |
||||||
|
StringBuilder sb = new StringBuilder(); |
||||||
|
Matcher mat = null; |
||||||
|
// pstree pid get sub pids
|
||||||
|
if (SystemUtils.IS_OS_MAC) { |
||||||
|
String pids = OSUtils.exeCmd(String.format("%s -sp %d", TaskConstants.PSTREE, processId)); |
||||||
|
if (null != pids) { |
||||||
|
mat = MACPATTERN.matcher(pids); |
||||||
|
} |
||||||
|
} else { |
||||||
|
String pids = OSUtils.exeCmd(String.format("%s -p %d", TaskConstants.PSTREE, processId)); |
||||||
|
mat = WINDOWSATTERN.matcher(pids); |
||||||
|
} |
||||||
|
|
||||||
|
if (null != mat) { |
||||||
|
while (mat.find()) { |
||||||
|
sb.append(mat.group(1)).append(" "); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
return sb.toString().trim(); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* kill yarn application. |
||||||
|
* |
||||||
|
* @param appIds app id list |
||||||
|
* @param logger logger |
||||||
|
* @param tenantCode tenant code |
||||||
|
* @param executePath execute path |
||||||
|
*/ |
||||||
|
public static void cancelApplication(List<String> appIds, Logger logger, String tenantCode, String executePath) { |
||||||
|
if (appIds == null || appIds.isEmpty()) { |
||||||
|
return; |
||||||
|
} |
||||||
|
|
||||||
|
for (String appId : appIds) { |
||||||
|
try { |
||||||
|
TaskExecutionStatus applicationStatus = getApplicationStatus(appId); |
||||||
|
|
||||||
|
if (!applicationStatus.isFinished()) { |
||||||
|
String commandFile = String.format("%s/%s.kill", executePath, appId); |
||||||
|
String cmd = getKerberosInitCommand() + "yarn application -kill " + appId; |
||||||
|
execYarnKillCommand(logger, tenantCode, appId, commandFile, cmd); |
||||||
|
} |
||||||
|
} catch (Exception e) { |
||||||
|
logger.error("Get yarn application app id [{}}] status failed", appId, e); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get the state of an application |
||||||
|
* |
||||||
|
* @param applicationId application id |
||||||
|
* @return the return may be null or there may be other parse exceptions |
||||||
|
*/ |
||||||
|
public static TaskExecutionStatus getApplicationStatus(String applicationId) throws BaseException { |
||||||
|
if (StringUtils.isEmpty(applicationId)) { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
String result; |
||||||
|
String applicationUrl = getApplicationUrl(applicationId); |
||||||
|
logger.debug("generate yarn application url, applicationUrl={}", applicationUrl); |
||||||
|
|
||||||
|
String responseContent = Boolean.TRUE |
||||||
|
.equals(PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false)) |
||||||
|
? KerberosHttpClient.get(applicationUrl) |
||||||
|
: HttpUtils.get(applicationUrl); |
||||||
|
if (responseContent != null) { |
||||||
|
ObjectNode jsonObject = JSONUtils.parseObject(responseContent); |
||||||
|
if (!jsonObject.has("app")) { |
||||||
|
return TaskExecutionStatus.FAILURE; |
||||||
|
} |
||||||
|
result = jsonObject.path("app").path("finalStatus").asText(); |
||||||
|
|
||||||
|
} else { |
||||||
|
// may be in job history
|
||||||
|
String jobHistoryUrl = getJobHistoryUrl(applicationId); |
||||||
|
logger.debug("generate yarn job history application url, jobHistoryUrl={}", jobHistoryUrl); |
||||||
|
responseContent = Boolean.TRUE |
||||||
|
.equals(PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false)) |
||||||
|
? KerberosHttpClient.get(jobHistoryUrl) |
||||||
|
: HttpUtils.get(jobHistoryUrl); |
||||||
|
|
||||||
|
if (null != responseContent) { |
||||||
|
ObjectNode jsonObject = JSONUtils.parseObject(responseContent); |
||||||
|
if (!jsonObject.has("job")) { |
||||||
|
return TaskExecutionStatus.FAILURE; |
||||||
|
} |
||||||
|
result = jsonObject.path("job").path("state").asText(); |
||||||
|
} else { |
||||||
|
return TaskExecutionStatus.FAILURE; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
return getExecutionStatus(result); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get application url |
||||||
|
* if rmHaIds contains xx, it signs not use resourcemanager |
||||||
|
* otherwise: |
||||||
|
* if rmHaIds is empty, single resourcemanager enabled |
||||||
|
* if rmHaIds not empty: resourcemanager HA enabled |
||||||
|
* |
||||||
|
* @param applicationId application id |
||||||
|
* @return url of application |
||||||
|
*/ |
||||||
|
private static String getApplicationUrl(String applicationId) throws BaseException { |
||||||
|
|
||||||
|
String appUrl = StringUtils.isEmpty(RM_HA_IDS) ? APP_ADDRESS : getAppAddress(APP_ADDRESS, RM_HA_IDS); |
||||||
|
if (StringUtils.isBlank(appUrl)) { |
||||||
|
throw new BaseException("yarn application url generation failed"); |
||||||
|
} |
||||||
|
logger.debug("yarn application url:{}, applicationId:{}", appUrl, applicationId); |
||||||
|
return String.format(appUrl, HADOOP_RESOURCE_MANAGER_HTTP_ADDRESS_PORT_VALUE, applicationId); |
||||||
|
} |
||||||
|
|
||||||
|
private static String getJobHistoryUrl(String applicationId) { |
||||||
|
// eg:application_1587475402360_712719 -> job_1587475402360_712719
|
||||||
|
String jobId = applicationId.replace("application", "job"); |
||||||
|
return String.format(JOB_HISTORY_ADDRESS, jobId); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* build kill command for yarn application |
||||||
|
* |
||||||
|
* @param logger logger |
||||||
|
* @param tenantCode tenant code |
||||||
|
* @param appId app id |
||||||
|
* @param commandFile command file |
||||||
|
* @param cmd cmd |
||||||
|
*/ |
||||||
|
private static void execYarnKillCommand(Logger logger, String tenantCode, String appId, String commandFile, |
||||||
|
String cmd) { |
||||||
|
try { |
||||||
|
StringBuilder sb = new StringBuilder(); |
||||||
|
sb.append("#!/bin/sh\n"); |
||||||
|
sb.append("BASEDIR=$(cd `dirname $0`; pwd)\n"); |
||||||
|
sb.append("cd $BASEDIR\n"); |
||||||
|
|
||||||
|
sb.append("\n\n"); |
||||||
|
sb.append(cmd); |
||||||
|
|
||||||
|
File f = new File(commandFile); |
||||||
|
|
||||||
|
if (!f.exists()) { |
||||||
|
org.apache.commons.io.FileUtils.writeStringToFile(new File(commandFile), sb.toString(), |
||||||
|
StandardCharsets.UTF_8); |
||||||
|
} |
||||||
|
|
||||||
|
String runCmd = String.format("%s %s", Constants.SH, commandFile); |
||||||
|
runCmd = org.apache.dolphinscheduler.common.utils.OSUtils.getSudoCmd(tenantCode, runCmd); |
||||||
|
logger.info("kill cmd:{}", runCmd); |
||||||
|
org.apache.dolphinscheduler.common.utils.OSUtils.exeCmd(runCmd); |
||||||
|
} catch (Exception e) { |
||||||
|
logger.error(String.format("Kill yarn application app id [%s] failed: [%s]", appId, e.getMessage())); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
private static TaskExecutionStatus getExecutionStatus(String result) { |
||||||
|
switch (result) { |
||||||
|
case Constants.ACCEPTED: |
||||||
|
return TaskExecutionStatus.SUBMITTED_SUCCESS; |
||||||
|
case Constants.SUCCEEDED: |
||||||
|
case Constants.ENDED: |
||||||
|
return TaskExecutionStatus.SUCCESS; |
||||||
|
case Constants.NEW: |
||||||
|
case Constants.NEW_SAVING: |
||||||
|
case Constants.SUBMITTED: |
||||||
|
case Constants.FAILED: |
||||||
|
return TaskExecutionStatus.FAILURE; |
||||||
|
case Constants.KILLED: |
||||||
|
return TaskExecutionStatus.KILL; |
||||||
|
case Constants.RUNNING: |
||||||
|
default: |
||||||
|
return TaskExecutionStatus.RUNNING_EXECUTION; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* getAppAddress |
||||||
|
* |
||||||
|
* @param appAddress app address |
||||||
|
* @param rmHa resource manager ha |
||||||
|
* @return app address |
||||||
|
*/ |
||||||
|
private static String getAppAddress(String appAddress, String rmHa) { |
||||||
|
|
||||||
|
String[] split1 = appAddress.split(Constants.DOUBLE_SLASH); |
||||||
|
|
||||||
|
if (split1.length != 2) { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
String start = split1[0] + Constants.DOUBLE_SLASH; |
||||||
|
String[] split2 = split1[1].split(Constants.COLON); |
||||||
|
|
||||||
|
if (split2.length != 2) { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
String end = Constants.COLON + split2[1]; |
||||||
|
|
||||||
|
// get active ResourceManager
|
||||||
|
String activeRM = YarnHAAdminUtils.getActiveRMName(start, rmHa); |
||||||
|
|
||||||
|
if (StringUtils.isEmpty(activeRM)) { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
return start + activeRM + end; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get kerberos init command |
||||||
|
*/ |
||||||
|
private static String getKerberosInitCommand() { |
||||||
|
logger.info("get kerberos init command"); |
||||||
|
StringBuilder kerberosCommandBuilder = new StringBuilder(); |
||||||
|
boolean hadoopKerberosState = |
||||||
|
PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false); |
||||||
|
if (hadoopKerberosState) { |
||||||
|
kerberosCommandBuilder.append("export KRB5_CONFIG=") |
||||||
|
.append(PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH)) |
||||||
|
.append("\n\n") |
||||||
|
.append(String.format("kinit -k -t %s %s || true", |
||||||
|
PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH), |
||||||
|
PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME))) |
||||||
|
.append("\n\n"); |
||||||
|
logger.info("kerberos init command: {}", kerberosCommandBuilder); |
||||||
|
} |
||||||
|
return kerberosCommandBuilder.toString(); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* yarn ha admin utils |
||||||
|
*/ |
||||||
|
private static final class YarnHAAdminUtils { |
||||||
|
|
||||||
|
/** |
||||||
|
* get active resourcemanager node |
||||||
|
* |
||||||
|
* @param protocol http protocol |
||||||
|
* @param rmIds yarn ha ids |
||||||
|
* @return yarn active node |
||||||
|
*/ |
||||||
|
public static String getActiveRMName(String protocol, String rmIds) { |
||||||
|
|
||||||
|
String[] rmIdArr = rmIds.split(Constants.COMMA); |
||||||
|
|
||||||
|
String yarnUrl = protocol + "%s:" + HADOOP_RESOURCE_MANAGER_HTTP_ADDRESS_PORT_VALUE + "/ws/v1/cluster/info"; |
||||||
|
|
||||||
|
try { |
||||||
|
|
||||||
|
/** |
||||||
|
* send http get request to rm |
||||||
|
*/ |
||||||
|
|
||||||
|
for (String rmId : rmIdArr) { |
||||||
|
String state = getRMState(String.format(yarnUrl, rmId)); |
||||||
|
if (Constants.HADOOP_RM_STATE_ACTIVE.equals(state)) { |
||||||
|
return rmId; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
} catch (Exception e) { |
||||||
|
logger.error("yarn ha application url generation failed, message:{}", e.getMessage()); |
||||||
|
} |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get ResourceManager state |
||||||
|
*/ |
||||||
|
public static String getRMState(String url) { |
||||||
|
|
||||||
|
String retStr = Boolean.TRUE |
||||||
|
.equals(PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false)) |
||||||
|
? KerberosHttpClient.get(url) |
||||||
|
: HttpUtils.get(url); |
||||||
|
|
||||||
|
if (StringUtils.isEmpty(retStr)) { |
||||||
|
return null; |
||||||
|
} |
||||||
|
// to json
|
||||||
|
ObjectNode jsonObject = JSONUtils.parseObject(retStr); |
||||||
|
|
||||||
|
// get ResourceManager state
|
||||||
|
if (!jsonObject.has("clusterInfo")) { |
||||||
|
return null; |
||||||
|
} |
||||||
|
return jsonObject.get("clusterInfo").path("haState").asText(); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue